Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

bench: fixes from the first real runs on release engines (LVM drive lookup, CUDA UMD banner, build runtime, engine log, fresh-load line, archive COMMIT)

Niko1221c644312d15ee
diff --git a/packaging/package_terminal.py b/packaging/package_terminal.py
index 7d1221061..6deee040e 100644
--- a/packaging/package_terminal.py
+++ b/packaging/package_terminal.py
@@ -281,6 +281,19 @@ def write_archive(root: Path, name: str, out_dir: Path, kind: str) -> Path:
return path
+def build_commit(explicit: str | None = None) -> str | None:
+ """The full commit this archive is built from: --commit, the STRATA_COMMIT variable, or `git rev-parse HEAD` of the
+ source tree. None when none of them gives 40 hex digits (a tree copy without .git)."""
+ for candidate in (explicit, os.environ.get("STRATA_COMMIT")):
+ if candidate and re.fullmatch(r"[0-9a-fA-F]{40}", candidate.strip()):
+ return candidate.strip().lower()
+ try:
+ out = subprocess.run(["git", "-C", str(ROOT), "rev-parse", "HEAD"], capture_output=True, text=True).stdout.strip().lower()
+ except OSError:
+ return None
+ return out if re.fullmatch(r"[0-9a-f]{40}", out) else None
+
+
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--out", required=True, help="directory to write the archive into")
@@ -289,6 +302,9 @@ def main() -> int:
ap.add_argument("--version", default=None, help="version label (default: read from the source)")
ap.add_argument("--skip-deps", action="store_true",
help="skip the locked install (layout checks only; the result is NOT distributable)")
+ ap.add_argument("--commit", default=None, metavar="SHA",
+ help="the full commit the sources come from, for a tree copy that has no .git (or set STRATA_COMMIT); "
+ "`strata bench run` records it, and a standard benchmark without it is nonconforming")
ap.add_argument("--keep-staging", action="store_true")
ap.add_argument("--no-installer", action="store_true",
help="leave out setup.py, tools/strata_mcp.py and data/ (the read-only terminal of earlier releases; "
@@ -302,6 +318,13 @@ def main() -> int:
if not args.skip_deps and target != host_target():
raise SystemExit(f"{target} must be built on {target} (this is {host_target()}): the locked install runs the "
"bundled interpreter. Use --skip-deps for a layout check.")
+ commit = build_commit(args.commit)
+ if commit is None:
+ message = ("the commit of these sources is unknown (no .git here, no --commit, no STRATA_COMMIT): the archive would carry "
+ "no COMMIT file, and every `strata bench run` on it would be nonconforming")
+ if not args.skip_deps:
+ raise SystemExit(message)
+ print("warning: " + message)
pin = PYTHON_PINS[target]
version = args.version or _source_version()
out_dir = Path(args.out).resolve()
@@ -345,9 +368,8 @@ def main() -> int:
installer = None if args.no_installer else add_installer(root)
(root / "VERSION").write_text(version + "\n", encoding="utf-8")
# the commit the archive was built from: `strata bench run` records it (an archive has no .git to ask)
- built = subprocess.run(["git", "-C", str(ROOT), "rev-parse", "HEAD"], capture_output=True, text=True).stdout.strip().lower()
- if re.fullmatch(r"[0-9a-f]{40}", built):
- (root / "COMMIT").write_text(built + "\n", encoding="utf-8")
+ if commit:
+ (root / "COMMIT").write_text(commit + "\n", encoding="utf-8")
bin_dir = root / "bin"
bin_dir.mkdir(exist_ok=True)
diff --git a/strata_cli/bench_facts.py b/strata_cli/bench_facts.py
index 7d3cc7d5d..309223bf7 100644
--- a/strata_cli/bench_facts.py
+++ b/strata_cli/bench_facts.py
@@ -166,7 +166,8 @@ def nvidia_gpus() -> tuple[list[dict], str | None, str | None]:
"memory_kind": "dedicated", "power_limit_w": watts, "driver": part[4]})
cuda = None
if cards:
- match = re.search(r"CUDA Version:\s*([\d.]+)", run_cmd(["nvidia-smi"]))
+ # driver 615 prints "CUDA UMD Version: 13.4" where older drivers print "CUDA Version: 13.0"
+ match = re.search(r"CUDA (?:UMD )?Version:\s*([\d.]+)", run_cmd(["nvidia-smi"]))
cuda = match.group(1) if match else None
return cards, (cards[0]["driver"] if cards else None), cuda
@@ -200,6 +201,24 @@ def other_gpus() -> tuple[list[dict], str | None]:
return cards, driver
+def lsblk_disk(data: dict) -> list[dict] | None:
+ """The first physical disk in `lsblk -J -s` output. `-s` lists the parents of a device as nested children: an LVM volume
+ is lvm -> partition -> disk, two levels down, so the walk goes all the way down."""
+ def disks(item):
+ if item.get("type") == "disk":
+ yield item
+ for child in item.get("children") or []:
+ yield from disks(child)
+
+ for device in data.get("blockdevices") or []:
+ for item in disks(device):
+ rotational = str(item.get("rota")).lower() in ("1", "true")
+ bus = str(item.get("tran") or "unknown").lower()
+ return [{"kind": "hdd" if rotational else ("nvme" if bus == "nvme" else "ssd"),
+ "model": str(item.get("model") or "unknown").strip(), "bus": bus}]
+ return None
+
+
def storage_of(path: Path | None) -> list[dict] | None:
"""The drive that holds `path`: [{kind, model, bus}], or None when it cannot be told."""
if path is None:
@@ -229,14 +248,7 @@ def storage_of(path: Path | None) -> list[dict] | None:
data = json.loads(run_cmd(["lsblk", "-J", "-s", "-o", "NAME,TYPE,ROTA,MODEL,TRAN", source]))
except ValueError:
return None
- for device in data.get("blockdevices") or []:
- chain = [device, *(device.get("children") or [])]
- for item in chain:
- if item.get("type") == "disk":
- rotational = str(item.get("rota")).lower() in ("1", "true")
- bus = str(item.get("tran") or "unknown").lower()
- return [{"kind": "hdd" if rotational else ("nvme" if bus == "nvme" else "ssd"),
- "model": str(item.get("model") or "unknown").strip(), "bus": bus}]
+ return lsblk_disk(data)
return None
@@ -328,10 +340,29 @@ def git_commit(folder: Path = ROOT) -> str | None:
return text if HEX40.match(text) else None
-def software(why: Why, machine: dict, *, engine_version: str | None, build_options: list[str] | None) -> dict:
+def engine_build_runtime(exe: str | None) -> dict | None:
+ """The GPU runtime a release engine was built with, from its engine-component.json (build_id "a7bdbfdabc9a+cuda12.9"):
+ {"name": "cuda", "version": "12.9"}. None for a source build or an engine without that file."""
+ if not exe:
+ return None
+ try:
+ record = json.loads((Path(exe).parent / "engine-component.json").read_text(encoding="utf-8"))
+ except (OSError, ValueError):
+ return None
+ match = re.search(r"\+(cuda|rocm|hip|sycl)([\d.]+)\Z", str(record.get("build_id") or "")) if isinstance(record, dict) else None
+ return {"name": "rocm" if match.group(1) == "hip" else match.group(1), "version": match.group(2)} if match else None
+
+
+def software(why: Why, machine: dict, *, engine_version: str | None, build_options: list[str] | None,
+ engine_build: dict | None = None) -> dict:
commit = git_commit()
runtime = []
- if machine.get("cuda"):
+ if engine_build:
+ # the library the engine runs on; nvidia-smi's "CUDA Version" is only the newest one the driver supports
+ runtime.append(engine_build)
+ if machine.get("cuda") and engine_build["name"] == "cuda":
+ runtime.append({"name": "cuda-driver-supports", "version": machine["cuda"]})
+ elif machine.get("cuda"):
runtime.append({"name": "cuda", "version": machine["cuda"]})
if sys.platform == "darwin":
runtime.append({"name": "metal", "version": platform.mac_ver()[0] or platform.release()})
diff --git a/strata_cli/bench_standard.py b/strata_cli/bench_standard.py
index c1552793f..7c6bbdc59 100644
--- a/strata_cli/bench_standard.py
+++ b/strata_cli/bench_standard.py
@@ -694,6 +694,24 @@ def start_engine(ref: str, target: Target, log_path: Path, log, timeout_s: float
return proc
+def engine_log_since(console_log: Path, run_log: Path | None, offset: int) -> Path:
+ """The log to keep with a result when the runner started the engine. `strata serve --quiet` prints nothing, so the
+ console capture is empty; the engine's own lines are in the run config's `log` file, from `offset` (its size before
+ the start). Falls back to the console capture when there is no such file or nothing new in it."""
+ try:
+ if run_log is not None and run_log.is_file() and run_log.stat().st_size > offset:
+ with run_log.open("rb") as stream:
+ stream.seek(offset)
+ data = stream.read()
+ if data.strip():
+ merged = console_log.with_name("engine-run.log")
+ merged.write_bytes(data)
+ return merged
+ except OSError:
+ pass
+ return console_log
+
+
def stop_engine(proc) -> None:
"""Stop only the process this run started."""
if proc is None:
@@ -732,7 +750,7 @@ def slug(text: str, limit: int = 40) -> str:
def run(target: Target, out: Path | None, *, out_root: Path | None = None, optional_concurrency: bool = False,
corpus: Corpus | None = None, declared: dict | None = None, plan: dict | None = None,
- engine_log: Path | None = None, command: str = "strata bench run --suite standard-v1",
+ engine_log: Path | None = None, engine_run_log: tuple[Path | None, int] | None = None, command: str = "strata bench run --suite standard-v1",
log=lambda message: None, timeout_s: float = REQUEST_TIMEOUT_S, started_engine: bool = False,
hash_model: bool = True) -> tuple[dict, Path]:
"""Run the protocol against `target` and write the report into the new directory `out` (or a new folder named
@@ -780,7 +798,8 @@ def run(target: Target, out: Path | None, *, out_root: Path | None = None, optio
hardware_block, machine_facts = facts.hardware(why, selected=selected, served_gpu=served_gpu, model_dir=model_dir,
declared_background=list(declared.get("background") or []))
software_block = facts.software(why, machine_facts, engine_version=engine_version,
- build_options=declared.get("build_options"))
+ build_options=declared.get("build_options"),
+ engine_build=facts.engine_build_runtime(plan.get("engine_exe")))
model_block = facts.model(why, props=props, status=status, files=files, declared=declared, log=log)
template_ref = {"path": "provenance/chat_template.jinja", "sha256": facts.sha256_text(template_text)} if template_text else None
configuration_block, other_options = facts.configuration(
@@ -878,6 +897,8 @@ def run(target: Target, out: Path | None, *, out_root: Path | None = None, optio
"source": "effective engine arguments of the profile" if options is not None
else "the server was already running; its engine options are not exposed"}, redact)
put("config", "config.json", (json.dumps(config_doc, ensure_ascii=False, indent=2, allow_nan=False) + "\n").encode("utf-8"))
+ if engine_log is not None and engine_run_log is not None:
+ engine_log = engine_log_since(engine_log, *engine_run_log)
if engine_log is not None and engine_log.is_file():
text = engine_log.read_text(encoding="utf-8", errors="replace")[-2_000_000:]
put("engine_log", "engine.log", redact(text).encode("utf-8"))
@@ -1177,7 +1198,9 @@ def report_markdown(doc: dict, redact=lambda text: text) -> str:
"the quantization repository's revision; see `missing` in results.json for every value it could not read.",
f"- The server had answered {doc['extensions']['server_requests_before_run']} request(s) before this run."
if doc["extensions"]["server_requests_before_run"] else
- "- The runner could not tell whether the server was freshly loaded.", ""]
+ ("- The server had answered no request before this run (a freshly loaded engine)."
+ if doc["extensions"]["server_requests_before_run"] == 0 else
+ "- The runner could not tell whether the server was freshly loaded."), ""]
return redact("\n".join(lines))
diff --git a/strata_cli/cli.py b/strata_cli/cli.py
index caf5c960b..04142e62f 100644
--- a/strata_cli/cli.py
+++ b/strata_cli/cli.py
@@ -1175,7 +1175,7 @@ def _bench_standard(a: argparse.Namespace, machine: bool) -> int:
import shutil
import tempfile
lease = benchmark_lease(saved["id"], out) if saved else contextlib.nullcontext()
- proc, scratch, engine_log = None, None, None
+ proc, scratch, engine_log, run_log, log_start = None, None, None, None, 0
try:
with lease, contextlib.redirect_stdout(sys.stderr):
def log(message: str) -> None:
@@ -1185,11 +1185,14 @@ def _bench_standard(a: argparse.Namespace, machine: bool) -> int:
raise ProfileError("no server is answering at the profile's address and --no-start was given", 5)
scratch = Path(tempfile.mkdtemp(prefix="strata-bench-"))
engine_log = scratch / "engine.log"
+ run_log = Path(str(plan["config"]["log"])).expanduser() if isinstance(plan["config"].get("log"), str) else None
+ log_start = run_log.stat().st_size if run_log is not None and run_log.is_file() else 0
proc = bench_standard.start_engine(a.profile, target, engine_log, log, a.load_timeout)
elif a.profile:
log("using the server that is already running at this profile's address; it will not be stopped")
doc, out_dir = bench_standard.run(target, out, out_root=out_root, optional_concurrency=a.optional_concurrency,
declared=declared, plan=plan, engine_log=engine_log, command=command, log=log,
+ engine_run_log=(run_log, log_start) if proc is not None else None,
started_engine=proc is not None, hash_model=not a.skip_model_hash)
finally:
bench_standard.stop_engine(proc)
diff --git a/tests/terminal/test_bench_standard.py b/tests/terminal/test_bench_standard.py
index b966f2cb6..3a5350c6f 100644
--- a/tests/terminal/test_bench_standard.py
+++ b/tests/terminal/test_bench_standard.py
@@ -968,5 +968,103 @@ class RealServerTests(unittest.TestCase):
self.assertEqual(doc["run"]["status"], "nonconforming") # no model file to hash, so the identity is incomplete
+class RealRunFindingsTests(StandardBase):
+ """What the first run on release engines (RTX 3060, 2x RTX A4000, 0.1.46) got wrong."""
+
+ LVM = {"blockdevices": [{"name": "ubuntu--vg-data", "type": "lvm", "rota": False, "model": None, "tran": None, "children": [
+ {"name": "nvme0n1p3", "type": "part", "rota": False, "model": None, "tran": "nvme", "children": [
+ {"name": "nvme0n1", "type": "disk", "rota": False, "model": "Lexar SSD NM620 512GB", "tran": "nvme"}]}]}]}
+
+ def test_the_drive_under_an_lvm_volume_is_found(self):
+ self.assertEqual(bench_facts.lsblk_disk(self.LVM), [{"kind": "nvme", "model": "Lexar SSD NM620 512GB", "bus": "nvme"}])
+
+ def test_a_partition_on_a_disk_and_a_missing_disk(self):
+ plain = {"blockdevices": [{"name": "sda1", "type": "part", "children": [
+ {"name": "sda", "type": "disk", "rota": True, "model": "Spinner", "tran": "sata"}]}]}
+ self.assertEqual(bench_facts.lsblk_disk(plain), [{"kind": "hdd", "model": "Spinner", "bus": "sata"}])
+ self.assertIsNone(bench_facts.lsblk_disk({"blockdevices": [{"name": "loop0", "type": "loop"}]}))
+
+ def test_the_runtime_is_the_one_the_engine_was_built_with(self):
+ engine = self.root / "engine"
+ engine.mkdir()
+ exe = engine / "strata"
+ exe.write_bytes(b"")
+ self.assertIsNone(bench_facts.engine_build_runtime(str(exe)))
+ (engine / "engine-component.json").write_text(json.dumps({"build_id": "a7bdbfdabc9a+cuda12.9"}), encoding="utf-8")
+ self.assertEqual(bench_facts.engine_build_runtime(str(exe)), {"name": "cuda", "version": "12.9"})
+ (engine / "engine-component.json").write_text(json.dumps({"build_id": "abc+hip6.4"}), encoding="utf-8")
+ self.assertEqual(bench_facts.engine_build_runtime(str(exe)), {"name": "rocm", "version": "6.4"})
+ why = bench_facts.Why()
+ block = bench_facts.software(why, {"driver": "580.1", "cuda": "13.0"}, engine_version="0.1.46", build_options=None,
+ engine_build={"name": "cuda", "version": "12.9"})
+ self.assertEqual(block["runtime"], [{"name": "cuda", "version": "12.9"}, {"name": "cuda-driver-supports", "version": "13.0"}])
+
+ def test_the_engine_log_comes_from_the_run_configs_log_file(self):
+ console = self.root / "engine.log"
+ console.write_bytes(b"")
+ run_log = self.root / "strata-model.log"
+ run_log.write_bytes(b"old line from an earlier start\n")
+ offset = run_log.stat().st_size
+ with run_log.open("ab") as stream:
+ stream.write(b"strata serve: expert cache 1500 slots\n")
+ kept = bs.engine_log_since(console, run_log, offset)
+ self.assertEqual(kept.read_bytes(), b"strata serve: expert cache 1500 slots\n")
+ self.assertEqual(bs.engine_log_since(console, None, 0), console)
+ self.assertEqual(bs.engine_log_since(console, self.root / "missing.log", 0), console)
+ self.assertEqual(bs.engine_log_since(console, run_log, run_log.stat().st_size), console)
+
+ def test_a_freshly_loaded_server_is_reported_as_fresh(self):
+ doc, out = self.run_standard()
+ doc["extensions"]["server_requests_before_run"] = 0
+ self.assertIn("freshly loaded engine", bs.report_markdown(doc))
+ self.assertNotIn("could not tell", bs.report_markdown(doc))
+ doc["extensions"]["server_requests_before_run"] = None
+ self.assertIn("could not tell", bs.report_markdown(doc))
+ doc["extensions"]["server_requests_before_run"] = 3
+ self.assertIn("answered 3 request(s)", bs.report_markdown(doc))
+
+
+class NvidiaBannerTests(unittest.TestCase):
+ def test_the_cuda_version_is_read_from_old_and_new_driver_banners(self):
+ query = "0, NVIDIA GeForce RTX 3060, 12288, 170.00, 615.71.09"
+ for banner, version in (("| NVIDIA-SMI 580.1 Driver Version: 580.1 CUDA Version: 13.0 |", "13.0"),
+ ("| NVIDIA-SMI 615.71 KMD Version: 615.71.09 CUDA UMD Version: 13.4 |", "13.4")):
+ def fake(cmd, banner=banner):
+ return query if any(a.startswith("--query-gpu") for a in cmd) else banner
+ with mock.patch.object(bench_facts, "run_cmd", side_effect=fake):
+ cards, driver, cuda = bench_facts.nvidia_gpus()
+ self.assertEqual((driver, cuda), ("615.71.09", version))
+
+
+class ArchiveCommitTests(unittest.TestCase):
+ """The release archive carried no COMMIT file, so every run on it was nonconforming."""
+
+ ROOT = Path(__file__).resolve().parents[2]
+
+ def setUp(self):
+ import sys
+ sys.path.insert(0, str(self.ROOT / "packaging"))
+ import package_terminal
+ self.pt = package_terminal
+
+ def test_an_explicit_commit_or_the_variable_wins_over_git(self):
+ sha = "ab" * 20
+ self.assertEqual(self.pt.build_commit(sha.upper()), sha)
+ with mock.patch.dict(os.environ, {"STRATA_COMMIT": "cd" * 20}):
+ self.assertEqual(self.pt.build_commit(None), "cd" * 20)
+
+ def test_a_tree_copy_without_git_has_no_commit(self):
+ with mock.patch.dict(os.environ, {}, clear=False), mock.patch.object(self.pt.subprocess, "run") as run:
+ os.environ.pop("STRATA_COMMIT", None)
+ run.return_value = mock.Mock(stdout="fatal: not a git repository\n")
+ self.assertIsNone(self.pt.build_commit("not a sha"))
+
+ def test_a_distributable_build_refuses_to_run_without_a_commit(self):
+ with tempfile.TemporaryDirectory() as out, mock.patch.object(self.pt, "build_commit", return_value=None), mock.patch("sys.argv", ["package_terminal.py", "--out", out]):
+ with self.assertRaises(SystemExit) as raised:
+ self.pt.main()
+ self.assertIn("COMMIT", str(raised.exception))
+
+
if __name__ == "__main__":
unittest.main()