Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

strata bench run: the standard benchmark protocol (standard-v1 is the default suite); fixtures and COMMIT file in the Terminal archive; tests with a mock server

Niko12216d41b37e769c
diff --git a/packaging/package_terminal.py b/packaging/package_terminal.py
index 9d0111c22..7d1221061 100644
--- a/packaging/package_terminal.py
+++ b/packaging/package_terminal.py
@@ -99,6 +99,7 @@ PAYLOAD = (
"docs/MANAGEMENT_PROTOCOL.md",
"docs/TERMINAL_LAUNCH.md", # the README below points at it (`strata launch`); it used to be missing from the archive
"data/gguf_fingerprints.json", # `strata models import --plan` names a known file set from this table
+ "bench/prompts/standard-v1", # `strata bench run` (the standard benchmark) sends these frozen requests; hashed byte for byte
)
#: `tools/` modules the terminal paths import. Adding one here is a deliberate act: this list is what keeps
@@ -343,6 +344,10 @@ def main() -> int:
shutil.copy2(src, tools_dir / item)
installer = None if args.no_installer else add_installer(root)
(root / "VERSION").write_text(version + "\n", encoding="utf-8")
+ # the commit the archive was built from: `strata bench run` records it (an archive has no .git to ask)
+ built = subprocess.run(["git", "-C", str(ROOT), "rev-parse", "HEAD"], capture_output=True, text=True).stdout.strip().lower()
+ if re.fullmatch(r"[0-9a-f]{40}", built):
+ (root / "COMMIT").write_text(built + "\n", encoding="utf-8")
bin_dir = root / "bin"
bin_dir.mkdir(exist_ok=True)
diff --git a/strata_cli/bench_facts.py b/strata_cli/bench_facts.py
new file mode 100644
index 000000000..7d3cc7d5d
--- /dev/null
+++ b/strata_cli/bench_facts.py
@@ -0,0 +1,488 @@
+"""What a standard benchmark report says about the machine, the build, the model and the settings (docs/BENCHMARK_PROTOCOL.md).
+
+Every probe is best effort and local: it reads the inference host's own tools (nvidia-smi, rocm-smi, PowerShell, /proc, sysctl) and the
+server's answers. A fact that cannot be read becomes `None` with a reason in `Why`, never a guess. Nothing here changes the machine.
+"""
+
+from __future__ import annotations
+
+import getpass
+import hashlib
+import json
+import os
+import platform
+import re
+import socket
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+LOOPBACK = frozenset({"localhost", "127.0.0.1", "::1"})
+HEX40 = re.compile(r"^[0-9a-f]{40}$")
+MODEL_SUFFIXES = (".gguf", ".safetensors", ".pack", ".bin")
+SECRET_KEY = re.compile(r"(?i)key|token|secret|password|auth")
+
+
+class Why(dict):
+ """JSON Pointer -> the reason a value is null. Built while the report is assembled and written as `missing`."""
+
+ def add(self, pointer: str, reason: str) -> None:
+ self[pointer] = reason
+
+
+def pointer(*parts) -> str:
+ return "".join("/" + str(p).replace("~", "~0").replace("/", "~1") for p in parts)
+
+
+# ------------------------------------------------------------------------------------------------------------ redaction
+def redactor(key: str = ""):
+ """text -> text with this machine's home folders, user and host names, the Strata folder, private IP addresses and the API key replaced."""
+ subs: list[tuple[re.Pattern, str]] = []
+
+ def add(value, label, word=False):
+ if value and len(str(value)) >= 4:
+ text = str(value)
+ for form in sorted({text, text.replace("\\", "/"), text.replace("/", "\\")}, key=len, reverse=True):
+ pattern = re.escape(form)
+ subs.append((re.compile(r"(?<![\w-])" + pattern + r"(?![\w-])" if word else pattern, re.I), label))
+
+ add(ROOT, "<strata>")
+ add(Path.home(), "<home>")
+ add(tempfile.gettempdir(), "<tmp>")
+ try:
+ add(getpass.getuser(), "<user>", True)
+ except Exception: # noqa: BLE001 - no user name is not an error
+ pass
+ add(socket.gethostname(), "<host>", True)
+ add(platform.node(), "<host>", True)
+ add(key, "<key>")
+ subs.sort(key=lambda item: -len(item[0].pattern))
+ path_user = re.compile(r"(?i)([A-Za-z]:)?([\\/])(Users|home)([\\/])(?!<)[^\\/\s\"':;,]+")
+ private_ip = re.compile(r"\b(?:192\.168|10\.\d{1,3}|172\.(?:1[6-9]|2\d|3[01])|100\.(?:6[4-9]|[7-9]\d|1[01]\d|12[0-7])"
+ r"|169\.254)\.\d{1,3}\.\d{1,3}\b")
+ bearer = re.compile(r"(?i)(bearer\s+)[A-Za-z0-9._~+/=-]{8,}")
+
+ def clean(text):
+ if not isinstance(text, str):
+ return text
+ for pattern, label in subs:
+ text = pattern.sub(label, text)
+ text = path_user.sub(lambda m: f"{m.group(1) or ''}{m.group(2)}{m.group(3)}{m.group(4)}<user>", text)
+ text = bearer.sub(r"\1<key>", text)
+ return private_ip.sub("<ip>", text)
+ return clean
+
+
+def redact_object(value, clean):
+ if isinstance(value, dict):
+ return {clean(k) if isinstance(k, str) else k: redact_object(v, clean) for k, v in value.items()}
+ if isinstance(value, list):
+ return [redact_object(v, clean) for v in value]
+ return clean(value) if isinstance(value, str) else value
+
+
+# ------------------------------------------------------------------------------------------------------------ commands
+def run_cmd(cmd: list[str], timeout: float = 15) -> str:
+ try:
+ done = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, encoding="utf-8", errors="replace")
+ return done.stdout if done.returncode == 0 else ""
+ except (OSError, subprocess.SubprocessError):
+ return ""
+
+
+def powershell(script: str, timeout: float = 20) -> str:
+ return run_cmd(["powershell", "-NoProfile", "-NonInteractive", "-Command", script], timeout)
+
+
+def sha256_file(path: Path, chunk: int = 8 << 20) -> str:
+ digest = hashlib.sha256()
+ with open(path, "rb") as stream:
+ while block := stream.read(chunk):
+ digest.update(block)
+ return digest.hexdigest()
+
+
+def sha256_text(text: str) -> str:
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+# ------------------------------------------------------------------------------------------------------------ hardware
+def ram_bytes() -> int | None:
+ try:
+ import psutil
+ return int(psutil.virtual_memory().total)
+ except Exception: # noqa: BLE001
+ pass
+ try:
+ for line in Path("/proc/meminfo").read_text().splitlines():
+ if line.startswith("MemTotal:"):
+ return int(line.split()[1]) * 1024
+ except OSError:
+ pass
+ if sys.platform == "darwin":
+ out = run_cmd(["sysctl", "-n", "hw.memsize"]).strip()
+ return int(out) if out.isdigit() else None
+ return None
+
+
+def cpu_name() -> str | None:
+ try:
+ for line in Path("/proc/cpuinfo").read_text().splitlines():
+ if line.lower().startswith("model name"):
+ return line.split(":", 1)[1].strip()
+ except OSError:
+ pass
+ if sys.platform == "darwin":
+ out = run_cmd(["sysctl", "-n", "machdep.cpu.brand_string"]).strip()
+ if out:
+ return out
+ if os.name == "nt":
+ out = powershell("(Get-CimInstance Win32_Processor | Select-Object -First 1).Name").strip()
+ if out:
+ return out
+ return platform.processor() or None
+
+
+def nvidia_gpus() -> tuple[list[dict], str | None, str | None]:
+ """-> (cards, driver version, CUDA version) from nvidia-smi."""
+ out = run_cmd(["nvidia-smi", "--query-gpu=index,name,memory.total,power.limit,driver_version",
+ "--format=csv,noheader,nounits"])
+ cards = []
+ for line in out.splitlines():
+ part = [x.strip() for x in line.split(",")]
+ if len(part) < 5 or not part[0].isdigit():
+ continue
+
+ def number(text):
+ try:
+ return float(text)
+ except ValueError:
+ return None
+ mib, watts = number(part[2]), number(part[3])
+ cards.append({"device_index": int(part[0]), "name": part[1],
+ "memory_bytes": int(mib * 1024 * 1024) if mib else None,
+ "memory_kind": "dedicated", "power_limit_w": watts, "driver": part[4]})
+ cuda = None
+ if cards:
+ match = re.search(r"CUDA Version:\s*([\d.]+)", run_cmd(["nvidia-smi"]))
+ cuda = match.group(1) if match else None
+ return cards, (cards[0]["driver"] if cards else None), cuda
+
+
+def other_gpus() -> tuple[list[dict], str | None]:
+ """AMD (rocm-smi, or the Windows registry), Apple silicon: -> (cards, driver version)."""
+ cards, driver = [], None
+ rocm = run_cmd(["rocm-smi", "--showproductname"])
+ names = re.findall(r"Card [Ss]eries:\s*(.+)", rocm) or re.findall(r"Card [Mm]odel:\s*(.+)", rocm)
+ if names:
+ match = re.search(r"Driver version:\s*(\S+)", run_cmd(["rocm-smi", "--showdriverversion"]))
+ driver = match.group(1) if match else None
+ cards = [{"device_index": i, "name": n.strip(), "memory_bytes": None, "memory_kind": "dedicated",
+ "power_limit_w": None} for i, n in enumerate(names)]
+ elif os.name == "nt":
+ try:
+ if str(ROOT) not in sys.path:
+ sys.path.insert(0, str(ROOT))
+ from serve.telemetry import windows_amd_cards
+ found = windows_amd_cards()
+ cards = [{"device_index": i, "name": c["name"], "memory_bytes": c.get("vram_bytes") or None,
+ "memory_kind": "dedicated", "power_limit_w": None} for i, c in enumerate(found)]
+ driver = next((c.get("driver") for c in found if c.get("driver")), None)
+ except Exception: # noqa: BLE001 - a report without the card's name is still a report
+ pass
+ elif sys.platform == "darwin":
+ chip = cpu_name()
+ if chip:
+ cards = [{"device_index": 0, "name": chip, "memory_bytes": ram_bytes(), "memory_kind": "shared",
+ "power_limit_w": None}]
+ return cards, driver
+
+
+def storage_of(path: Path | None) -> list[dict] | None:
+ """The drive that holds `path`: [{kind, model, bus}], or None when it cannot be told."""
+ if path is None:
+ return None
+ try:
+ path = path.resolve()
+ except OSError:
+ return None
+ if os.name == "nt":
+ letter = path.drive[:1]
+ if not letter.isalpha():
+ return None
+ out = powershell(f"Get-Partition -DriveLetter {letter} | Get-Disk | Get-PhysicalDisk | "
+ "Select-Object MediaType,FriendlyName,BusType | ConvertTo-Json -Compress")
+ try:
+ data = json.loads(out)
+ except ValueError:
+ return None
+ rows = data if isinstance(data, list) else [data]
+ return [{"kind": str(r.get("MediaType") or "unknown").lower(), "model": str(r.get("FriendlyName") or "unknown"),
+ "bus": str(r.get("BusType") or "unknown").lower()} for r in rows if isinstance(r, dict)] or None
+ if sys.platform.startswith("linux"):
+ source = run_cmd(["findmnt", "-n", "-o", "SOURCE", "--target", str(path)]).strip()
+ if not source.startswith("/dev/"):
+ return None
+ try:
+ data = json.loads(run_cmd(["lsblk", "-J", "-s", "-o", "NAME,TYPE,ROTA,MODEL,TRAN", source]))
+ except ValueError:
+ return None
+ for device in data.get("blockdevices") or []:
+ chain = [device, *(device.get("children") or [])]
+ for item in chain:
+ if item.get("type") == "disk":
+ rotational = str(item.get("rota")).lower() in ("1", "true")
+ bus = str(item.get("tran") or "unknown").lower()
+ return [{"kind": "hdd" if rotational else ("nvme" if bus == "nvme" else "ssd"),
+ "model": str(item.get("model") or "unknown").strip(), "bus": bus}]
+ return None
+
+
+def gpu_selection(config: dict | None, argv: list[str] | None) -> list[int]:
+ """The card numbers the engine is told to use, from the run config's "gpu" or a `--gpu` in its arguments."""
+ def parse(value):
+ if value is None or value == "":
+ return []
+ items = value if isinstance(value, (list, tuple)) else str(value).split(",")
+ return [int(x) for x in (str(i).strip() for i in items) if x.isdigit()]
+ chosen = parse((config or {}).get("gpu"))
+ if not chosen and argv and "--gpu" in argv and argv.index("--gpu") + 1 < len(argv):
+ chosen = parse(argv[argv.index("--gpu") + 1])
+ return chosen
+
+
+def hardware(why: Why, *, selected: list[int], served_gpu: str | None, model_dir: Path | None,
+ declared_background: list[str]) -> tuple[dict, dict]:
+ """-> (the report's `hardware`, the driver and runtime facts the `software` block uses)."""
+ cards, driver, cuda = nvidia_gpus()
+ if not cards:
+ cards, driver = other_gpus()
+ if selected:
+ cards = [c for c in cards if c["device_index"] in selected] or cards
+ elif len(cards) > 1 and served_gpu:
+ named = [c for c in cards if c["name"] == served_gpu]
+ cards = named[:1] or cards[:1]
+ served = [n for n in (served_gpu or "").split(" + ") if n and n != "?"]
+ if served and cards and not all(n in {c["name"] for c in cards} for n in served):
+ # the server reads cards the local tools do not list (another vendor's, or a different numbering)
+ cards = [{"device_index": i, "name": n, "memory_bytes": None, "memory_kind": "dedicated", "power_limit_w": None}
+ for i, n in enumerate(served)]
+ driver = cuda = None
+ if not cards and served:
+ cards = [{"device_index": i, "name": n, "memory_bytes": None, "memory_kind": "dedicated", "power_limit_w": None}
+ for i, n in enumerate(served)]
+ gpus = []
+ for i, card in enumerate(cards):
+ gpus.append({k: card.get(k) for k in ("device_index", "name", "memory_bytes", "memory_kind", "power_limit_w")})
+ if gpus[-1]["memory_bytes"] is None:
+ why.add(pointer("hardware", "gpus", i, "memory_bytes"), "the local tools did not report this card's memory")
+ if gpus[-1]["power_limit_w"] is None:
+ why.add(pointer("hardware", "gpus", i, "power_limit_w"), "no power limit available for this card from the local tools")
+ ram = ram_bytes()
+ cpu = cpu_name()
+ drive = storage_of(model_dir)
+ block = {"gpus": gpus, "cpu": cpu, "ram_bytes": ram, "storage": drive,
+ "background_workloads": declared_background or None}
+ if not gpus:
+ why.add(pointer("hardware", "gpus"), "no GPU was found by the local tools or reported by the server")
+ if cpu is None:
+ why.add(pointer("hardware", "cpu"), "the CPU model could not be read")
+ if ram is None:
+ why.add(pointer("hardware", "ram_bytes"), "installed memory could not be read")
+ if drive is None:
+ why.add(pointer("hardware", "storage"), "the drive holding the model files could not be identified")
+ if not declared_background:
+ why.add(pointer("hardware", "background_workloads"),
+ "the runner does not inspect other programs; none were declared with --background")
+ return block, {"driver": driver, "cuda": cuda}
+
+
+# ------------------------------------------------------------------------------------------------------------ software
+def os_info() -> dict:
+ if os.name == "nt":
+ return {"name": "Windows", "version": platform.version()}
+ if sys.platform == "darwin":
+ return {"name": "macOS", "version": platform.mac_ver()[0] or platform.release()}
+ name = None
+ try:
+ for line in Path("/etc/os-release").read_text().splitlines():
+ if line.startswith("PRETTY_NAME="):
+ name = line.split("=", 1)[1].strip().strip('"')
+ except OSError:
+ pass
+ return {"name": name or platform.system(), "version": platform.release()}
+
+
+def git_commit(folder: Path = ROOT) -> str | None:
+ """The full commit of this checkout, or the COMMIT file a release archive carries; None for neither."""
+ if (folder / ".git").exists():
+ out = run_cmd(["git", "-C", str(folder), "rev-parse", "HEAD"]).strip().lower()
+ if HEX40.match(out):
+ return out
+ try:
+ text = (folder / "COMMIT").read_text(encoding="utf-8").strip().lower()
+ except OSError:
+ return None
+ return text if HEX40.match(text) else None
+
+
+def software(why: Why, machine: dict, *, engine_version: str | None, build_options: list[str] | None) -> dict:
+ commit = git_commit()
+ runtime = []
+ if machine.get("cuda"):
+ runtime.append({"name": "cuda", "version": machine["cuda"]})
+ if sys.platform == "darwin":
+ runtime.append({"name": "metal", "version": platform.mac_ver()[0] or platform.release()})
+ drivers = [{"name": "gpu-driver", "version": machine["driver"]}] if machine.get("driver") else []
+ block = {"os": os_info(), "drivers": drivers or None, "runtime": runtime or None,
+ "strata_commit": commit, "engine_version": engine_version,
+ "build_kind": "source" if (ROOT / ".git").exists() else "release",
+ "runner_commit": commit, "build_options": build_options}
+ if not drivers:
+ why.add("/software/drivers", "no GPU driver version was reported by the local tools")
+ if not runtime:
+ why.add("/software/runtime", "no CUDA, ROCm or Metal runtime version was reported by the local tools")
+ if commit is None:
+ for field in ("strata_commit", "runner_commit"):
+ why.add(f"/software/{field}", "not a git checkout and no COMMIT file in this install; a release must carry its commit")
+ if engine_version is None:
+ why.add("/software/engine_version", "the server reported no engine version (/props build_info, /v1/status engine)")
+ if build_options is None:
+ why.add("/software/build_options", "the server does not report build options; pass --build-options for a source build")
+ return block
+
+
+# ------------------------------------------------------------------------------------------------------------ model
+QUANT = re.compile(r"(?i)(?:^|[-_.])((?:UD-)?(?:IQ\d_[A-Z]+|Q\d_[A-Z0-9_]+|Q\d_K(?:_[A-Z]+)?|IQ\d_?[A-Z]*|BF16|F16|NVFP4|Q\d))(?=[-_.]|$)")
+SHARD = re.compile(r"^(.*)-(\d{5})-of-(\d{5})(\.gguf)$")
+
+
+def model_files_from(paths: list[str]) -> list[Path]:
+ """The model files behind these paths: each existing file, and the other shards of a split GGUF."""
+ found: dict[str, Path] = {}
+ for raw in paths:
+ try:
+ path = Path(raw)
+ if not path.is_file() or path.suffix.lower() not in MODEL_SUFFIXES:
+ continue
+ except (OSError, ValueError):
+ continue
+ found[str(path.resolve())] = path
+ match = SHARD.match(path.name)
+ if match:
+ total = int(match.group(3))
+ for index in range(1, total + 1):
+ sibling = path.with_name(f"{match.group(1)}-{index:05d}-of-{match.group(3)}{match.group(4)}")
+ if sibling.is_file():
+ found[str(sibling.resolve())] = sibling
+ return sorted(found.values(), key=lambda p: p.name)
+
+
+def detect_quantization(names: list[str]) -> str | None:
+ for name in names:
+ match = QUANT.search(name)
+ if match:
+ return match.group(1).upper()
+ return None
+
+
+def model(why: Why, *, props: dict, status: dict, files: list[Path], declared: dict, log) -> dict:
+ served = props.get("model_alias") or status.get("model")
+ rows = []
+ for index, path in enumerate(files):
+ log(f"hashing model file {index + 1}/{len(files)}: {path.name}")
+ rows.append({"path": path.name, "sha256": sha256_file(path)})
+ quant = declared.get("quantization") or detect_quantization([p.name for p in files] + [str(served or "")])
+ block = {"id": str(served) if served else None,
+ "revision": declared.get("model_revision"), "source_revision": declared.get("source_revision"),
+ "quantization": quant, "files": rows or None, "tokenizer_revision": declared.get("tokenizer_revision"),
+ "custom_pack": declared.get("custom_pack")}
+ reasons = {"id": "the server reported no model name", "quantization": "no quantization name found in the model file names; pass --quantization",
+ "files": "no local model file could be found to hash (the server reports no model_path or it is not on this host)",
+ "revision": "not given: the quantization repository's immutable revision cannot be read from the engine; pass --model-revision",
+ "source_revision": "not given: pass --source-revision",
+ "tokenizer_revision": "not given: pass --tokenizer-revision",
+ "custom_pack": "no custom pack or expert profile was declared by the operator (--custom-pack)"}
+ for field, reason in reasons.items():
+ if block[field] is None:
+ why.add(f"/model/{field}", reason)
+ return block
+
+
+# ------------------------------------------------------------------------------------------------------------ configuration
+GROUPS = (
+ ("kv_cache", ("--kv",)),
+ ("expert_cache", ("--expert",)),
+ ("offloading", ("--gpu", "--layer-split", "--low-ram", "--offload", "--split", "--vram", "--pcie", "--host-core",
+ "--pool", "--numa", "--unmap", "--cpu-expert", "--tail-skip")),
+ ("prefill", ("--prefill",)),
+ ("mtp", ("--mtp", "--spec", "--draft", "--batch-mtp")),
+ ("calibration", ("--calib",)),
+ ("experimental_options", ("--experimental", "--exp-")),
+)
+
+
+def base_if_path(value):
+ if isinstance(value, str) and (("/" in value or "\\" in value) or re.search(r"\.(gguf|bin|json|pack|txt|jinja|so|dll|exe)$", value, re.I)):
+ return re.split(r"[\\/]", value.rstrip("\\/"))[-1]
+ return value
+
+
+def parse_options(argv: list[str]) -> dict:
+ """['--kv', 'int8', '--low-ram'] -> {'--kv': 'int8', '--low-ram': True}; a repeated option becomes a list."""
+ options: dict = {}
+ i = 0
+ while i < len(argv):
+ token = str(argv[i])
+ if token.startswith("--"):
+ name, value = (token.split("=", 1) + [None])[:2] if "=" in token else (token, None)
+ if value is None:
+ nxt = argv[i + 1] if i + 1 < len(argv) else None
+ if nxt is not None and not (str(nxt).startswith("--") and not re.fullmatch(r"-?\d+(\.\d+)?", str(nxt))):
+ value, i = str(nxt), i + 1
+ else:
+ value = True
+ value = base_if_path(value)
+ if name in options:
+ options[name] = [*(options[name] if isinstance(options[name], list) else [options[name]]), value]
+ else:
+ options[name] = value
+ i += 1
+ return options
+
+
+def configuration(why: Why, *, options: dict | None, env: dict | None, props: dict, declared: dict,
+ sampling: dict, template_ref: dict | None) -> tuple[dict, dict]:
+ """-> (the report's `configuration`, the engine options no group takes: they go to extensions)."""
+ grouped = {name: {} for name, _ in GROUPS}
+ other = {}
+ for name, value in (options or {}).items():
+ for group, prefixes in GROUPS:
+ if name.startswith(prefixes):
+ grouped[group][name] = value
+ break
+ else:
+ other[name] = value
+ for name, value in sorted((env or {}).items()):
+ if name.startswith("STRATA_") and not SECRET_KEY.search(name):
+ grouped["experimental_options"][name] = base_if_path(str(value))
+ n_ctx = ((props.get("default_generation_settings") or {}).get("n_ctx"))
+ slots = props.get("total_slots")
+ block = {"track": declared["track"], "track_notes": declared["track_notes"],
+ "context_tokens": n_ctx if isinstance(n_ctx, int) and not isinstance(n_ctx, bool) else None,
+ "server_capacity": slots if isinstance(slots, int) and not isinstance(slots, bool) else None,
+ **({name: grouped[name] for name, _ in GROUPS} if options is not None else {name: None for name, _ in GROUPS}),
+ "template": template_ref, "sampling": dict(sampling["sampling"]),
+ "chat_template_kwargs": dict(sampling["chat_template_kwargs"])}
+ if block["context_tokens"] is None:
+ why.add("/configuration/context_tokens", "the server's /props reported no context size")
+ if block["server_capacity"] is None:
+ why.add("/configuration/server_capacity", "the server's /props reported no slot count")
+ if options is None:
+ for name, _ in GROUPS:
+ why.add(f"/configuration/{name}", "engine options are not exposed by a running server; start it from a profile or pass --config")
+ if template_ref is None:
+ why.add("/configuration/template", "the server's /props returned no chat template")
+ return block, other
diff --git a/strata_cli/bench_standard.py b/strata_cli/bench_standard.py
new file mode 100644
index 000000000..c1552793f
--- /dev/null
+++ b/strata_cli/bench_standard.py
@@ -0,0 +1,1208 @@
+"""`strata bench run` for the standard benchmark protocol (docs/BENCHMARK_PROTOCOL.md).
+
+The runner sends the frozen requests of bench/prompts/standard-v1 to a Strata server through streaming chat completions, in the
+protocol's order, and writes a `strata-standard-benchmark` result that other people's results can be compared with:
+
+ warm-up (1, excluded) -> performance (short, prompt-4k, prompt-16k, three times) -> sanity (I1-I4, R1-R4, T1-T4)
+ -> optional concurrency (four short requests released together, three rounds)
+
+Speeds are the engine's own timings (the `timings` of the last stream chunk); first-token times are the client's monotonic clock. A value
+that cannot be read stays `None` with a reason in `missing`; nothing is estimated from chunks, characters or words. Prompts, answers and
+timings are saved without home folders, host names, private addresses or keys.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import http.client
+import json
+import math
+import os
+import re
+import socket
+import statistics
+import subprocess
+import sys
+import threading
+import time
+import urllib.parse
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime, timezone
+from pathlib import Path
+
+from strata_cli import bench_facts as facts
+from strata_cli.bench import _private_mkdir, _write_new
+from strata_control.profiles import ProfileError
+
+FORMAT = "strata-standard-benchmark"
+SCHEMA_VERSION = 1
+PROTOCOL_ID = "strata-qwen38-thinking-v1-draft.1"
+PROFILE = "qwen3.8-flash-next-thinking"
+SUITE = "standard-v1"
+#: The model-card revision the profile was defined from. The owner has not pinned one yet, so every result says so.
+MODEL_CARD_REVISION: str | None = None
+MODEL_CARD_URL = "https://huggingface.co/Qwen/Qwen3.8-Flash-Next"
+
+REQUEST_TIMEOUT_S = 1800.0
+#: Prompt tokens the cache may report beyond the longest prefix a request shares with the rest of the bundle before the
+#: user payload counts as reused. The slack covers token merges at the boundary.
+REUSE_SLACK_TOKENS = 8
+LENGTH_TOLERANCE = 0.02
+
+SAMPLING = {"temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0,
+ "repetition_penalty": 1.0, "reasoning_effort": "xhigh"}
+TEMPLATE_KWARGS = {"enable_thinking": True, "preserve_thinking": True}
+PHASES = ("warmup", "performance", "sanity", "concurrency")
+PLANNED = {"warmup": 1, "performance": 9, "sanity": 12, "concurrency": 12}
+PERFORMANCE_CASES = ("short", "prompt-4k", "prompt-16k")
+METRICS = (("prompt_tps", "tokens/s"), ("decode_tps", "tokens/s"), ("ttft_s", "s"),
+ ("first_answer_s", "s"), ("total_s", "s"))
+HEX64 = re.compile(r"^[0-9a-f]{64}$")
+
+
+class CorpusError(ProfileError):
+ pass
+
+
+def _now() -> str:
+ return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _number(value) -> float | None:
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
+ return None
+ return float(value) if math.isfinite(value) and value >= 0 else None
+
+
+def _count(value) -> int | None:
+ if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+ if isinstance(value, float) and value.is_integer() and value >= 0 and math.isfinite(value):
+ return int(value)
+ return None
+ return value
+
+
+# ------------------------------------------------------------------------------------------------------------ the fixtures
+class Corpus:
+ def __init__(self, sha256: str, specs: list[dict], build: dict, manifest: bytes, directory: Path):
+ self.sha256, self.specs, self.build, self.manifest, self.directory = sha256, specs, build, manifest, directory
+
+
+def corpus_dir() -> Path:
+ return Path(__file__).resolve().parents[1] / "bench" / "prompts" / SUITE
+
+
+def load_corpus(directory: Path | None = None) -> Corpus:
+ """Read the fixture bundle and check every file against its manifest. Any difference is an error: the run would not be the protocol's."""
+ root = Path(directory) if directory else corpus_dir()
+ try:
+ manifest = (root / "MANIFEST.sha256").read_bytes()
+ lines = manifest.decode("utf-8").splitlines()
+ except (OSError, UnicodeError):
+ raise CorpusError("the standard benchmark fixtures are missing (bench/prompts/standard-v1/MANIFEST.sha256)", 5) from None
+ entries: dict[str, str] = {}
+ for line in lines:
+ digest, _, name = line.partition(" ")
+ if not HEX64.match(digest) or not name or name.startswith(("/", "\\")) or ".." in Path(name).parts:
+ raise CorpusError("the fixture manifest is malformed", 5)
+ entries[name] = digest
+ for name, digest in entries.items():
+ try:
+ data = (root / name).read_bytes()
+ except OSError:
+ raise CorpusError(f"fixture file missing: {name}", 5) from None
+ if hashlib.sha256(data).hexdigest() != digest:
+ raise CorpusError(f"fixture file changed: {name} (line endings converted? bench/prompts is hashed byte for byte)", 5)
+ extra = {p.relative_to(root).as_posix() for p in (root / "requests").glob("*.json")} - set(entries)
+ if extra:
+ raise CorpusError(f"fixture file not in the manifest: {sorted(extra)[0]}", 5)
+ try:
+ build = json.loads((root / "build.json").read_text(encoding="utf-8"))
+ specs = [json.loads((root / "requests" / f"{row['id']}.json").read_text(encoding="utf-8")) for row in build["requests"]]
+ except (OSError, ValueError, KeyError, TypeError):
+ raise CorpusError("the fixture bundle is unreadable", 5) from None
+ counts = {p: sum(1 for s in specs if s.get("phase") == p) for p in PHASES}
+ if counts != PLANNED or len({s["id"] for s in specs}) != len(specs) or build.get("protocol_id") != PROTOCOL_ID:
+ raise CorpusError("the fixture bundle does not match this protocol", 5)
+ return Corpus(hashlib.sha256(manifest).hexdigest(), specs, build, manifest, root)
+
+
+# ------------------------------------------------------------------------------------------------------------ HTTP
+class Target:
+ def __init__(self, root: str, key: str | None):
+ parts = urllib.parse.urlsplit(root)
+ self.scheme, self.host = parts.scheme, parts.hostname or ""
+ self.port = parts.port or (443 if parts.scheme == "https" else 80)
+ self.key = key
+ self.root = root.rstrip("/")
+
+ @property
+ def loopback(self) -> bool:
+ return self.host in facts.LOOPBACK
+
+ def connection(self, timeout: float) -> http.client.HTTPConnection:
+ cls = http.client.HTTPSConnection if self.scheme == "https" else http.client.HTTPConnection
+ return cls(self.host, self.port, timeout=timeout)
+
+ def headers(self, extra: dict | None = None) -> dict:
+ out = {"User-Agent": "Strata-Bench/1", "Accept": "application/json", "Connection": "close", **(extra or {})}
+ if self.key:
+ out["Authorization"] = "Bearer " + self.key
+ return out
+
+ def get(self, path: str, timeout: float = 10) -> dict | None:
+ conn = self.connection(timeout)
+ try:
+ conn.request("GET", path, headers=self.headers())
+ response = conn.getresponse()
+ body = response.read(8 << 20)
+ if response.status != 200:
+ return None
+ value = json.loads(body)
+ return value if isinstance(value, dict) else None
+ except (OSError, http.client.HTTPException, ValueError):
+ return None
+ finally:
+ conn.close()
+
+
+def payload_of(spec: dict, model: str | None) -> dict:
+ """The request body: the protocol's sampling and thinking settings, every one explicit, plus the fixture's messages and tools."""
+ body = {"model": model or "strata", "messages": spec["messages"], "stream": True,
+ "stream_options": {"include_usage": True}, "max_tokens": spec["max_output_tokens"], "seed": spec["seed"],
+ **SAMPLING, "reasoning_budget_tokens": 0, "chat_template_kwargs": dict(TEMPLATE_KWARGS)}
+ if spec.get("tools"):
+ body["tools"] = spec["tools"]
+ body["tool_choice"] = spec.get("tool_choice") or "auto"
+ return body
+
+
+def stream_chat(target: Target, spec: dict, model: str | None, *, timeout_s: float,
+ gate: threading.Event | None = None) -> dict:
+ """Send one streamed request and keep what the client saw, with monotonic offsets from the moment of dispatch."""
+ body = json.dumps(payload_of(spec, model)).encode("utf-8")
+ raw: dict = {"http_status": None, "error": None, "finish_reason": None, "usage": None, "timings": None,
+ "events": [], "reasoning": [], "content": [], "tool_calls": {}, "done": False,
+ "timed_out": False, "interrupted": False}
+ if gate is not None:
+ gate.wait()
+ raw["t0"] = start = time.monotonic_ns()
+ conn = target.connection(timeout_s)
+
+ def expire():
+ raw["timed_out"] = True
+ sock = getattr(conn, "sock", None)
+ if sock is not None:
+ try:
+ sock.shutdown(socket.SHUT_RDWR)
+ except OSError:
+ pass
+ try:
+ sock.close()
+ except OSError:
+ pass
+ timer = threading.Timer(timeout_s, expire)
+ timer.daemon = True
+ timer.start()
+
+ def offset() -> float:
+ return round((time.monotonic_ns() - start) / 1e9, 6)
+
+ def mark(kind: str) -> None:
+ raw["events"].append([offset(), kind])
+
+ try:
+ conn.request("POST", "/v1/chat/completions", body,
+ target.headers({"Content-Type": "application/json", "Accept": "text/event-stream"}))
+ response = conn.getresponse()
+ raw["http_status"] = response.status
+ if response.status != 200:
+ text = response.read(4096).decode("utf-8", "replace")
+ raw["error"] = {"kind": f"http_{response.status}", "message": text.strip()[:300] or f"HTTP {response.status}"}
+ else:
+ while True:
+ line = response.readline(1 << 20)
+ if not line:
+ break
+ if not line.startswith(b"data:"):
+ continue
+ data = line[5:].strip()
+ if data == b"[DONE]":
+ raw["done"] = True
+ break
+ try:
+ event = json.loads(data)
+ except ValueError:
+ raw["error"] = {"kind": "malformed_stream", "message": "a stream event was not JSON"}
+ break
+ if not isinstance(event, dict):
+ raw["error"] = {"kind": "malformed_stream", "message": "a stream event was not an object"}
+ break
+ if "error" in event:
+ err = event["error"]
+ kind = (err.get("code") or err.get("type")) if isinstance(err, dict) else None
+ raw["error"] = {"kind": str(kind or "stream_error")[:60],
+ "message": str(err.get("message") if isinstance(err, dict) else err)[:300]}
+ break
+ if isinstance(event.get("usage"), dict):
+ raw["usage"] = event["usage"]
+ if isinstance(event.get("timings"), dict):
+ raw["timings"] = event["timings"]
+ for choice in event.get("choices") or []:
+ if not isinstance(choice, dict):
+ continue
+ delta = choice.get("delta") if isinstance(choice.get("delta"), dict) else {}
+ reasoning, text = delta.get("reasoning_content"), delta.get("content")
+ if isinstance(reasoning, str) and reasoning:
+ raw["reasoning"].append(reasoning)
+ mark("reasoning")
+ if isinstance(text, str) and text:
+ raw["content"].append(text)
+ mark("answer")
+ calls = delta.get("tool_calls")
+ if isinstance(calls, list):
+ qualifies = False
+ for call in calls:
+ if not isinstance(call, dict):
+ continue
+ slot = raw["tool_calls"].setdefault(call.get("index", len(raw["tool_calls"])),
+ {"id": "", "name": "", "arguments": ""})
+ function = call.get("function") if isinstance(call.get("function"), dict) else {}
+ for field, value in (("id", call.get("id")), ("name", function.get("name")),
+ ("arguments", function.get("arguments"))):
+ if isinstance(value, str) and value:
+ slot[field] += value
+ qualifies = True
+ if qualifies:
+ mark("tool_call")
+ if choice.get("finish_reason"):
+ raw["finish_reason"] = str(choice["finish_reason"])
+ if raw["error"] is None:
+ if not raw["done"]:
+ raw["error"] = {"kind": "malformed_stream", "message": "the stream ended without [DONE]"}
+ elif raw["finish_reason"] is None:
+ raw["error"] = {"kind": "malformed_stream", "message": "the stream carried no finish_reason"}
+ except KeyboardInterrupt:
+ raw["interrupted"] = True
+ raw["error"] = {"kind": "cancelled", "message": "cancelled by the operator"}
+ except (OSError, http.client.HTTPException, ValueError) as exc:
+ if raw["timed_out"]:
+ raw["error"] = {"kind": "timeout", "message": f"no complete answer within {timeout_s:g} seconds"}
+ else:
+ raw["error"] = {"kind": "connection_" + type(exc).__name__, "message": str(exc)[:200] or type(exc).__name__}
+ finally:
+ timer.cancel()
+ try:
+ conn.close()
+ except OSError:
+ pass
+ if raw["timed_out"] and raw["error"] is None:
+ raw["error"] = {"kind": "timeout", "message": f"no complete answer within {timeout_s:g} seconds"}
+ raw["t_end"] = time.monotonic_ns()
+ return raw
+
+
+# ------------------------------------------------------------------------------------------------------------ scoring
+def strict_json(text: str):
+ """One complete JSON value: no duplicate keys, no NaN or Infinity, no prose or fence around it."""
+ def pairs(items):
+ out = {}
+ for key, value in items:
+ if key in out:
+ raise ValueError(f"duplicate key {key!r}")
+ out[key] = value
+ return out
+
+ def constant(name):
+ raise ValueError(f"{name} is not JSON")
+ return json.loads(text.strip(" \t\r\n\x0b\x0c"), object_pairs_hook=pairs, parse_constant=constant)
+
+
+def same_json(a, b) -> bool:
+ """Equal values of equal JSON types (true is not 1, 3 is not 3.0); object key order does not matter."""
+ if isinstance(a, bool) or isinstance(b, bool):
+ return isinstance(a, bool) and isinstance(b, bool) and a == b
+ if isinstance(a, dict) and isinstance(b, dict):
+ return a.keys() == b.keys() and all(same_json(a[k], b[k]) for k in a)
+ if isinstance(a, list) and isinstance(b, list):
+ return len(a) == len(b) and all(same_json(x, y) for x, y in zip(a, b))
+ return type(a) is type(b) and a == b
+
+
+def evaluate(spec: dict, status: str, answer: str, calls: list[dict]) -> dict:
+ expected = spec["expected"]
+ category = spec["category"]
+
+ def verdict(passed: bool, reason: str) -> dict:
+ return {"category": category, "passed": passed, "reason": reason}
+ if status != "completed":
+ return verdict(False, {"truncated": "the output cap was reached before the answer was complete",
+ "failed": "the request failed", "skipped": "the request was skipped"}[status])
+ kind = expected["kind"]
+ if kind == "tool_call":
+ if len(calls) != 1:
+ return verdict(False, f"expected one structured tool call, got {len(calls)}")
+ call = calls[0]
+ if call["name"] != expected["name"]:
+ return verdict(False, f"wrong function {call['name']!r}")
+ try:
+ arguments = strict_json(call["arguments"] or "{}")
+ except ValueError as exc:
+ return verdict(False, f"the tool arguments are not strict JSON ({exc})")
+ if not isinstance(arguments, dict) or not same_json(arguments, expected["arguments"]):
+ return verdict(False, "the tool arguments differ from the expected values")
+ return verdict(True, "the expected tool call")
+ if kind == "text":
+ text = answer.strip(" \t\r\n\x0b\x0c")
+ return verdict(text == expected["value"],
+ "the exact expected text" if text == expected["value"] else "the answer text differs from the expected text")
+ try:
+ value = strict_json(answer)
+ except ValueError as exc:
+ return verdict(False, f"the answer is not one strict JSON value ({exc})")
+ return verdict(same_json(value, expected["value"]),
+ "the expected JSON value" if same_json(value, expected["value"]) else "the JSON value differs from the expected value")
+
+
+# ------------------------------------------------------------------------------------------------------------ one request
+def _round(value: float | None) -> float | None:
+ return None if value is None else round(value, 6)
+
+
+def record_of(spec: dict, raw: dict, meta: dict, why: facts.Why | None = None) -> tuple[dict, dict, dict]:
+ """-> (the report's request record, evidence for outputs.jsonl, extras for the report's `extensions`)."""
+ usage = raw["usage"] if isinstance(raw.get("usage"), dict) else {}
+ timings = raw["timings"] if isinstance(raw.get("timings"), dict) else {}
+ details = usage.get("prompt_tokens_details") if isinstance(usage.get("prompt_tokens_details"), dict) else {}
+ error = raw["error"]
+ finish = raw["finish_reason"]
+ if error is not None:
+ status = "failed"
+ elif finish == "length":
+ status = "truncated"
+ elif finish in ("stop", "tool_calls"):
+ status = "completed"
+ else:
+ status = "failed"
+ error = {"kind": "unexpected_finish_reason", "message": f"finish_reason {finish!r}"}
+ t0 = raw["t0"]
+ events = raw["events"]
+ first = events[0] if events else None
+ answer = next((e for e in events if e[1] in ("answer", "tool_call")), None)
+
+ input_tokens, output_tokens = _count(usage.get("prompt_tokens")), _count(usage.get("completion_tokens"))
+ cached, cached_source = _count(details.get("cached_tokens")), "usage.prompt_tokens_details.cached_tokens"
+ if cached is None:
+ cached, cached_source = _count(timings.get("cache_n")), "timings.cache_n"
+ read, read_source = _count(timings.get("prompt_n")), "timings.prompt_n"
+ if read is None and input_tokens is not None and cached is not None and cached <= input_tokens:
+ read, read_source = input_tokens - cached, "derived: usage.prompt_tokens - cached tokens"
+ prompt_s = _number(timings.get("prompt_ms"))
+ prompt_s = prompt_s / 1000 if prompt_s is not None else None
+ decode_s = _number(timings.get("predicted_ms"))
+ decode_s = decode_s / 1000 if decode_s is not None else None
+ rate, predicted_n = _number(timings.get("predicted_per_second")), _count(timings.get("predicted_n"))
+ decode_tokens, decode_source, note = None, None, None
+ if rate is not None and decode_s:
+ implied = round(rate * decode_s)
+ # the engine rounds its rate to 0.1 tokens/s, so the implied count is good to about 0.05 * decode_s tokens
+ if predicted_n is not None and abs(predicted_n - implied) <= 0.05 * decode_s + 1:
+ decode_tokens, decode_source = predicted_n, "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)"
+ else:
+ decode_tokens = implied
+ decode_source = "derived: timings.predicted_per_second x timings.predicted_ms / 1000 (the count the engine's rate used)"
+ if predicted_n is not None:
+ note = (f"timings.predicted_n is {predicted_n} but the engine's decode rate and time imply {implied} tokens; "
+ "the implied count is used because it is the one the rate was computed from")
+ elif predicted_n is not None:
+ decode_tokens, decode_source = predicted_n, "timings.predicted_n (no decode rate to check it against)"
+ if note is None and output_tokens is not None and decode_tokens is not None and output_tokens != decode_tokens:
+ note = (f"API usage.completion_tokens is {output_tokens}; the engine's decode count is {decode_tokens}. The engine count "
+ "feeds decode_tps; the API count feeds the concurrent aggregate")
+ prompt_tps = read / prompt_s if read and prompt_s else None
+ decode_tps = decode_tokens / decode_s if decode_tokens and decode_s else None
+
+ text = "".join(raw["content"])
+ calls = [raw["tool_calls"][k] for k in sorted(raw["tool_calls"], key=lambda k: (not isinstance(k, int), k))]
+ ttft, first_answer = (first[0] if first else None), (answer[0] if answer else None)
+ total = round((raw["t_end"] - t0) / 1e9, 6)
+ kind_names = {"reasoning": "reasoning", "answer": "answer", "tool_call": "tool_call"}
+ record = {
+ "id": spec["id"], "phase": spec["phase"], "case_id": spec["case_id"], "repetition": spec["repetition"],
+ "seed": spec["seed"], "max_output_tokens": spec["max_output_tokens"], "concurrency": spec["concurrency"],
+ "round": spec["round"], "status": status, "finish_reason": finish,
+ "error": None if status in ("completed", "truncated") else error,
+ "tokens": {"input_tokens": input_tokens, "cached_tokens": cached, "read_tokens": read,
+ "output_tokens": output_tokens, "decode_tokens": decode_tokens},
+ "timings": {"prompt_s": _round(prompt_s), "decode_s": _round(decode_s), "ttft_s": _round(ttft),
+ "first_answer_s": _round(first_answer), "total_s": total},
+ "metrics": {"prompt_tps": _round(prompt_tps), "decode_tps": _round(decode_tps)},
+ "sources": {"input_tokens": "usage.prompt_tokens of the final stream chunk" if input_tokens is not None else None,
+ "cached_tokens": cached_source if cached is not None else None,
+ "read_tokens": read_source if read is not None else None,
+ "output_tokens": "usage.completion_tokens of the final stream chunk" if output_tokens is not None else None,
+ "decode_tokens": decode_source,
+ "engine_timings": ("timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream "
+ "chunk (raw values in outputs.jsonl)") if timings else None,
+ "client_timings": ("client monotonic clock from dispatch; event offsets in outputs.jsonl")},
+ "first_delta_kind": kind_names.get(first[1]) if first else None,
+ "token_accounting_note": note, "evaluation": None}
+ if spec["phase"] == "sanity":
+ record["evaluation"] = evaluate(spec, status, text, [c for c in calls if c["name"] or c["arguments"]])
+ reuse_limit = meta["max_shared_prefix_tokens"] + REUSE_SLACK_TOKENS
+ extras = {"max_shared_prefix_tokens": meta["max_shared_prefix_tokens"],
+ "frozen_input_tokens": meta["input_tokens"],
+ "unexpected_payload_reuse": bool(cached is not None and cached > reuse_limit),
+ "length_outside_tolerance": bool(spec.get("target_input_tokens") and input_tokens is not None and
+ abs(input_tokens - spec["target_input_tokens"]) >
+ LENGTH_TOLERANCE * spec["target_input_tokens"])}
+ evidence = {"request_id": spec["id"], "phase": spec["phase"], "case_id": spec["case_id"],
+ "request": {k: v for k, v in payload_of(spec, None).items() if k not in ("messages", "tools", "model")},
+ "messages": spec["messages"], "tools": spec.get("tools"),
+ "final_answer": text, "reasoning": "".join(raw["reasoning"]),
+ "tool_calls": [{"id": c["id"], "name": c["name"], "arguments": c["arguments"]} for c in calls],
+ "finish_reason": finish, "usage": raw.get("usage"), "timings": raw.get("timings"), "error": error,
+ "http_status": raw["http_status"], "dispatch_offset_s": 0.0, "end_offset_s": total,
+ "first_delta": {"offset_s": first[0], "kind": first[1]} if first else None,
+ "first_answer_offset_s": first_answer, "client_events": events}
+ return record, evidence, extras
+
+
+def skipped_record(spec: dict, reason: str) -> tuple[dict, dict, dict]:
+ record = {"id": spec["id"], "phase": spec["phase"], "case_id": spec["case_id"], "repetition": spec["repetition"],
+ "seed": spec["seed"], "max_output_tokens": spec["max_output_tokens"], "concurrency": spec["concurrency"],
+ "round": spec["round"], "status": "skipped", "finish_reason": None,
+ "error": {"kind": "skipped", "message": reason},
+ "tokens": {k: None for k in ("input_tokens", "cached_tokens", "read_tokens", "output_tokens", "decode_tokens")},
+ "timings": {k: None for k in ("prompt_s", "decode_s", "ttft_s", "first_answer_s", "total_s")},
+ "metrics": {"prompt_tps": None, "decode_tps": None},
+ "sources": {k: None for k in ("input_tokens", "cached_tokens", "read_tokens", "output_tokens", "decode_tokens",
+ "engine_timings", "client_timings")},
+ "first_delta_kind": None, "token_accounting_note": None, "evaluation": None}
+ if spec["phase"] == "sanity":
+ record["evaluation"] = {"category": spec["category"], "passed": False, "reason": "the request was skipped: " + reason}
+ return record, {"request_id": spec["id"], "skipped": reason}, {"unexpected_payload_reuse": False,
+ "length_outside_tolerance": False}
+
+
+# ------------------------------------------------------------------------------------------------------------ the run
+class Collector:
+ """Runs the scheduled requests in order and keeps their records."""
+
+ def __init__(self, target: Target, corpus: Corpus, model: str | None, log, timeout_s: float):
+ self.target, self.corpus, self.model, self.log, self.timeout_s = target, corpus, model, log, timeout_s
+ self.meta = {row["id"]: row for row in corpus.build["requests"]}
+ self.results: dict[str, tuple[dict, dict, dict]] = {}
+ self.abort: str | None = None
+ self.number = 0
+ self.total = sum(PLANNED[p] for p in ("warmup", "performance", "sanity"))
+
+ def finish(self, spec: dict, raw: dict) -> dict:
+ result = record_of(spec, raw, self.meta[spec["id"]])
+ self.results[spec["id"]] = result
+ return result[0]
+
+ def alive(self) -> bool:
+ health = self.target.get("/health", timeout=5)
+ return bool(health and health.get("service") == "strata")
+
+ def one(self, spec: dict) -> None:
+ if self.abort:
+ self.results[spec["id"]] = skipped_record(spec, self.abort)
+ return
+ self.number += 1
+ self.log(f"[{self.number}/{self.total}] {spec['id']} (up to {spec['max_output_tokens']} tokens)")
+ raw = stream_chat(self.target, spec, self.model, timeout_s=self.timeout_s)
+ record = self.finish(spec, raw)
+ timing = record["timings"]
+ speed = record["metrics"]
+ self.log(f" {record['status']}: prompt {speed['prompt_tps'] or 'n/a'} tok/s, decode {speed['decode_tps'] or 'n/a'} tok/s, "
+ f"first token {timing['ttft_s'] if timing['ttft_s'] is not None else 'n/a'} s"
+ + (f"; {record['error']['kind']}" if record["error"] else ""))
+ if raw["interrupted"]:
+ self.abort = "the operator cancelled the run"
+ elif record["status"] == "failed":
+ if spec["phase"] == "warmup":
+ self.abort = f"the warm-up request failed ({record['error']['kind']}); the measured stages were not run"
+ elif not self.alive():
+ self.abort = f"the server stopped answering after {spec['id']} ({record['error']['kind']})"
+
+ def concurrent_round(self, specs: list[dict]) -> dict:
+ gate = threading.Event()
+ raws: dict[str, dict] = {}
+
+ def work(spec):
+ raws[spec["id"]] = stream_chat(self.target, spec, self.model, timeout_s=self.timeout_s, gate=gate)
+ with ThreadPoolExecutor(max_workers=len(specs)) as pool:
+ futures = [pool.submit(work, spec) for spec in specs]
+ time.sleep(0.05)
+ gate.set()
+ for future in futures:
+ future.result()
+ records = [self.finish(spec, raws[spec["id"]]) for spec in specs]
+ start = min(raws[s["id"]]["t0"] for s in specs)
+ end = max(raws[s["id"]]["t_end"] for s in specs)
+ wall = round((end - start) / 1e9, 6)
+ counted = [r for r in records if r["status"] in ("completed", "truncated")]
+ outputs = [r["tokens"]["output_tokens"] for r in counted]
+ total = sum(outputs) if outputs and all(o is not None for o in outputs) else None
+ failed = sum(r["status"] == "failed" for r in records)
+ self.log(f" round {specs[0]['round']}: {len(counted)} completed or capped, {failed} failed, {wall} s")
+ if any(raws[s["id"]]["interrupted"] for s in specs):
+ self.abort = "the operator cancelled the run"
+ return {"round": specs[0]["round"], "request_ids": [s["id"] for s in specs], "wall_s": wall,
+ "output_tokens": total, "aggregate_tps": _round(total / wall) if total is not None and wall > 0 else None,
+ "failed": failed}
+
+
+def summaries_of(records: list[dict], extras: dict, phase: str, cases: tuple[str, ...], concurrency: int) -> list[dict]:
+ rows = []
+ for case in cases:
+ group = [r for r in records if r["phase"] == phase and r["case_id"] == case]
+ for metric, unit in METRICS:
+ included, excluded, values = [], [], []
+ for r in group:
+ reason = None
+ if r["status"] == "skipped":
+ reason = "skipped: " + r["error"]["message"]
+ elif r["status"] == "failed":
+ reason = "failed: " + r["error"]["kind"]
+ elif extras[r["id"]]["unexpected_payload_reuse"]:
+ reason = (f"unexpected cache reuse: {r['tokens']['cached_tokens']} cached tokens exceed the "
+ f"{extras[r['id']]['max_shared_prefix_tokens']}-token shared template prefix")
+ else:
+ value = r["metrics"].get(metric) if metric in ("prompt_tps", "decode_tps") else r["timings"].get(metric)
+ if value is None:
+ reason = f"{metric} is not available for this request"
+ else:
+ values.append(value)
+ included.append(r["id"])
+ if reason:
+ excluded.append({"request_id": r["id"], "reason": reason})
+ rows.append({"phase": phase, "case_id": case, "concurrency": concurrency, "metric": metric, "unit": unit,
+ "median": _round(statistics.median(values)) if values else None,
+ "min": _round(min(values)) if values else None, "max": _round(max(values)) if values else None,
+ "n": len(values), "included_request_ids": included, "excluded": excluded})
+ return rows
+
+
+def counts_of(records: list[dict], phase: str, planned: int) -> dict:
+ rows = [r for r in records if r["phase"] == phase]
+ tally = {s: sum(r["status"] == s for r in rows) for s in ("completed", "truncated", "failed", "skipped")}
+ attempted = tally["completed"] + tally["truncated"] + tally["failed"]
+ return {"planned": planned, "attempted": attempted, **tally}
+
+
+def quality_of(records: list[dict]) -> dict:
+ sanity = [r for r in records if r["phase"] == "sanity"]
+ cats = {name: {"passed": sum(r["evaluation"]["passed"] for r in sanity if r["evaluation"]["category"] == name),
+ "total": 4} for name in ("instruction", "retrieval", "tool_call")}
+ return {"passed": sum(c["passed"] for c in cats.values()), "total": 12, "categories": cats}
+
+
+def scan_nulls(value, path: str = ""):
+ if value is None:
+ yield path
+ elif isinstance(value, dict):
+ for key, item in value.items():
+ if path == "" and key in ("missing", "extensions"):
+ continue
+ yield from scan_nulls(item, path + facts.pointer(key))
+ elif isinstance(value, list):
+ for index, item in enumerate(value):
+ yield from scan_nulls(item, path + facts.pointer(index))
+
+
+def default_reason(doc: dict, pointer: str) -> str:
+ parts = [p.replace("~1", "/").replace("~0", "~") for p in pointer.split("/")[1:]]
+ if parts[0] == "requests" and len(parts) >= 3:
+ record = doc["requests"][int(parts[1])]
+ field = ".".join(parts[2:])
+ if record["status"] == "skipped":
+ return "not measured: the request was skipped (" + record["error"]["message"] + ")"
+ if field == "error":
+ return "none: the request ended normally"
+ if field == "round":
+ return "not applicable outside the concurrency stage"
+ if field == "evaluation":
+ return "not applicable outside the sanity stage"
+ if field == "token_accounting_note":
+ return "none: the API and engine counts agree or only one exists"
+ if field == "finish_reason":
+ return "the server sent no finish_reason"
+ if field == "first_delta_kind":
+ return "no nonempty reasoning, answer or tool-call delta arrived"
+ if field in ("timings.ttft_s", "timings.first_answer_s"):
+ return ("no nonempty delta arrived" if field == "timings.ttft_s"
+ else "no answer text or tool call arrived (reasoning alone does not count)")
+ if field.startswith("metrics."):
+ return ("the rate needs a nonzero freshly read count (or generated count) and an engine duration, and one is missing or zero")
+ if field.startswith("sources."):
+ return "no source: the value was not reported"
+ if record["status"] == "failed":
+ return f"the request failed ({record['error']['kind']}) before this value was reported"
+ return "the server's response did not include this value (usage or timings)"
+ if parts[0] == "summaries" and len(parts) >= 3:
+ row = doc["summaries"][int(parts[1])]
+ return f"no valid samples of {row['metric']} for {row['case_id']} ({len(row['excluded'])} excluded)"
+ if parts[0] == "concurrency_rounds":
+ return "an included request has no output token count, or the round wall time is zero"
+ if parts[0] == "run" and parts[1] == "optional_duration_s":
+ return "the optional concurrency stage was not selected or did not run"
+ if parts[0] == "protocol" and parts[1] == "model_card_revision":
+ return "the profile's model-card revision is not pinned yet (protocol draft)"
+ return "not available"
+
+
+def _start_wait(target: Target, proc, timeout_s: float, log, log_path: Path | None) -> bool:
+ began, announced = time.monotonic(), 0.0
+ while time.monotonic() - began < timeout_s:
+ if proc is not None and proc.poll() is not None:
+ return False
+ health = target.get("/health", timeout=5)
+ if health and health.get("service") == "strata" and health.get("loaded"):
+ return True
+ if time.monotonic() - announced > 30:
+ log(f"waiting for the model to load ({int(time.monotonic() - began)} s) ...")
+ announced = time.monotonic()
+ time.sleep(2)
+ return False
+
+
+def start_engine(ref: str, target: Target, log_path: Path, log, timeout_s: float):
+ """Start `strata serve --profile REF` as a child of this run. -> the process."""
+ root = Path(__file__).resolve().parents[1]
+ command = [sys.executable, str(root / "strata.py"), "serve", "--profile", ref, "--quiet"]
+ handle = open(log_path, "wb")
+ kwargs = {}
+ if os.name == "nt":
+ kwargs["creationflags"] = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
+ else:
+ kwargs["start_new_session"] = True
+ proc = subprocess.Popen(command, cwd=str(root), stdout=handle, stderr=subprocess.STDOUT, **kwargs)
+ proc._strata_log = handle # closed by stop_engine
+ log("starting the engine (the model load is not part of the benchmark time) ...")
+ if not _start_wait(target, proc, timeout_s, log, log_path):
+ stop_engine(proc)
+ tail = ""
+ try:
+ tail = log_path.read_text(encoding="utf-8", errors="replace")[-600:]
+ except OSError:
+ pass
+ raise ProfileError("the engine did not become ready" + (": " + tail.strip().splitlines()[-1] if tail.strip() else ""), 6)
+ return proc
+
+
+def stop_engine(proc) -> None:
+ """Stop only the process this run started."""
+ if proc is None:
+ return
+ try:
+ if proc.poll() is None:
+ if os.name == "nt":
+ subprocess.run(["taskkill", "/PID", str(proc.pid), "/T", "/F"], capture_output=True, timeout=60)
+ else:
+ import signal
+ try:
+ os.killpg(proc.pid, signal.SIGTERM)
+ except OSError:
+ proc.terminate()
+ try:
+ proc.wait(90)
+ except subprocess.TimeoutExpired:
+ try:
+ os.killpg(proc.pid, signal.SIGKILL)
+ except OSError:
+ proc.kill()
+ try:
+ proc.wait(30)
+ except subprocess.TimeoutExpired:
+ pass
+ finally:
+ handle = getattr(proc, "_strata_log", None)
+ if handle is not None:
+ handle.close()
+
+
+def slug(text: str, limit: int = 40) -> str:
+ cleaned = re.sub(r"(?i)\b(nvidia|geforce|amd|radeon|intel|graphics|corporation)\b", " ", text or "")
+ return re.sub(r"[^a-z0-9]+", "-", cleaned.lower()).strip("-")[:limit].strip("-") or "unknown"
+
+
+def run(target: Target, out: Path | None, *, out_root: Path | None = None, optional_concurrency: bool = False,
+ corpus: Corpus | None = None, declared: dict | None = None, plan: dict | None = None,
+ engine_log: Path | None = None, command: str = "strata bench run --suite standard-v1",
+ log=lambda message: None, timeout_s: float = REQUEST_TIMEOUT_S, started_engine: bool = False,
+ hash_model: bool = True) -> tuple[dict, Path]:
+ """Run the protocol against `target` and write the report into the new directory `out` (or a new folder named
+ <date>-standard-<gpu>-<run id> under `out_root`). -> (the result, the folder)."""
+ declared = {"track": "custom", "track_notes": "no notes given by the operator", **(declared or {})}
+ if declared["track"] not in ("default", "custom"):
+ raise ProfileError("--track must be default or custom", 2)
+ corpus = corpus or load_corpus()
+ if out is not None and (out.exists() or out.is_symlink()):
+ raise ProfileError("benchmark output directory already exists", 2)
+ if out is None and out_root is None:
+ raise ProfileError("benchmark needs an output directory", 2)
+ health = target.get("/health", timeout=10)
+ if not health or health.get("service") != "strata":
+ raise ProfileError("no Strata server answers at that address", 5)
+ if health.get("loaded") is not True:
+ raise ProfileError("the server has no model loaded; the benchmark needs a loaded model", 5)
+ props = target.get("/props", timeout=15)
+ status = target.get("/v1/status", timeout=15) or {}
+ if props is None:
+ raise ProfileError("the server did not answer /props (an API key may be needed: STRATA_API_KEY)", 5)
+ redact = facts.redactor(target.key or "")
+ why = facts.Why()
+ reasons_nonconforming: list[str] = []
+ if not target.loopback:
+ reasons_nonconforming.append("the server is not on loopback: standard latency is measured over loopback")
+
+ served_model = props.get("model_alias") or status.get("model")
+ prior = ((status.get("activity") or {}).get("requests"))
+ # the server's own view of its hardware and build
+ machine = target.get("/metrics", timeout=20) or {}
+ static = machine.get("hardware_static") or {}
+ served_gpu = ((status.get("machine") or {}).get("gpu") or {}).get("name") or static.get("gpu_name")
+ engine_version = ((machine.get("engine") or {}).get("version") or status.get("engine")
+ or (str(props.get("build_info", "")).replace("Strata ", "") or None))
+
+ plan = plan or {}
+ options = facts.parse_options(plan["engine_argv"]) if plan.get("engine_argv") else None
+ model_paths = [str(props.get("model_path") or "")] + [str(v) for v in (plan.get("engine_argv") or [])]
+ files = facts.model_files_from([p for p in model_paths if p]) if hash_model else []
+ model_dir = files[0].parent if files else None
+ selected = facts.gpu_selection(plan.get("config"), plan.get("engine_argv"))
+ template_text = props.get("chat_template") if isinstance(props.get("chat_template"), str) else None
+
+ hardware_block, machine_facts = facts.hardware(why, selected=selected, served_gpu=served_gpu, model_dir=model_dir,
+ declared_background=list(declared.get("background") or []))
+ software_block = facts.software(why, machine_facts, engine_version=engine_version,
+ build_options=declared.get("build_options"))
+ model_block = facts.model(why, props=props, status=status, files=files, declared=declared, log=log)
+ template_ref = {"path": "provenance/chat_template.jinja", "sha256": facts.sha256_text(template_text)} if template_text else None
+ configuration_block, other_options = facts.configuration(
+ why, options=options, env=(plan.get("config") or {}).get("env") if isinstance(plan.get("config"), dict) else None,
+ props=props, declared=declared,
+ sampling={"sampling": SAMPLING, "chat_template_kwargs": TEMPLATE_KWARGS}, template_ref=template_ref)
+
+ if software_block["strata_commit"] is None:
+ reasons_nonconforming.append("the Strata commit is unknown (not a git checkout and no COMMIT file)")
+ if not model_block["files"]:
+ reasons_nonconforming.append("no model file could be hashed")
+ if template_ref is None:
+ reasons_nonconforming.append("the effective chat template could not be read from the server")
+ if props.get("is_sleeping"):
+ reasons_nonconforming.append("the server reported its model as sleeping")
+
+ run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid.uuid4().hex[:8]
+ if out is None:
+ names = "-plus-".join(slug(g["name"], 30) for g in hardware_block["gpus"]) or "cpu"
+ out = out_root / (datetime.now(timezone.utc).strftime("%Y-%m-%d") + "-standard-" + names[:60].strip("-") + "-" + run_id[-8:])
+ out_root.mkdir(parents=True, exist_ok=True)
+ _private_mkdir(out, parents=False, exist_ok=False)
+ started_at = _now()
+ collector = Collector(target, corpus, served_model, log, timeout_s)
+ by_phase = {p: [s for s in corpus.specs if s["phase"] == p] for p in PHASES}
+ required_start = time.monotonic()
+ interrupted = False
+ try:
+ for phase in ("warmup", "performance", "sanity"):
+ for spec in by_phase[phase]:
+ collector.one(spec)
+ required_duration = round(time.monotonic() - required_start, 3)
+ optional_duration = None
+ rounds: list[dict] = []
+ if optional_concurrency:
+ optional_start = time.monotonic()
+ specs = by_phase["concurrency"]
+ for round_no in (1, 2, 3):
+ group = [s for s in specs if s["round"] == round_no]
+ if collector.abort:
+ for spec in group:
+ collector.results[spec["id"]] = skipped_record(spec, collector.abort)
+ rounds.append({"round": round_no, "request_ids": [s["id"] for s in group], "wall_s": None,
+ "output_tokens": None, "aggregate_tps": None, "failed": 0})
+ continue
+ log(f"concurrency round {round_no}/3: four requests released together")
+ rounds.append(collector.concurrent_round(group))
+ if not collector.abort and any(collector.results[s["id"]][0]["status"] == "failed" for s in group) \
+ and not collector.alive():
+ collector.abort = "the server stopped answering during the concurrency stage"
+ optional_duration = round(time.monotonic() - optional_start, 3)
+ except KeyboardInterrupt:
+ interrupted = True
+ required_duration = round(time.monotonic() - required_start, 3)
+ optional_duration, rounds = None, []
+ collector.abort = "the operator cancelled the run"
+ for spec in corpus.specs:
+ if spec["id"] not in collector.results and (spec["phase"] != "concurrency" or optional_concurrency):
+ collector.results[spec["id"]] = skipped_record(spec, collector.abort)
+ for spec in corpus.specs: # a stage that never started is skipped, not missing
+ if spec["id"] not in collector.results and (spec["phase"] != "concurrency" or optional_concurrency):
+ collector.results[spec["id"]] = skipped_record(spec, collector.abort or "not run")
+
+ selected_specs = [s for s in corpus.specs if s["id"] in collector.results]
+ records = [collector.results[s["id"]][0] for s in selected_specs]
+ evidence = [collector.results[s["id"]][1] for s in selected_specs]
+ extras = {s["id"]: collector.results[s["id"]][2] for s in selected_specs}
+
+ for r in records:
+ if r["phase"] == "performance" and extras[r["id"]]["unexpected_payload_reuse"]:
+ reasons_nonconforming.append(f"unexpected reuse of the user payload in {r['id']}")
+ if extras[r["id"]]["length_outside_tolerance"]:
+ reasons_nonconforming.append(f"{r['id']} measured {r['tokens']['input_tokens']} prompt tokens, outside 2% of its target")
+ if r["status"] == "failed" and r["error"]["kind"] in ("http_400", "http_422"):
+ reasons_nonconforming.append(f"the server refused the protocol's required settings in {r['id']} ({r['error']['kind']})")
+ skipped_required = any(r["status"] == "skipped" for r in records if r["phase"] != "concurrency")
+ skipped_optional = any(r["status"] == "skipped" for r in records if r["phase"] == "concurrency")
+ run_status = ("nonconforming" if reasons_nonconforming else
+ "partial" if skipped_required or interrupted else "complete")
+ reasons_nonconforming = list(dict.fromkeys(reasons_nonconforming))
+
+ # the artifacts first, so the report can hash their final bytes
+ artifacts: list[dict] = []
+
+ def put(kind: str, relative: str, data: bytes) -> None:
+ path = out / relative
+ path.parent.mkdir(parents=True, exist_ok=True)
+ _write_new(path, data)
+ artifacts.append({"kind": kind, "path": relative, "sha256": hashlib.sha256(data).hexdigest()})
+ lines = [json.dumps(facts.redact_object(row, redact), ensure_ascii=False, allow_nan=False) for row in evidence]
+ put("outputs", "outputs.jsonl", ("\n".join(lines) + "\n").encode("utf-8"))
+ config_doc = facts.redact_object({"engine_options": options, "other_engine_options": other_options,
+ "server_defaults": (props.get("default_generation_settings") or {}).get("params"),
+ "total_slots": props.get("total_slots"), "context": configuration_block["context_tokens"],
+ "source": "effective engine arguments of the profile" if options is not None
+ else "the server was already running; its engine options are not exposed"}, redact)
+ put("config", "config.json", (json.dumps(config_doc, ensure_ascii=False, indent=2, allow_nan=False) + "\n").encode("utf-8"))
+ if engine_log is not None and engine_log.is_file():
+ text = engine_log.read_text(encoding="utf-8", errors="replace")[-2_000_000:]
+ put("engine_log", "engine.log", redact(text).encode("utf-8"))
+ if template_text:
+ put("provenance", "provenance/chat_template.jinja", template_text.encode("utf-8"))
+ put("provenance", "provenance/corpus-manifest.sha256", corpus.manifest)
+ put("provenance", "provenance/corpus-build.json", (corpus.directory / "build.json").read_bytes())
+
+ summaries = summaries_of(records, extras, "performance", PERFORMANCE_CASES, 1)
+ if optional_concurrency:
+ summaries += summaries_of(records, extras, "concurrency", ("short-burst",), 4)
+ doc = {
+ "format": FORMAT, "schema_version": SCHEMA_VERSION,
+ "protocol": {"id": PROTOCOL_ID, "git_commit": software_block["runner_commit"], "profile": PROFILE,
+ "corpus_sha256": corpus.sha256, "model_card_revision": MODEL_CARD_REVISION},
+ "run": {"id": run_id, "started_at": started_at, "status": run_status, "required_duration_s": required_duration,
+ "optional_duration_s": optional_duration, "optional_concurrency": bool(optional_concurrency),
+ "command": command},
+ "hardware": hardware_block, "software": software_block, "model": model_block,
+ "configuration": configuration_block,
+ "request_counts": {p: counts_of(records, p, PLANNED[p] if p != "concurrency" or optional_concurrency else 0)
+ for p in PHASES},
+ "requests": records, "summaries": summaries, "concurrency_rounds": rounds if optional_concurrency else [],
+ "quality": quality_of(records), "artifacts": artifacts, "missing": {},
+ "extensions": {"suite": SUITE, "runner": "strata_cli.bench_standard",
+ "status_reasons": reasons_nonconforming or (
+ [collector.abort] if collector.abort else []),
+ "started_engine": started_engine,
+ "server_requests_before_run": prior if isinstance(prior, int) else None,
+ "other_engine_options": other_options,
+ "request_checks": {rid: v for rid, v in extras.items()},
+ "optional_stage_skipped": skipped_optional,
+ "api_key_sent": bool(target.key)}}
+ if doc["run"]["optional_duration_s"] is None:
+ why.add("/run/optional_duration_s", "the optional concurrency stage was not selected" if not optional_concurrency
+ else "the optional concurrency stage did not run")
+ for pointer_text in scan_nulls(doc):
+ doc["missing"][pointer_text] = why.get(pointer_text) or default_reason(doc, pointer_text)
+ problems = validate(doc)
+ if problems:
+ raise ProfileError("the benchmark report failed its own validation: " + problems[0], 6)
+ encoded = (json.dumps(doc, ensure_ascii=False, indent=2, allow_nan=False) + "\n").encode("utf-8")
+ _write_new(out / "results.json", encoded)
+ _write_new(out / "README.md", report_markdown(doc, redact).encode("utf-8"))
+ return doc, out
+
+
+# ------------------------------------------------------------------------------------------------------------ validation
+def validate(doc: dict) -> list[str]:
+ """A strict check of the result contract: shape, types, count identities, null reasons and summaries rebuilt from the requests."""
+ bad: list[str] = []
+
+ def need(condition: bool, message: str) -> None:
+ if not condition:
+ bad.append(message)
+
+ if not isinstance(doc, dict):
+ return ["the result is not an object"]
+ need(doc.get("format") == FORMAT and doc.get("schema_version") == SCHEMA_VERSION, "unknown format or schema version")
+ required = ("protocol", "run", "hardware", "software", "model", "configuration", "request_counts", "requests", "summaries",
+ "concurrency_rounds", "quality", "artifacts", "missing")
+ for key in required:
+ need(key in doc, f"missing top-level field {key}")
+ if bad:
+ return bad
+ allowed = {"format", "schema_version", *required, "extensions"}
+ need(set(doc) <= allowed, "unknown top-level fields must be under extensions")
+ protocol, run_block = doc["protocol"], doc["run"]
+ need(all(k in protocol for k in ("id", "git_commit", "profile", "corpus_sha256", "model_card_revision")), "protocol fields")
+ need(isinstance(protocol.get("corpus_sha256"), str) and bool(HEX64.match(protocol["corpus_sha256"])), "corpus_sha256 is not a SHA-256")
+ for field in ("id", "started_at", "status", "required_duration_s", "optional_duration_s", "optional_concurrency", "command"):
+ need(field in run_block, f"run.{field} missing")
+ need(run_block.get("status") in ("complete", "partial", "nonconforming"), "run.status")
+ need(bool(re.fullmatch(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ", str(run_block.get("started_at")))), "run.started_at is not UTC ISO 8601")
+ for field in ("gpus", "cpu", "ram_bytes", "storage", "background_workloads"):
+ need(field in doc["hardware"], f"hardware.{field} missing")
+ for gpu in doc["hardware"].get("gpus") or []:
+ need(gpu.get("memory_kind") in ("dedicated", "shared"), "gpu memory_kind")
+ need(isinstance(gpu.get("device_index"), int), "gpu device_index")
+ for field in ("os", "drivers", "runtime", "strata_commit", "engine_version", "build_kind", "runner_commit", "build_options"):
+ need(field in doc["software"], f"software.{field} missing")
+ need(doc["software"].get("build_kind") in ("release", "source"), "software.build_kind")
+ for field in ("id", "revision", "source_revision", "quantization", "files", "tokenizer_revision", "custom_pack"):
+ need(field in doc["model"], f"model.{field} missing")
+ for entry in doc["model"].get("files") or []:
+ need(bool(HEX64.match(str(entry.get("sha256")))) and "/" not in str(entry.get("path")) and "\\" not in str(entry.get("path")),
+ "model file entry")
+ cfg = doc["configuration"]
+ for field in ("track", "track_notes", "context_tokens", "server_capacity", "kv_cache", "expert_cache", "offloading", "prefill",
+ "mtp", "calibration", "experimental_options", "template", "sampling", "chat_template_kwargs"):
+ need(field in cfg, f"configuration.{field} missing")
+ need(cfg.get("track") in ("default", "custom"), "configuration.track")
+ need(all(k in cfg.get("sampling", {}) for k in SAMPLING), "configuration.sampling")
+
+ records = doc["requests"]
+ ids = [r.get("id") for r in records]
+ need(len(set(ids)) == len(ids), "duplicate request ids")
+ need(len(records) in (22, 34), "the requests array has neither 22 nor 34 records")
+ sanity_ids = {f"sanity-{n}" for n in ("I1", "I2", "I3", "I4", "R1", "R2", "R3", "R4", "T1", "T2", "T3", "T4")}
+ need(sanity_ids <= set(ids), "a sanity case is missing")
+ for r in records:
+ need(r.get("status") in ("completed", "truncated", "failed", "skipped"), f"{r.get('id')}: status")
+ need(r.get("phase") in PHASES, f"{r.get('id')}: phase")
+ if r.get("status") in ("completed", "truncated"):
+ need(r.get("error") is None, f"{r.get('id')}: error on a finished request")
+ else:
+ need(isinstance(r.get("error"), dict) and "kind" in r["error"] and "message" in r["error"], f"{r.get('id')}: error object")
+ for group, keys in (("tokens", ("input_tokens", "cached_tokens", "read_tokens", "output_tokens", "decode_tokens")),
+ ("timings", ("prompt_s", "decode_s", "ttft_s", "first_answer_s", "total_s")),
+ ("metrics", ("prompt_tps", "decode_tps")),
+ ("sources", ("input_tokens", "cached_tokens", "read_tokens", "output_tokens", "decode_tokens",
+ "engine_timings", "client_timings"))):
+ need(isinstance(r.get(group), dict) and all(k in r[group] for k in keys), f"{r.get('id')}: {group}")
+ for k, v in (r.get("tokens") or {}).items():
+ need(v is None or (isinstance(v, int) and not isinstance(v, bool) and v >= 0), f"{r.get('id')}: tokens.{k}")
+ for group in ("timings", "metrics"):
+ for k, v in (r.get(group) or {}).items():
+ need(v is None or (isinstance(v, (int, float)) and not isinstance(v, bool) and math.isfinite(v) and v >= 0),
+ f"{r.get('id')}: {group}.{k}")
+ if r.get("phase") == "sanity":
+ need(isinstance(r.get("evaluation"), dict) and isinstance(r["evaluation"].get("passed"), bool), f"{r.get('id')}: evaluation")
+ else:
+ need(r.get("evaluation") is None, f"{r.get('id')}: evaluation outside sanity")
+ need((r.get("round") is not None) == (r.get("phase") == "concurrency"), f"{r.get('id')}: round")
+ counts = doc["request_counts"]
+ for phase in PHASES:
+ rows = [r for r in records if r.get("phase") == phase]
+ c = counts.get(phase) or {}
+ need(c.get("attempted") == c.get("completed", 0) + c.get("truncated", 0) + c.get("failed", 0), f"{phase}: attempted identity")
+ need(c.get("planned") == c.get("attempted", 0) + c.get("skipped", 0), f"{phase}: planned identity")
+ need(len(rows) == c.get("planned"), f"{phase}: record count")
+ need(doc["quality"] == quality_of(records) if len(records) >= 22 else False, "quality does not match the sanity verdicts")
+ need(isinstance(run_block.get("optional_concurrency"), bool), "run.optional_concurrency")
+ need((counts["concurrency"]["planned"] == 12) == bool(run_block.get("optional_concurrency")), "concurrency count")
+
+ extras = {k: v for k, v in (doc.get("extensions") or {}).get("request_checks", {}).items()}
+ if extras:
+ rebuilt = summaries_of(records, extras, "performance", PERFORMANCE_CASES, 1)
+ if run_block.get("optional_concurrency"):
+ rebuilt += summaries_of(records, extras, "concurrency", ("short-burst",), 4)
+ need(rebuilt == doc["summaries"], "summaries cannot be rebuilt from the request records")
+ for row in doc["summaries"]:
+ need(row["n"] == len(row["included_request_ids"]), "summary n")
+ if row["n"] == 0:
+ need(row["median"] is None and row["min"] is None and row["max"] is None, "empty summary must be null")
+ for artifact in doc["artifacts"]:
+ need(artifact.get("kind") in ("config", "engine_log", "outputs", "provenance") and bool(HEX64.match(str(artifact.get("sha256"))))
+ and not str(artifact.get("path")).startswith(("/", "\\")) and ".." not in Path(str(artifact.get("path"))).parts
+ and not re.match(r"^[A-Za-z]:", str(artifact.get("path"))), "artifact entry")
+ nulls = set(scan_nulls(doc))
+ need(nulls == set(doc["missing"]), "missing must list exactly the null values: "
+ f"{sorted(nulls ^ set(doc['missing']))[:3]}")
+ need(all(isinstance(v, str) and v for v in doc["missing"].values()), "every missing entry needs a reason")
+ try:
+ json.dumps(doc, allow_nan=False)
+ except ValueError:
+ bad.append("the result contains NaN or infinity")
+ return bad
+
+
+# ------------------------------------------------------------------------------------------------------------ reading it out
+def _span(row: dict | None, digits: int = 1) -> str:
+ if not row or row["n"] == 0:
+ return "not measured"
+ return f"{row['median']:.{digits}f} ({row['min']:.{digits}f}-{row['max']:.{digits}f}) n={row['n']}"
+
+
+def _find(doc: dict, phase: str, case: str, metric: str) -> dict | None:
+ return next((s for s in doc["summaries"] if s["phase"] == phase and s["case_id"] == case and s["metric"] == metric), None)
+
+
+def _minutes(seconds: float | None) -> str:
+ if seconds is None:
+ return "not measured"
+ return f"{int(seconds // 60)} min {int(seconds % 60)} s"
+
+
+def table_lines(doc: dict, phase: str, cases: tuple[str, ...]) -> list[str]:
+ head = ["Case", "Prompt tok/s", "Decode tok/s", "TTFT s", "Total s"]
+ rows = [head]
+ for case in cases:
+ rows.append([case, _span(_find(doc, phase, case, "prompt_tps")), _span(_find(doc, phase, case, "decode_tps")),
+ _span(_find(doc, phase, case, "ttft_s"), 2), _span(_find(doc, phase, case, "total_s"), 1)])
+ widths = [max(len(r[i]) for r in rows) for i in range(len(head))]
+ return [" " + " ".join(cell.ljust(widths[i]) for i, cell in enumerate(row)).rstrip() for row in rows]
+
+
+def format_plain(doc: dict) -> str:
+ """The summary printed at the end of a run: what ran, how fast, what passed, what to be careful about."""
+ hw, sw, run_block = doc["hardware"], doc["software"], doc["run"]
+ gpu = ", ".join(g["name"] for g in hw["gpus"]) or "no GPU found"
+ counts = doc["request_counts"]
+ done = sum(counts[p]["completed"] + counts[p]["truncated"] for p in ("performance", "sanity"))
+ quality = doc["quality"]
+ lines = [f"Strata standard benchmark ({doc['protocol']['id']})",
+ f"Status: {run_block['status']}. {done} of 21 required requests finished, "
+ f"{sum(counts[p]['failed'] for p in PHASES)} failed, {sum(counts[p]['skipped'] for p in PHASES)} skipped. "
+ f"Time after the model loaded: {_minutes(run_block['required_duration_s'])}.",
+ f"Machine: {gpu}; {hw['cpu'] or 'CPU not read'}; "
+ f"{round(hw['ram_bytes'] / 2 ** 30) if hw['ram_bytes'] else 'unknown'} GB RAM; {sw['os']['name']} {sw['os']['version']}.",
+ f"Model: {doc['model']['id'] or 'unknown'}, {doc['model']['quantization'] or 'quantization not read'}, "
+ f"context {doc['configuration']['context_tokens'] or 'unknown'}; engine {sw['engine_version'] or 'unknown'}; "
+ f"track {doc['configuration']['track']}.", "",
+ "Speed (median, min-max, n; the engine's own prompt and decode timings, the client's first-token time):"]
+ lines += table_lines(doc, "performance", PERFORMANCE_CASES)
+ lines += ["", f"Sanity checks: {quality['passed']} of 12 passed "
+ f"(instruction {quality['categories']['instruction']['passed']}/4, retrieval "
+ f"{quality['categories']['retrieval']['passed']}/4, tool calls {quality['categories']['tool_call']['passed']}/4)."]
+ failed = [r["evaluation"] and f"{r['case_id']}: {r['evaluation']['reason']}" for r in doc["requests"]
+ if r["phase"] == "sanity" and not r["evaluation"]["passed"]]
+ lines += [" Not passed: " + "; ".join(failed)] if failed else []
+ if doc["run"]["optional_concurrency"]:
+ rounds = [r["aggregate_tps"] for r in doc["concurrency_rounds"] if r["aggregate_tps"] is not None]
+ lines += ["", "Four requests at once (server capacity "
+ f"{doc['configuration']['server_capacity'] or 'unknown'}): "
+ + (", ".join(f"{v:.1f}" for v in rounds) + " tok/s in total per round" if rounds else "no round finished")]
+ reasons = doc["extensions"]["status_reasons"]
+ if run_block["status"] != "complete" and reasons:
+ lines += ["", "Why this is not a complete standard result:"] + [f" - {r}" for r in reasons]
+ failures = [f"{r['id']}: {r['error']['kind']}" for r in doc["requests"] if r["status"] == "failed"]
+ if failures:
+ lines += ["", "Failed requests: " + "; ".join(failures)]
+ return "\n".join(lines)
+
+
+def report_markdown(doc: dict, redact=lambda text: text) -> str:
+ hw, sw, run_block, model, cfg = doc["hardware"], doc["software"], doc["run"], doc["model"], doc["configuration"]
+ counts = doc["request_counts"]
+ quality = doc["quality"]
+ gpu = "; ".join(("%s (%.1f GiB %s)" % (g["name"], g["memory_bytes"] / 2 ** 30, g["memory_kind"])) if g["memory_bytes"]
+ else g["name"] for g in hw["gpus"]) or "not measured"
+ nm = "not measured"
+ ram = "%.1f GiB" % (hw["ram_bytes"] / 2 ** 30) if hw["ram_bytes"] else nm
+ storage = ", ".join("%s %s (%s)" % (s["kind"], s["model"], s["bus"]) for s in hw["storage"]) if hw["storage"] else nm
+ drivers = ", ".join("%s %s" % (d["name"], d["version"]) for d in sw["drivers"]) if sw["drivers"] else nm
+ runtime = ", ".join("%s %s" % (d["name"], d["version"]) for d in sw["runtime"]) if sw["runtime"] else nm
+ lines = [
+ "# Strata standard benchmark: %s" % (", ".join(g["name"] for g in hw["gpus"]) or "unknown GPU"), "",
+ "Protocol `%s` (draft), run `%s`, started %s. **Status: %s.**" % (doc["protocol"]["id"], run_block["id"],
+ run_block["started_at"], run_block["status"]), "",
+ "Written by `strata bench run`. The machine-readable result is [results.json](results.json); every request, its raw usage "
+ "and timings and the client event offsets are in [outputs.jsonl](outputs.jsonl).", ""]
+ reasons = doc["extensions"]["status_reasons"]
+ if run_block["status"] != "complete":
+ lines += ["This run is not a complete standard result:", ""] + ["- " + r for r in reasons] + [""]
+ lines += [
+ "## Hardware and software", "",
+ "- GPU: " + gpu,
+ "- CPU: %s; RAM: %s" % (hw["cpu"] or nm, ram),
+ "- Storage of the model files: " + storage,
+ "- OS: %s %s; drivers: %s; runtime: %s" % (sw["os"]["name"], sw["os"]["version"], drivers, runtime),
+ "- Strata commit: `%s`; engine %s; %s build" % (sw["strata_commit"] or nm, sw["engine_version"] or nm, sw["build_kind"]),
+ "- Background work: " + ("; ".join(hw["background_workloads"]) if hw["background_workloads"] else nm), "",
+ "## Model and configuration", "",
+ f"- Model: {model['id'] or nm}; quantization {model['quantization'] or nm}; revision {model['revision'] or nm}; "
+ f"source revision {model['source_revision'] or nm}",
+ f"- Files: " + ("; ".join(f"`{f['path']}` `{f['sha256'][:16]}...`" for f in model['files']) if model['files'] else nm),
+ f"- Track: **{cfg['track']}** ({cfg['track_notes']})",
+ f"- Context {cfg['context_tokens'] or nm} tokens; server capacity {cfg['server_capacity'] or nm} slot(s)",
+ f"- Sampling: " + ", ".join(f"{k} {v}" for k, v in cfg["sampling"].items()) + "; thinking on, preserved",
+ f"- Engine options: `config.json`; chat template: `provenance/chat_template.jinja`", "",
+ "## Method", "",
+ f"The frozen requests of `bench/prompts/standard-v1` (corpus `{doc['protocol']['corpus_sha256'][:16]}...`) were sent over "
+ "loopback HTTP, streaming, one at a time, in this order: 1 warm-up (excluded), 3 repetitions of the short, 4K and 16K prompts, "
+ "and the twelve sanity checks. Prompt and decode speed are the engine's own `timings`; time to first token is the client's "
+ "monotonic clock from dispatch to the first nonempty reasoning, answer or tool-call delta. No request was retried. "
+ f"Time after the model loaded: {_minutes(run_block['required_duration_s'])}.", "",
+ "## Results", ""]
+ lines += ["| Case | Prompt tok/s | Decode tok/s | TTFT s | First answer s | Total s |", "| --- | --- | --- | --- | --- | --- |"]
+ for case in PERFORMANCE_CASES:
+ lines.append(f"| {case} | " + " | ".join(
+ _span(_find(doc, "performance", case, m), d) for m, d in (("prompt_tps", 1), ("decode_tps", 1), ("ttft_s", 2),
+ ("first_answer_s", 2), ("total_s", 1))) + " |")
+ lines += ["", "Cells are median (min-max) n=samples.", "",
+ "| Phase | Planned | Completed | Truncated | Failed | Skipped |", "| --- | ---: | ---: | ---: | ---: | ---: |"]
+ for phase in PHASES:
+ c = counts[phase]
+ lines.append(f"| {phase} | {c['planned']} | {c['completed']} | {c['truncated']} | {c['failed']} | {c['skipped']} |")
+ lines += ["", f"Sanity checks: **{quality['passed']} of 12**.", "", "| Case | Result | Reason |", "| --- | --- | --- |"]
+ for r in doc["requests"]:
+ if r["phase"] == "sanity":
+ lines.append(f"| {r['case_id']} | {'pass' if r['evaluation']['passed'] else 'fail'} | {r['evaluation']['reason']} |")
+ if run_block["optional_concurrency"]:
+ lines += ["", "Concurrency rounds (four requests released together):", "",
+ "| Round | Wall s | Output tokens | Aggregate tok/s | Failed |", "| ---: | ---: | ---: | ---: | ---: |"]
+ for r in doc["concurrency_rounds"]:
+ lines.append(f"| {r['round']} | {r['wall_s']} | {r['output_tokens']} | {r['aggregate_tps']} | {r['failed']} |")
+ failures = [r for r in doc["requests"] if r["status"] == "failed"]
+ lines += ["", "## Failures, exclusions and limitations", ""]
+ lines += [f"- {r['id']} failed: {r['error']['kind']} ({r['error']['message']})" for r in failures] or ["- No request failed."]
+ excluded = {(e["request_id"], e["reason"]) for s in doc["summaries"] for e in s["excluded"]}
+ lines += [f"- Excluded from speed summaries: {rid}: {reason}" for rid, reason in sorted(excluded)]
+ lines += ["- Prompt-prefix reuse is the shared template prefix only; the cache was not emptied between requests, and the expert "
+ "cache stays warm and evolves. These are not cold-model measurements.",
+ "- Output caps include reasoning; a request that reaches its cap is `truncated` and fails its sanity check.",
+ "- The runner cannot see other programs on the machine, the repeat-stop and loop-recovery switches of the server, or "
+ "the quantization repository's revision; see `missing` in results.json for every value it could not read.",
+ f"- The server had answered {doc['extensions']['server_requests_before_run']} request(s) before this run."
+ if doc["extensions"]["server_requests_before_run"] else
+ "- The runner could not tell whether the server was freshly loaded.", ""]
+ return redact("\n".join(lines))
+
+
+def load(path: Path) -> dict:
+ """Read a saved standard result and validate it. A result in another format or schema version is refused, never merged."""
+ target = path / "results.json" if path.is_dir() else path
+ try:
+ if target.stat().st_size > 64 << 20:
+ raise ProfileError("benchmark result is larger than 64 MiB", 5)
+ doc = json.loads(target.read_text(encoding="utf-8"))
+ except (OSError, UnicodeError, ValueError):
+ raise ProfileError("cannot read the benchmark result", 5) from None
+ problems = validate(doc)
+ if problems:
+ raise ProfileError("not a valid standard benchmark result: " + problems[0], 5)
+ return doc
+
+
+def load_if_standard(path: Path) -> dict | None:
+ """The validated result at `path` when it is a standard benchmark result; None when it is another kind of report."""
+ target = path / "results.json" if path.is_dir() else path
+ try:
+ head = json.loads(target.read_text(encoding="utf-8"))
+ except (OSError, UnicodeError, ValueError):
+ return None
+ if not isinstance(head, dict) or head.get("format") != FORMAT:
+ return None
+ return load(path)
diff --git a/strata_cli/cli.py b/strata_cli/cli.py
index 276d1f72f..caf5c960b 100644
--- a/strata_cli/cli.py
+++ b/strata_cli/cli.py
@@ -278,24 +278,48 @@ def parser() -> argparse.ArgumentParser:
help="require the candidate run-config bytes shown in a restart recipe plan")
bench = sp.add_parser("bench", help="explicit serving benchmark with local JSON evidence")
bench_sp = bench.add_subparsers(dest="bench_command", required=True)
- bench_run = bench_sp.add_parser("run", help="measure streaming requests and sample serving telemetry")
+ bench_run = bench_sp.add_parser("run", help="the standard benchmark (docs/BENCHMARK_PROTOCOL.md), or a serving suite")
bench_source = bench_run.add_mutually_exclusive_group(required=True)
bench_source.add_argument("--profile", help="saved serving profile or installed run config")
bench_source.add_argument("--url", help="server root; remote connections require HTTPS")
- bench_run.add_argument("--suite", choices=("serving-smoke", "parallel-mixed-v1", "parallel-overload-v1",
+ bench_run.add_argument("--suite", choices=("standard-v1", "serving-smoke", "parallel-mixed-v1", "parallel-overload-v1",
"parallel-cancel-v1", "parallel-prefill-cancel-v1"),
- default="serving-smoke")
- bench_run.add_argument("--concurrency", type=int, default=4,
- help="simultaneous requests (1..16, or 1..64 for parallel-overload-v1)")
- bench_run.add_argument("--repeat", type=int, default=1, help="repeat measured phases (1..5)")
- bench_run.add_argument("--condition", help="operator label for this run, e.g. baseline or monitor-default")
+ default="standard-v1",
+ help="standard-v1 (default) follows docs/BENCHMARK_PROTOCOL.md; the others are serving suites for your own comparisons")
+ bench_run.add_argument("--concurrency", type=int, default=None,
+ help="serving suites: simultaneous requests (1..16, or 1..64 for parallel-overload-v1; default 4)")
+ bench_run.add_argument("--repeat", type=int, default=None, help="serving suites: repeat measured phases (1..5; default 1)")
+ bench_run.add_argument("--condition", help="serving suites: operator label for this run, e.g. baseline or monitor-default")
bench_run.add_argument("--require-managed-match", action="store_true",
- help="require the local owner host and current profile to match the serving instance")
- bench_run.add_argument("--request-timeout", type=float, default=120)
- bench_run.add_argument("--out", required=True, metavar="NEW_DIRECTORY")
+ help="serving suites: require the local owner host and current profile to match the serving instance")
+ bench_run.add_argument("--request-timeout", type=float, default=None,
+ help="serving suites: seconds per request (1..600; default 120). The standard suite fixes it at 1800")
+ bench_run.add_argument("--out", metavar="NEW_DIRECTORY",
+ help="new output directory (required for serving suites). Standard suite: a new folder "
+ "<date>-standard-<gpu>-<run id> under ./bench/results or ./bench-results")
bench_run.add_argument("--credential-env", default="STRATA_API_KEY")
bench_run.add_argument("--telemetry-credential-env", default=None)
bench_run.add_argument("--output", choices=("human", "json"), default="human")
+ standard = bench_run.add_argument_group("standard suite (docs/BENCHMARK_PROTOCOL.md)")
+ standard.add_argument("--optional-concurrency", action="store_true",
+ help="also run the optional stage: four short requests released together, three rounds")
+ standard.add_argument("--no-start", action="store_true",
+ help="with --profile: never start the engine; use only a server that is already running")
+ standard.add_argument("--load-timeout", type=float, default=1800, metavar="SECONDS",
+ help="with --profile: how long to wait for the engine to load the model (default 1800)")
+ standard.add_argument("--track", choices=("default", "custom"), default="custom",
+ help="default: an unmodified release's setup-generated settings; custom (the default here): anything else")
+ standard.add_argument("--track-notes", default=None, metavar="TEXT",
+ help="the release and setup choices (default track) or the changes made (custom track)")
+ standard.add_argument("--model-revision", metavar="REV", help="immutable revision of the quantization repository")
+ standard.add_argument("--source-revision", metavar="REV", help="revision of the original Qwen/Qwen3.8-Flash-Next weights")
+ standard.add_argument("--tokenizer-revision", metavar="REV")
+ standard.add_argument("--quantization", metavar="NAME", help="when it cannot be read from the model file names")
+ standard.add_argument("--custom-pack", metavar="TEXT", help="describe a prepared pack or custom expert profile")
+ standard.add_argument("--background", action="append", metavar="TEXT", help="a program that ran during the benchmark (repeatable)")
+ standard.add_argument("--build-options", metavar="LIST", help="comma-separated build options of a source build")
+ standard.add_argument("--skip-model-hash", action="store_true",
+ help="do not hash the model files (the run is then marked nonconforming)")
bench_cancel = bench_sp.add_parser("cancel", help="stop the exact active saved-profile benchmark")
bench_cancel.add_argument("--profile", required=True, help="saved serving profile")
bench_cancel.add_argument("--path", required=True, help="absolute run directory or results.json")
@@ -1000,6 +1024,22 @@ def _bench(a: argparse.Namespace) -> int:
if a.bench_command in ("report", "compare") and a.out and a.output != "markdown":
return _error("--out requires --output markdown", 2, machine)
if a.bench_command == "report":
+ from strata_cli import bench_standard
+ standard_report = bench_standard.load_if_standard(Path(a.path))
+ if standard_report is not None:
+ if a.output == "markdown":
+ rendered = bench_standard.report_markdown(standard_report)
+ if a.out:
+ bench.write_markdown(Path(a.out), rendered)
+ print(f"Markdown report: {Path(a.out).resolve()}")
+ else:
+ print(rendered, end="")
+ return 0
+ return _success({"summary": {"format": standard_report["format"], "run_id": standard_report["run"]["id"],
+ "status": standard_report["run"]["status"],
+ "quality": standard_report["quality"],
+ "summaries": standard_report["summaries"]}}, machine=machine,
+ human=bench_standard.format_plain(standard_report))
report = bench.load(Path(a.path))
result = bench.summary(report)
if a.output == "markdown":
@@ -1035,6 +1075,13 @@ def _bench(a: argparse.Namespace) -> int:
if not a.credential_env.isidentifier() or (a.telemetry_credential_env and
not a.telemetry_credential_env.isidentifier()):
return _error("credential options must name environment variables", 2, machine)
+ if a.suite == "standard-v1":
+ return _bench_standard(a, machine)
+ if a.out is None:
+ return _error("--out is required for the serving suites", 2, machine)
+ a.concurrency = 4 if a.concurrency is None else a.concurrency
+ a.repeat = 1 if a.repeat is None else a.repeat
+ a.request_timeout = 120 if a.request_timeout is None else a.request_timeout
maximum = 64 if a.suite == "parallel-overload-v1" else 16
if not 1 <= a.concurrency <= maximum or not 1 <= a.repeat <= 5 or not 1 <= a.request_timeout <= 600:
return _error(f"concurrency must be 1..{maximum}, repeat 1..5 and request timeout 1..600 seconds", 2,
@@ -1079,6 +1126,86 @@ def _bench(a: argparse.Namespace) -> int:
return 0 if report["successful"] else 6
+def _bench_standard(a: argparse.Namespace, machine: bool) -> int:
+ """`strata bench run` for docs/BENCHMARK_PROTOCOL.md: against a running server (--url, or a profile that is already serving),
+ or by starting the profile's engine for the run and stopping it afterwards."""
+ from strata_cli import bench_facts, bench_standard
+ fixed = [name for name, value in (("--concurrency", a.concurrency), ("--repeat", a.repeat),
+ ("--request-timeout", a.request_timeout), ("--condition", a.condition),
+ ("--require-managed-match", a.require_managed_match or None),
+ ("--telemetry-credential-env", a.telemetry_credential_env))
+ if value is not None]
+ if fixed:
+ return _error(f"{', '.join(fixed)} do not apply to the standard suite: the protocol fixes the workload "
+ "(use --suite serving-smoke for a serving benchmark)", 2, machine)
+ if not a.credential_env.isidentifier():
+ return _error("credential options must name environment variables", 2, machine)
+ if not 1 <= a.load_timeout <= 86400:
+ return _error("--load-timeout must be 1..86400 seconds", 2, machine)
+ plan = None
+ saved = None
+ key = os.environ.get(a.credential_env)
+ if a.profile:
+ _, saved, _, cfg = _source(a.profile)
+ plan, _ = _launch_plan(a.profile)
+ plan = {**plan, "config": cfg}
+ host = plan["effective"]["host"]
+ host = "127.0.0.1" if host in ("0.0.0.0", "::") else host
+ authority = f"[{host}]" if ":" in host and not host.startswith("[") else host
+ url = f"http://{authority}:{plan['effective']['port']}"
+ if not key and cfg.get("api_key"):
+ from serve.server import api_key_of, key_list
+ keys = key_list(api_key_of(cfg["api_key"]))
+ key = keys[0] if keys else None
+ else:
+ url = a.url
+ from strata_cli.client import server_url
+ target = bench_standard.Target(server_url(url), key)
+ redact = bench_facts.redactor(key or "")
+ command = redact("strata " + " ".join(str(part) for part in (a.raw_argv or ["bench", "run", "--suite", "standard-v1"])))
+ declared = {"track": a.track, "track_notes": a.track_notes or "no notes given by the operator",
+ "model_revision": a.model_revision, "source_revision": a.source_revision,
+ "tokenizer_revision": a.tokenizer_revision, "quantization": a.quantization,
+ "custom_pack": a.custom_pack, "background": a.background or [],
+ "build_options": [x.strip() for x in a.build_options.split(",") if x.strip()] if a.build_options else None}
+ out = Path(a.out) if a.out else None
+ out_root = None if out else (Path.cwd() / "bench" / "results" if (Path.cwd() / "bench" / "results").is_dir()
+ else Path.cwd() / "bench-results")
+ from strata_control.benchmarking import benchmark_lease
+ import shutil
+ import tempfile
+ lease = benchmark_lease(saved["id"], out) if saved else contextlib.nullcontext()
+ proc, scratch, engine_log = None, None, None
+ try:
+ with lease, contextlib.redirect_stdout(sys.stderr):
+ def log(message: str) -> None:
+ print(message, file=sys.stderr, flush=True)
+ if a.profile and target.get("/health", timeout=5) is None:
+ if a.no_start:
+ raise ProfileError("no server is answering at the profile's address and --no-start was given", 5)
+ scratch = Path(tempfile.mkdtemp(prefix="strata-bench-"))
+ engine_log = scratch / "engine.log"
+ proc = bench_standard.start_engine(a.profile, target, engine_log, log, a.load_timeout)
+ elif a.profile:
+ log("using the server that is already running at this profile's address; it will not be stopped")
+ doc, out_dir = bench_standard.run(target, out, out_root=out_root, optional_concurrency=a.optional_concurrency,
+ declared=declared, plan=plan, engine_log=engine_log, command=command, log=log,
+ started_engine=proc is not None, hash_model=not a.skip_model_hash)
+ finally:
+ bench_standard.stop_engine(proc)
+ if scratch is not None:
+ shutil.rmtree(scratch, ignore_errors=True)
+ ok = doc["run"]["status"] == "complete"
+ path = out_dir / "results.json"
+ if machine:
+ print(json.dumps({"schema_version": 1, "ok": ok, "path": str(path.resolve()), "status": doc["run"]["status"],
+ "quality": doc["quality"], "summaries": doc["summaries"]}, ensure_ascii=False, allow_nan=False))
+ else:
+ print(bench_standard.format_plain(doc))
+ print(f"\nReport: {out_dir.resolve() / 'README.md'}\nResult: {path.resolve()}")
+ return 0 if ok else 6
+
+
def _runtime(a: argparse.Namespace) -> int:
from strata_control import management
machine = a.output == "json"
@@ -1965,6 +2092,7 @@ def main(argv: list[str] | None = None) -> int:
pass
ap = parser()
a, extras = ap.parse_known_args(argv)
+ a.raw_argv = list(sys.argv[1:] if argv is None else argv)
if extras:
if getattr(a, "command", None) in ("launch", "connect"):
from strata_cli.launch import cmd as launch_cmd
diff --git a/tests/terminal/test_bench.py b/tests/terminal/test_bench.py
index e066ea86b..b90d751d1 100644
--- a/tests/terminal/test_bench.py
+++ b/tests/terminal/test_bench.py
@@ -394,7 +394,7 @@ class BenchTests(unittest.TestCase):
out = self.root / "cli-baseline"
output = io.StringIO()
with mock.patch.dict(os.environ, {"STRATA_API_KEY": self.service.api_key}), redirect_stdout(output):
- code = main(["bench", "run", "--url", self.url, "--condition", "baseline",
+ code = main(["bench", "run", "--suite", "serving-smoke", "--url", self.url, "--condition", "baseline",
"--concurrency", "1", "--out", str(out), "--output", "json"])
self.assertEqual(code, 0)
self.assertEqual(json.loads(output.getvalue())["summary"]["condition"], "baseline")
@@ -454,7 +454,7 @@ class BenchTests(unittest.TestCase):
def test_managed_match_flag_requires_a_saved_profile(self):
output = io.StringIO()
with redirect_stdout(output):
- code = main(["bench", "run", "--url", self.url, "--require-managed-match",
+ code = main(["bench", "run", "--suite", "serving-smoke", "--url", self.url, "--require-managed-match",
"--out", str(self.root / "unused"), "--output", "json"])
self.assertEqual(code, 2)
self.assertFalse(json.loads(output.getvalue())["ok"])
diff --git a/tests/terminal/test_bench_standard.py b/tests/terminal/test_bench_standard.py
new file mode 100644
index 000000000..b966f2cb6
--- /dev/null
+++ b/tests/terminal/test_bench_standard.py
@@ -0,0 +1,972 @@
+"""The standard benchmark (docs/BENCHMARK_PROTOCOL.md) against a scripted mock server: no engine, no model weights."""
+
+from __future__ import annotations
+
+import contextlib
+import copy
+import http.client
+import io
+import json
+import os
+import re
+import shutil
+import tempfile
+import threading
+import time
+import unittest
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+from unittest import mock
+
+from strata_cli import bench, bench_facts, bench_standard as bs
+from strata_cli.cli import main
+from strata_control.profiles import ProfileError
+
+COMMIT = "a" * 40
+TEMPLATE = "{{ messages }} mock chat template"
+
+
+def corpus_specs() -> dict[str, dict]:
+ return {spec["id"]: spec for spec in bs.load_corpus().specs}
+
+
+class MockStrata:
+ """A server that speaks enough of Strata's API: /health, /props, /v1/status, /metrics and streamed chat completions."""
+
+ def __init__(self, model_file: Path):
+ self.specs = corpus_specs()
+ self.model_file = model_file
+ self.script: dict[str, dict] = {}
+ self.bodies: list[dict] = []
+ self.order: list[str] = []
+ self.arrivals: list[float] = []
+ self.active = 0
+ self.max_active = 0
+ self.dead = False
+ self.lock = threading.Lock()
+ self.requests_before = 0
+ self.gpu_name = "Mock GPU 12 GB"
+ owner = self
+
+ class Handler(BaseHTTPRequestHandler):
+ def log_message(self, *args):
+ pass
+
+ def _json(self, code, value):
+ data = json.dumps(value).encode()
+ self.send_response(code)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(data)))
+ self.end_headers()
+ self.wfile.write(data)
+
+ def do_GET(self):
+ if owner.dead and self.path == "/health":
+ return self._json(503, {"error": "down"})
+ if self.path == "/health":
+ return self._json(200, {"service": "strata", "loaded": True})
+ if self.path == "/props":
+ return self._json(200, {"model_alias": "qwen3.8-flash-next", "chat_template": TEMPLATE,
+ "default_generation_settings": {"n_ctx": 131072, "params": {"temperature": 0.7}},
+ "total_slots": 1, "build_info": "Strata 0.1.45.3",
+ "model_path": str(owner.model_file), "is_sleeping": False})
+ if self.path == "/v1/status":
+ return self._json(200, {"service": "strata", "model": "qwen3.8-flash-next", "engine": "0.1.45.3",
+ "machine": {"gpu": {"name": owner.gpu_name}},
+ "activity": {"requests": owner.requests_before}})
+ if self.path == "/metrics":
+ return self._json(200, {"engine": {"version": "0.1.45.3"}, "hardware_static": {"gpu_name": owner.gpu_name}})
+ self._json(404, {})
+
+ def do_POST(self):
+ body = json.loads(self.rfile.read(int(self.headers.get("Content-Length", "0"))))
+ rid = re.match(r"\[case ([^\]]+)\]", body["messages"][0]["content"]).group(1)
+ with owner.lock:
+ owner.bodies.append(body)
+ owner.order.append(rid)
+ owner.arrivals.append(time.monotonic())
+ owner.active += 1
+ owner.max_active = max(owner.max_active, owner.active)
+ try:
+ self.respond(rid, body)
+ finally:
+ with owner.lock:
+ owner.active -= 1
+
+ def respond(self, rid, body):
+ spec = owner.specs[rid]
+ plan = owner.script.get(rid, {})
+ if plan.get("hold_for_arrivals"):
+ deadline = time.monotonic() + 3
+ while time.monotonic() < deadline and owner.active < plan["hold_for_arrivals"]:
+ time.sleep(0.01)
+ if plan.get("die"):
+ owner.dead = True
+ self.connection.close()
+ return
+ if plan.get("http"):
+ return self._json(plan["http"], {"error": {"message": "refused"}})
+ self.send_response(200)
+ self.send_header("Content-Type", "text/event-stream")
+ self.end_headers()
+
+ def send(event):
+ self.wfile.write(b"data: " + json.dumps(event).encode() + b"\n\n")
+ self.wfile.flush()
+ chunk = lambda delta, finish=None: {"choices": [{"index": 0, "delta": delta, "finish_reason": finish}]} # noqa: E731
+ send(chunk({"role": "assistant", "content": ""}))
+ self.wfile.write(b": keep-alive\n\n")
+ send(chunk({"content": ""}))
+ if plan.get("hang"):
+ time.sleep(plan["hang"])
+ return
+ time.sleep(plan.get("first_delay", 0))
+ send(chunk({"reasoning_content": "Thinking about it. "}))
+ expected = spec.get("expected") or {}
+ finish = plan.get("finish", "stop")
+ if "tool_calls" in plan:
+ calls = plan["tool_calls"]
+ finish = plan.get("finish", "tool_calls" if calls else "stop")
+ elif expected.get("kind") == "tool_call":
+ calls = [{"name": expected["name"], "arguments": json.dumps(expected["arguments"])}]
+ finish = plan.get("finish", "tool_calls")
+ else:
+ calls = []
+ if "content" in plan:
+ text = plan["content"]
+ elif expected.get("kind") == "text":
+ text = expected["value"]
+ elif expected.get("kind") == "json":
+ text = json.dumps(expected["value"], separators=(",", ":"))
+ elif expected.get("kind") == "tool_call":
+ text = ""
+ else:
+ text = "A hash table hashes the key."
+ if text:
+ send(chunk({"content": text}))
+ for index, call in enumerate(calls):
+ send(chunk({"tool_calls": [{"index": index, "id": f"call_{index}", "type": "function",
+ "function": {"name": call["name"], "arguments": ""}}]}))
+ send(chunk({"tool_calls": [{"index": index, "function": {"arguments": call["arguments"]}}]}))
+ final = chunk({}, finish)
+ prompt = spec["input_tokens_frozen"]
+ output = plan.get("completion_tokens", 50)
+ if plan.get("usage", True):
+ final["usage"] = {"prompt_tokens": prompt, "completion_tokens": output, "total_tokens": prompt + output,
+ "prompt_tokens_details": {"cached_tokens": plan.get("cached", 0)}}
+ if plan.get("timings", True):
+ cached = plan.get("cached", 0)
+ read = prompt - cached
+ final["timings"] = {"cache_n": cached, "prompt_n": read, "prompt_ms": float(read), "prompt_per_second": 1000.0,
+ "predicted_n": plan.get("predicted_n", output), "predicted_ms": 500.0,
+ "predicted_per_second": plan.get("rate", 100.0)}
+ send(final)
+ if not plan.get("no_done"):
+ self.wfile.write(b"data: [DONE]\n\n")
+ self.wfile.flush()
+
+ self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ self.httpd.daemon_threads = True
+ self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True)
+ self.thread.start()
+ self.url = f"http://127.0.0.1:{self.httpd.server_address[1]}"
+
+ def close(self):
+ self.httpd.shutdown()
+ self.httpd.server_close()
+
+
+class StandardBase(unittest.TestCase):
+ def setUp(self):
+ self.temp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temp.cleanup)
+ self.root = Path(self.temp.name)
+ self.model = self.root / "Qwen3.8-Flash-Next-IQ2_XS-00001-of-00001.gguf"
+ self.model.write_bytes(b"model bytes")
+ self.server = MockStrata(self.model)
+ self.addCleanup(self.server.close)
+ for name, value in (("nvidia_gpus", ([], None, None)), ("other_gpus", ([], None)), ("cpu_name", lambda: "Mock CPU"),
+ ("ram_bytes", lambda: 64 * 2 ** 30), ("storage_of", lambda path: [{"kind": "nvme", "model": "Mock SSD", "bus": "nvme"}]),
+ ("git_commit", lambda folder=None: COMMIT)):
+ patcher = mock.patch.object(bench_facts, name, **({"return_value": value} if isinstance(value, tuple) else {"side_effect": value}))
+ patcher.start()
+ self.addCleanup(patcher.stop)
+ self.target = bs.Target(self.server.url, None)
+
+ def run_standard(self, name="out", **kwargs):
+ kwargs.setdefault("hash_model", True)
+ doc, out = bs.run(self.target, self.root / name, **kwargs)
+ return doc, out
+
+ def record(self, doc, rid):
+ return next(r for r in doc["requests"] if r["id"] == rid)
+
+ def summary(self, doc, case, metric, phase="performance"):
+ return next(s for s in doc["summaries"] if s["phase"] == phase and s["case_id"] == case and s["metric"] == metric)
+
+
+class FixtureTests(unittest.TestCase):
+ def test_bundle_follows_the_protocol_tables(self):
+ corpus = bs.load_corpus()
+ specs = {s["id"]: s for s in corpus.specs}
+ self.assertEqual([s["id"] for s in corpus.specs][:10],
+ ["warmup", "performance-short-r1", "performance-prompt-4k-r1", "performance-prompt-16k-r1",
+ "performance-short-r2", "performance-prompt-4k-r2", "performance-prompt-16k-r2",
+ "performance-short-r3", "performance-prompt-4k-r3", "performance-prompt-16k-r3"])
+ self.assertEqual([s["id"] for s in corpus.specs if s["phase"] == "sanity"],
+ [f"sanity-{n}" for n in ("I1", "I2", "I3", "I4", "R1", "R2", "R3", "R4", "T1", "T2", "T3", "T4")])
+ self.assertEqual(len(corpus.specs), 34)
+ self.assertEqual((specs["warmup"]["seed"], specs["warmup"]["max_output_tokens"]), (1, 128))
+ for repetition in (1, 2, 3):
+ for case in ("short", "prompt-4k", "prompt-16k"):
+ spec = specs[f"performance-{case}-r{repetition}"]
+ self.assertEqual((spec["seed"], spec["max_output_tokens"], spec["concurrency"]), (repetition, 512, 1))
+ for spec in corpus.specs:
+ if spec["phase"] == "sanity":
+ self.assertEqual((spec["seed"], spec["max_output_tokens"]), (1, 2048))
+ if spec["phase"] == "concurrency":
+ self.assertEqual(spec["seed"], 4 * (spec["round"] - 1) + int(spec["id"][-1]))
+ self.assertEqual((spec["max_output_tokens"], spec["concurrency"]), (512, 4))
+ self.assertTrue(spec["messages"][0]["content"].startswith(f"[case {spec['id']}]\n"))
+ self.assertEqual(len(spec["messages"]), 1)
+ self.assertEqual(specs["performance-short-r1"]["messages"][0]["content"].split("\n", 1)[1],
+ "Explain how a hash table finds a value, including collisions and one small example.")
+
+ def test_records_and_retrieval_placement(self):
+ specs = corpus_specs()
+ body = specs["performance-prompt-4k-r1"]["messages"][0]["content"].split("\n", 1)[1]
+ lines = body.split("\n")
+ self.assertEqual(lines[0], "record000001: alpha beta gamma delta epsilon zeta eta theta.")
+ self.assertEqual(lines[-1], "Summarize the structure of these records and explain how to look up a record by its ID.")
+ count = len(lines) - 1
+ for rid, key, fraction in (("sanity-R1", "access_code = ORBIT-731", 0.10), ("sanity-R2", "storage_label = CEDAR-284", 0.50)):
+ rows = specs[rid]["messages"][0]["content"].split("\n", 1)[1].split("\n")
+ self.assertEqual(rows.index(key), int(fraction * count)) # inserted before record index floor(f * count)
+ self.assertEqual(sum("record" not in r for r in rows), 2) # the inserted line and the question
+ long_count = len(specs["performance-prompt-16k-r1"]["messages"][0]["content"].split("\n")) - 2
+ rows = specs["sanity-R4"]["messages"][0]["content"].split("\n", 1)[1].split("\n")
+ self.assertEqual(rows.index("north = N-124"), int(0.10 * long_count))
+ self.assertEqual(rows.index("south = S-875") - 1, int(0.90 * long_count)) # one earlier insert shifts it by one line
+ self.assertEqual(specs["sanity-R4"]["expected"]["value"], {"north": "N-124", "south": "S-875"})
+ for rid, name, props in (("sanity-T1", "lookup_record", {"id": "string"}),
+ ("sanity-T3", "convert_units", {"value": "number", "from_unit": "string", "to_unit": "string"}),
+ ("sanity-T4", "get_build_status", {"project": "string", "build": "integer"})):
+ function = specs[rid]["tools"][0]["function"]
+ self.assertEqual(function["name"], name)
+ self.assertEqual({k: v["type"] for k, v in function["parameters"]["properties"].items()}, props)
+ self.assertEqual(function["parameters"]["required"], list(props))
+ self.assertIs(function["parameters"]["additionalProperties"], False)
+ self.assertEqual(function["description"], "Synthetic benchmark lookup; no external action is performed.")
+ self.assertEqual(specs[rid]["tool_choice"], "auto")
+ self.assertIsNone(specs["sanity-I1"]["tools"])
+
+ def test_prompt_lengths_are_within_two_percent_of_their_targets(self):
+ for spec in bs.load_corpus().specs:
+ if spec["target_input_tokens"]:
+ self.assertLessEqual(abs(spec["input_tokens_frozen"] - spec["target_input_tokens"]) / spec["target_input_tokens"], 0.02)
+
+ def test_a_changed_or_converted_file_is_refused(self):
+ with tempfile.TemporaryDirectory() as temp:
+ copy_of = Path(temp) / "bundle"
+ shutil.copytree(bs.corpus_dir(), copy_of)
+ self.assertEqual(bs.load_corpus(copy_of).sha256, bs.load_corpus().sha256)
+ target = copy_of / "requests" / "warmup.json"
+ target.write_bytes(target.read_bytes().replace(b"\n", b"\r\n"))
+ with self.assertRaisesRegex(ProfileError, "fixture file changed: requests/warmup.json"):
+ bs.load_corpus(copy_of)
+ target.write_bytes(bs.load_corpus().manifest[:0] + (bs.corpus_dir() / "requests" / "warmup.json").read_bytes())
+ (copy_of / "requests" / "extra.json").write_text("{}", encoding="utf-8")
+ with self.assertRaisesRegex(ProfileError, "not in the manifest"):
+ bs.load_corpus(copy_of)
+
+ def test_gitattributes_keeps_fixture_bytes(self):
+ text = (Path(__file__).resolve().parents[2] / ".gitattributes").read_text(encoding="utf-8")
+ self.assertIn("bench/prompts/** -text", text)
+
+
+class DocsAndPackagingTests(unittest.TestCase):
+ ROOT = Path(__file__).resolve().parents[2]
+
+ def test_the_protocol_names_the_frozen_bundle(self):
+ text = (self.ROOT / "docs" / "BENCHMARK_PROTOCOL.md").read_text(encoding="utf-8")
+ self.assertIn(bs.load_corpus().sha256, text)
+ self.assertIn(bs.PROTOCOL_ID, text)
+ for heading in ("## Run it", "## Submit and review", "## Activation checklist", "### Where the numbers come from"):
+ self.assertIn(heading, text)
+ self.assertNotIn("do not implement this protocol", text)
+
+ def test_the_terminal_archive_carries_the_fixtures(self):
+ import sys
+ sys.path.insert(0, str(self.ROOT / "packaging"))
+ import package_terminal
+ self.assertIn("bench/prompts/standard-v1", package_terminal.PAYLOAD)
+
+ def test_the_launchers_point_at_the_standard_benchmark(self):
+ for name in ("BENCH.bat", "bench.sh"):
+ self.assertIn("strata.py bench run", (self.ROOT / name).read_text(encoding="utf-8"))
+
+
+class ScoringTests(unittest.TestCase):
+ def test_strict_json(self):
+ self.assertEqual(bs.strict_json(' {"a": 1}\n'), {"a": 1})
+ for bad in ('{"a":1,"a":2}', "```json\n{}\n```", '{"a":1} extra', "NaN", "Infinity", "", '{"a":1}{"b":2}'):
+ with self.assertRaises(ValueError, msg=bad):
+ bs.strict_json(bad)
+
+ def test_json_equality_checks_types(self):
+ self.assertTrue(bs.same_json({"a": [1, 2], "b": True}, {"b": True, "a": [1, 2]}))
+ self.assertFalse(bs.same_json({"ok": True}, {"ok": 1}))
+ self.assertFalse(bs.same_json(3, 3.0))
+ self.assertFalse(bs.same_json([1, 2], [2, 1]))
+ self.assertFalse(bs.same_json({"a": 1}, {"a": 1, "b": 2}))
+
+ def test_text_check_strips_only_ascii_whitespace(self):
+ spec = {"expected": {"kind": "text", "value": "READY"}, "category": "instruction"}
+ self.assertTrue(bs.evaluate(spec, "completed", " \nREADY\t", [])["passed"])
+ self.assertFalse(bs.evaluate(spec, "completed", "READY.", [])["passed"])
+ self.assertFalse(bs.evaluate(spec, "completed", "ready", [])["passed"])
+ self.assertFalse(bs.evaluate(spec, "truncated", "READY", [])["passed"])
+
+ def test_tool_call_needs_one_exact_structured_call(self):
+ spec = {"expected": {"kind": "tool_call", "name": "get_weather", "arguments": {"city": "Zagreb", "unit": "celsius"}},
+ "category": "tool_call"}
+ good = {"name": "get_weather", "arguments": '{"unit":"celsius","city":"Zagreb"}'}
+ self.assertTrue(bs.evaluate(spec, "completed", "", [good])["passed"])
+ self.assertFalse(bs.evaluate(spec, "completed", "get_weather(...)", [])["passed"])
+ self.assertFalse(bs.evaluate(spec, "completed", "", [good, good])["passed"])
+ self.assertFalse(bs.evaluate(spec, "completed", "", [{**good, "name": "other"}])["passed"])
+ self.assertFalse(bs.evaluate(spec, "completed", "", [{**good, "arguments": '{"city":"Zagreb"}'}])["passed"])
+ self.assertFalse(bs.evaluate(spec, "completed", "", [{**good, "arguments": '{"city":"Zagreb","unit":"celsius","x":1}'}])["passed"])
+ self.assertFalse(bs.evaluate(spec, "completed", "", [{**good, "arguments": '{"city":"a","city":"Zagreb","unit":"celsius"}'}])["passed"])
+
+
+class RunTests(StandardBase):
+ def test_a_full_run_follows_the_protocol(self):
+ doc, out = self.run_standard()
+ self.assertEqual(bs.validate(doc), [])
+ run = doc["run"]
+ self.assertEqual((doc["format"], doc["schema_version"], run["status"]), ("strata-standard-benchmark", 1, "complete"))
+ self.assertEqual(doc["protocol"]["id"], "strata-qwen38-thinking-v1-draft.1")
+ self.assertEqual(doc["protocol"]["corpus_sha256"], bs.load_corpus().sha256)
+ self.assertEqual(doc["protocol"]["git_commit"], COMMIT)
+ self.assertEqual(len(doc["requests"]), 22)
+ self.assertEqual(self.server.order, [r["id"] for r in doc["requests"]])
+ self.assertEqual(doc["request_counts"]["warmup"], {"planned": 1, "attempted": 1, "completed": 1, "truncated": 0, "failed": 0, "skipped": 0})
+ self.assertEqual(doc["request_counts"]["performance"]["completed"], 9)
+ self.assertEqual(doc["request_counts"]["concurrency"]["planned"], 0)
+ self.assertEqual(doc["quality"], {"passed": 12, "total": 12, "categories": {
+ "instruction": {"passed": 4, "total": 4}, "retrieval": {"passed": 4, "total": 4}, "tool_call": {"passed": 4, "total": 4}}})
+ self.assertEqual(len(doc["summaries"]), 15)
+ self.assertEqual(doc["concurrency_rounds"], [])
+ self.assertEqual(self.summary(doc, "prompt-4k", "prompt_tps")["median"], 1000.0)
+ self.assertEqual(self.summary(doc, "short", "decode_tps")["median"], 100.0)
+ for case in ("short", "prompt-4k", "prompt-16k"):
+ for metric in ("prompt_tps", "decode_tps", "ttft_s", "first_answer_s", "total_s"):
+ row = self.summary(doc, case, metric)
+ self.assertEqual(row["n"], 3)
+ self.assertLessEqual(row["min"], row["median"])
+ self.assertLessEqual(row["median"], row["max"])
+ # the warm-up and the sanity cases do not enter the speed summaries
+ self.assertNotIn("warmup", {i for s in doc["summaries"] for i in s["included_request_ids"]})
+ self.assertFalse(any(i.startswith("sanity") for s in doc["summaries"] for i in s["included_request_ids"]))
+ self.assertTrue(run["required_duration_s"] > 0)
+ self.assertIsNone(run["optional_duration_s"])
+ self.assertIn("optional_duration_s", "".join(doc["missing"]))
+
+ def test_every_request_carries_the_required_settings(self):
+ self.run_standard()
+ self.assertEqual(len(self.server.bodies), 22)
+ specs = corpus_specs()
+ for body, rid in zip(self.server.bodies, self.server.order):
+ spec = specs[rid]
+ self.assertIs(body["stream"], True)
+ self.assertEqual(body["stream_options"], {"include_usage": True})
+ self.assertEqual((body["temperature"], body["top_p"], body["top_k"], body["min_p"]), (1.0, 0.95, 20, 0.0))
+ self.assertEqual((body["presence_penalty"], body["repetition_penalty"]), (0.0, 1.0))
+ self.assertEqual(body["reasoning_effort"], "xhigh")
+ self.assertEqual(body["chat_template_kwargs"], {"enable_thinking": True, "preserve_thinking": True})
+ self.assertEqual(body["reasoning_budget_tokens"], 0)
+ self.assertEqual((body["seed"], body["max_tokens"]), (spec["seed"], spec["max_output_tokens"]))
+ self.assertNotIn("stop", body)
+ self.assertEqual(body["messages"], spec["messages"])
+ self.assertEqual(body.get("tools"), spec["tools"])
+ if spec["tools"]:
+ self.assertEqual(body["tool_choice"], "auto")
+
+ def test_the_report_files_and_their_hashes(self):
+ doc, out = self.run_standard()
+ for name in ("results.json", "README.md", "outputs.jsonl", "config.json", "provenance/chat_template.jinja",
+ "provenance/corpus-manifest.sha256", "provenance/corpus-build.json"):
+ self.assertTrue((out / name).is_file(), name)
+ saved = json.loads((out / "results.json").read_text(encoding="utf-8"))
+ self.assertEqual(saved, doc)
+ for artifact in doc["artifacts"]:
+ self.assertEqual(bench_facts.sha256_file(out / artifact["path"]), artifact["sha256"])
+ self.assertNotIn("..", artifact["path"])
+ self.assertEqual({a["kind"] for a in doc["artifacts"]}, {"outputs", "config", "provenance"})
+ lines = (out / "outputs.jsonl").read_text(encoding="utf-8").splitlines()
+ self.assertEqual([json.loads(line)["request_id"] for line in lines], [r["id"] for r in doc["requests"]])
+ first = json.loads(lines[0])
+ self.assertEqual(first["dispatch_offset_s"], 0.0)
+ self.assertEqual(first["first_delta"]["kind"], "reasoning")
+ self.assertEqual(doc["configuration"]["template"], {"path": "provenance/chat_template.jinja",
+ "sha256": bench_facts.sha256_text(TEMPLATE)})
+ self.assertEqual(doc["model"]["files"], [{"path": self.model.name, "sha256": bench_facts.sha256_file(self.model)}])
+ self.assertEqual(doc["model"]["quantization"], "IQ2_XS")
+ readme = (out / "README.md").read_text(encoding="utf-8")
+ self.assertIn("Status: complete", readme)
+ self.assertIn("12 of 12", readme)
+ self.assertNotIn(str(self.root), readme)
+
+ def test_every_null_has_a_reason_and_the_request_shape_is_complete(self):
+ doc, _ = self.run_standard()
+ self.assertEqual(set(doc["missing"]), set(bs.scan_nulls(doc)))
+ self.assertIn("/hardware/gpus/0/power_limit_w", doc["missing"])
+ self.assertIn("/protocol/model_card_revision", doc["missing"])
+ self.assertIn("/requests/0/round", doc["missing"])
+ record = self.record(doc, "performance-short-r1")
+ self.assertEqual(set(record), {"id", "phase", "case_id", "repetition", "seed", "max_output_tokens", "concurrency", "round",
+ "status", "finish_reason", "error", "tokens", "timings", "metrics", "sources",
+ "first_delta_kind", "token_accounting_note", "evaluation"})
+ self.assertEqual(record["tokens"], {"input_tokens": record["tokens"]["input_tokens"], "cached_tokens": 0,
+ "read_tokens": record["tokens"]["input_tokens"], "output_tokens": 50, "decode_tokens": 50})
+ self.assertEqual(record["timings"]["decode_s"], 0.5)
+ self.assertEqual(record["first_delta_kind"], "reasoning")
+ self.assertIsNone(record["token_accounting_note"])
+
+ def test_the_run_names_its_default_folder_after_the_gpu(self):
+ self.server.gpu_name = "NVIDIA GeForce RTX 3060"
+ with mock.patch.object(bench_facts, "nvidia_gpus", return_value=([{
+ "device_index": 0, "name": "NVIDIA GeForce RTX 3060", "memory_bytes": 12 * 2 ** 30, "memory_kind": "dedicated",
+ "power_limit_w": 170.0, "driver": "560.1"}], "560.1", "12.6")):
+ doc, out = bs.run(self.target, None, out_root=self.root / "results", hash_model=True)
+ self.assertRegex(out.name, r"^\d{4}-\d\d-\d\d-standard-rtx-3060-[0-9a-f]{8}$")
+ self.assertEqual(doc["hardware"]["gpus"][0]["power_limit_w"], 170.0)
+ self.assertEqual(doc["software"]["runtime"], [{"name": "cuda", "version": "12.6"}])
+
+ def test_early_eos_and_a_reached_cap_stay_in_the_speed_summaries(self):
+ self.server.script["performance-short-r1"] = {"completion_tokens": 5, "predicted_n": 5, "rate": 10.0}
+ self.server.script["performance-short-r2"] = {"finish": "length", "completion_tokens": 512, "predicted_n": 512, "rate": 1024.0}
+ doc, _ = self.run_standard()
+ self.assertEqual(self.record(doc, "performance-short-r2")["status"], "truncated")
+ self.assertEqual(self.record(doc, "performance-short-r2")["finish_reason"], "length")
+ row = self.summary(doc, "short", "decode_tps")
+ self.assertEqual((row["n"], row["min"], row["max"]), (3, 10.0, 1024.0))
+ self.assertEqual(doc["request_counts"]["performance"]["truncated"], 1)
+
+ def test_a_failed_request_is_excluded_and_listed_but_the_run_goes_on(self):
+ self.server.script["performance-prompt-4k-r2"] = {"http": 500}
+ doc, out = self.run_standard()
+ self.assertEqual(doc["run"]["status"], "complete")
+ record = self.record(doc, "performance-prompt-4k-r2")
+ self.assertEqual((record["status"], record["error"]["kind"]), ("failed", "http_500"))
+ row = self.summary(doc, "prompt-4k", "decode_tps")
+ self.assertEqual(row["n"], 2)
+ self.assertEqual(row["excluded"], [{"request_id": "performance-prompt-4k-r2", "reason": "failed: http_500"}])
+ self.assertEqual(doc["request_counts"]["performance"]["failed"], 1)
+ self.assertEqual(len(self.server.order), 22)
+ self.assertIn("performance-prompt-4k-r2 failed", (out / "README.md").read_text(encoding="utf-8"))
+ self.assertEqual(bs.validate(doc), [])
+
+ def test_a_failed_warmup_skips_everything_else(self):
+ self.server.script["warmup"] = {"http": 500}
+ doc, _ = self.run_standard()
+ self.assertEqual(doc["run"]["status"], "partial")
+ self.assertEqual(len(self.server.order), 1)
+ self.assertEqual(len(doc["requests"]), 22)
+ self.assertEqual(doc["request_counts"]["performance"]["skipped"], 9)
+ self.assertEqual(doc["request_counts"]["sanity"], {"planned": 12, "attempted": 0, "completed": 0, "truncated": 0, "failed": 0, "skipped": 12})
+ self.assertEqual(doc["quality"]["passed"], 0)
+ skipped = self.record(doc, "sanity-T4")
+ self.assertEqual((skipped["status"], skipped["error"]["kind"]), ("skipped", "skipped"))
+ self.assertFalse(skipped["evaluation"]["passed"])
+ self.assertTrue(all(s["n"] == 0 and s["median"] is None for s in doc["summaries"]))
+ self.assertEqual(bs.validate(doc), [])
+
+ def test_a_dead_server_marks_the_rest_skipped(self):
+ self.server.script["performance-prompt-4k-r1"] = {"die": True}
+ doc, _ = self.run_standard()
+ self.assertEqual(doc["run"]["status"], "partial")
+ self.assertEqual(self.record(doc, "performance-prompt-4k-r1")["status"], "failed")
+ self.assertEqual(self.record(doc, "performance-prompt-16k-r1")["status"], "skipped")
+ self.assertIn("server stopped answering", self.record(doc, "sanity-I1")["error"]["message"])
+ self.assertEqual(len(doc["requests"]), 22)
+ self.assertEqual(bs.validate(doc), [])
+
+ def test_a_request_that_never_finishes_times_out(self):
+ self.server.script["performance-short-r1"] = {"hang": 3}
+ started = time.monotonic()
+ doc, _ = self.run_standard(timeout_s=0.6)
+ record = self.record(doc, "performance-short-r1")
+ self.assertEqual((record["status"], record["error"]["kind"]), ("failed", "timeout"))
+ self.assertLess(record["timings"]["total_s"], 2.0)
+ self.assertGreaterEqual(record["timings"]["total_s"], 0.5)
+ self.assertEqual(doc["run"]["status"], "complete")
+ self.assertLess(time.monotonic() - started, 30)
+
+ def test_a_stream_without_done_or_finish_reason_is_a_failure(self):
+ self.server.script["performance-short-r1"] = {"no_done": True}
+ self.server.script["performance-short-r2"] = {"finish": None}
+ doc, _ = self.run_standard()
+ self.assertEqual(self.record(doc, "performance-short-r1")["error"]["kind"], "malformed_stream")
+ self.assertEqual(self.record(doc, "performance-short-r2")["error"]["kind"], "malformed_stream")
+ self.assertEqual(self.summary(doc, "short", "decode_tps")["n"], 1)
+
+ def test_empty_deltas_and_keepalives_do_not_count_as_the_first_token(self):
+ self.server.script["performance-short-r1"] = {"first_delay": 0.4}
+ doc, out = self.run_standard()
+ record = self.record(doc, "performance-short-r1")
+ self.assertGreaterEqual(record["timings"]["ttft_s"], 0.35)
+ self.assertEqual(record["first_delta_kind"], "reasoning")
+ self.assertGreaterEqual(record["timings"]["first_answer_s"], record["timings"]["ttft_s"])
+ tool = self.record(doc, "sanity-T1")
+ self.assertIsNotNone(tool["timings"]["first_answer_s"]) # a tool-call delta is an answer
+ self.assertIsNone(self.record(doc, "sanity-T1")["error"])
+
+ def test_missing_usage_and_timings_leave_the_rates_null_with_reasons(self):
+ self.server.script["performance-short-r1"] = {"usage": False, "timings": False}
+ doc, _ = self.run_standard()
+ record = self.record(doc, "performance-short-r1")
+ self.assertEqual(record["status"], "completed")
+ self.assertIsNone(record["metrics"]["prompt_tps"])
+ self.assertIsNone(record["metrics"]["decode_tps"])
+ self.assertIsNone(record["tokens"]["output_tokens"])
+ self.assertIsNotNone(record["timings"]["ttft_s"])
+ index = doc["requests"].index(record)
+ self.assertIn(f"/requests/{index}/metrics/decode_tps", doc["missing"])
+ row = self.summary(doc, "short", "decode_tps")
+ self.assertEqual(row["n"], 2)
+ self.assertIn("decode_tps is not available", row["excluded"][0]["reason"])
+ self.assertEqual(bs.validate(doc), [])
+
+ def test_a_decode_count_that_disagrees_with_the_rate_is_explained(self):
+ # the engine's rate and time imply 50 tokens (100/s over 0.5 s) while predicted_n says 60 and the API says 61
+ self.server.script["performance-short-r1"] = {"predicted_n": 60, "completion_tokens": 61}
+ doc, _ = self.run_standard()
+ record = self.record(doc, "performance-short-r1")
+ self.assertEqual(record["tokens"]["decode_tokens"], 50)
+ self.assertEqual(record["tokens"]["output_tokens"], 61)
+ self.assertIn("implied", record["token_accounting_note"])
+ self.assertEqual(record["metrics"]["decode_tps"], 100.0)
+ self.assertIn("derived", record["sources"]["decode_tokens"])
+
+ def test_wrong_answers_are_kept_and_scored_without_changing_the_run_status(self):
+ specs = corpus_specs()
+ script = self.server.script
+ script["sanity-I1"] = {"content": '```json\n{"ok":true,"count":3}\n```'}
+ script["sanity-I2"] = {"content": "Here you go: [-4,1,2,9]"}
+ script["sanity-I3"] = {"content": '{"project":"Orion","project":"Orion","build":17}'}
+ script["sanity-R1"] = {"finish": "length", "content": "ORB", "completion_tokens": 2048, "predicted_n": 2048}
+ script["sanity-R2"] = {"content": "cedar-284"}
+ script["sanity-T1"] = {"tool_calls": [], "content": "lookup_record({\"id\":\"A-17\"})"}
+ script["sanity-T2"] = {"tool_calls": [{"name": "get_weather", "arguments": '{"city":"Zagreb","unit":"celsius"}'}] * 2}
+ script["sanity-T3"] = {"tool_calls": [{"name": "convert_units", "arguments": '{"value":3.0,"from_unit":"km","to_unit":"m"}'}]}
+ doc, _ = self.run_standard()
+ verdicts = {r["case_id"]: r["evaluation"]["passed"] for r in doc["requests"] if r["phase"] == "sanity"}
+ self.assertEqual(verdicts, {"I1": False, "I2": False, "I3": False, "I4": True, "R1": False, "R2": False, "R3": True, "R4": True,
+ "T1": False, "T2": False, "T3": False, "T4": True})
+ self.assertEqual(doc["quality"]["passed"], 4)
+ self.assertEqual(doc["quality"]["categories"]["instruction"]["passed"], 1)
+ self.assertEqual(self.record(doc, "sanity-R1")["status"], "truncated")
+ self.assertIn("output cap", self.record(doc, "sanity-R1")["evaluation"]["reason"])
+ self.assertEqual(doc["run"]["status"], "complete")
+ self.assertEqual(specs["sanity-I1"]["expected"]["value"], {"ok": True, "count": 3})
+ text = bs.format_plain(doc)
+ self.assertIn("Sanity checks: 4 of 12 passed", text)
+ self.assertIn("I1:", text)
+
+ def test_unexpected_reuse_of_the_user_payload_is_excluded_and_nonconforming(self):
+ spec = corpus_specs()["performance-prompt-4k-r2"]
+ self.server.script["performance-prompt-4k-r2"] = {"cached": spec["input_tokens_frozen"] - 10}
+ # the shared template prefix alone is allowed
+ self.server.script["performance-prompt-4k-r1"] = {"cached": 40}
+ doc, _ = self.run_standard()
+ self.assertEqual(doc["run"]["status"], "nonconforming")
+ row = self.summary(doc, "prompt-4k", "prompt_tps")
+ self.assertEqual(row["n"], 2)
+ self.assertIn("unexpected cache reuse", row["excluded"][0]["reason"])
+ self.assertTrue(any("unexpected reuse of the user payload in performance-prompt-4k-r2" in r
+ for r in doc["extensions"]["status_reasons"]))
+ self.assertEqual(self.record(doc, "performance-prompt-4k-r1")["tokens"]["cached_tokens"], 40)
+ self.assertEqual(bs.validate(doc), [])
+
+ def test_a_prompt_length_off_its_target_is_nonconforming(self):
+ original = bs.load_corpus()
+ specs = copy.deepcopy(original.specs)
+ for spec in specs:
+ if spec["id"] == "performance-prompt-4k-r1":
+ spec["target_input_tokens"] = 3000
+ patched = bs.Corpus(original.sha256, specs, original.build, original.manifest, original.directory)
+ doc, _ = self.run_standard(corpus=patched)
+ self.assertEqual(doc["run"]["status"], "nonconforming")
+ self.assertTrue(any("outside 2%" in r for r in doc["extensions"]["status_reasons"]))
+
+ def test_a_refused_setting_is_nonconforming(self):
+ self.server.script["warmup"] = {"http": 400}
+ doc, _ = self.run_standard()
+ self.assertEqual(doc["run"]["status"], "nonconforming")
+ self.assertTrue(any("refused the protocol's required settings" in r for r in doc["extensions"]["status_reasons"]))
+ self.assertEqual(self.record(doc, "sanity-I1")["status"], "skipped")
+
+ def test_missing_identity_makes_the_run_nonconforming(self):
+ with mock.patch.object(bench_facts, "git_commit", return_value=None):
+ doc, _ = self.run_standard("nocommit")
+ self.assertEqual(doc["run"]["status"], "nonconforming")
+ self.assertIn("/software/strata_commit", doc["missing"])
+ self.assertIsNone(doc["protocol"]["git_commit"])
+ doc2, _ = self.run_standard("nohash", hash_model=False)
+ self.assertEqual(doc2["run"]["status"], "nonconforming")
+ self.assertIsNone(doc2["model"]["files"])
+ self.assertIn("/model/files", doc2["missing"])
+
+ def test_a_remote_server_is_nonconforming(self):
+ with mock.patch.object(bs.Target, "loopback", new_callable=mock.PropertyMock, return_value=False):
+ doc, _ = self.run_standard()
+ self.assertEqual(doc["run"]["status"], "nonconforming")
+ self.assertTrue(any("not on loopback" in r for r in doc["extensions"]["status_reasons"]))
+
+ def test_cancelling_keeps_the_finished_requests_and_skips_the_rest(self):
+ real = bs.stream_chat
+ calls = []
+
+ def interrupted(target, spec, model, **kwargs):
+ calls.append(spec["id"])
+ if len(calls) == 6:
+ class Boom:
+ sock = None
+
+ def request(self, *args, **kw):
+ raise KeyboardInterrupt
+
+ def close(self):
+ pass
+ with mock.patch.object(bs.Target, "connection", return_value=Boom()):
+ return real(target, spec, model, **kwargs)
+ return real(target, spec, model, **kwargs)
+ with mock.patch.object(bs, "stream_chat", interrupted):
+ doc, _ = self.run_standard()
+ self.assertEqual(doc["run"]["status"], "partial")
+ self.assertEqual(self.record(doc, "performance-prompt-4k-r2")["error"]["kind"], "cancelled")
+ self.assertEqual(self.record(doc, "performance-prompt-4k-r2")["status"], "failed")
+ self.assertEqual(self.record(doc, "performance-prompt-16k-r3")["status"], "skipped")
+ self.assertEqual(self.record(doc, "sanity-T4")["error"]["message"], "the operator cancelled the run")
+ self.assertEqual(self.summary(doc, "short", "decode_tps")["n"], 3 - 1)
+ self.assertEqual(bs.validate(doc), [])
+
+ def test_the_optional_concurrency_stage_releases_four_together(self):
+ for r in (1, 2, 3):
+ for c in (1, 2, 3, 4):
+ self.server.script[f"concurrency-r{r}-c{c}"] = {"hold_for_arrivals": 4}
+ doc, _ = self.run_standard(optional_concurrency=True)
+ self.assertEqual(doc["run"]["status"], "complete")
+ self.assertEqual(len(doc["requests"]), 34)
+ self.assertEqual(self.server.max_active, 4)
+ self.assertEqual(doc["request_counts"]["concurrency"]["planned"], 12)
+ self.assertEqual(len(doc["concurrency_rounds"]), 3)
+ for round_ in doc["concurrency_rounds"]:
+ self.assertEqual(round_["output_tokens"], 200)
+ self.assertEqual(round_["failed"], 0)
+ self.assertAlmostEqual(round_["aggregate_tps"], 200 / round_["wall_s"], places=3)
+ self.assertEqual(len(round_["request_ids"]), 4)
+ rows = [s for s in doc["summaries"] if s["phase"] == "concurrency"]
+ self.assertEqual(len(rows), 5)
+ self.assertEqual({(s["case_id"], s["concurrency"]) for s in rows}, {("short-burst", 4)})
+ self.assertEqual(rows[0]["n"], 12)
+ self.assertIsNotNone(doc["run"]["optional_duration_s"])
+ for round_no in (1, 2, 3): # a round finishes before the next one is released
+
+ self.assertEqual({n for n in self.server.order[22 + 4 * (round_no - 1): 22 + 4 * round_no]},
+ {f"concurrency-r{round_no}-c{c}" for c in (1, 2, 3, 4)})
+ self.assertEqual(bs.validate(doc), [])
+
+ def test_a_failed_concurrent_request_keeps_its_time_in_the_round(self):
+ self.server.script["concurrency-r1-c2"] = {"http": 500}
+ doc, _ = self.run_standard(optional_concurrency=True)
+ first = doc["concurrency_rounds"][0]
+ self.assertEqual((first["failed"], first["output_tokens"]), (1, 150))
+ self.assertEqual(doc["concurrency_rounds"][1]["failed"], 0)
+ self.assertEqual(doc["request_counts"]["concurrency"]["failed"], 1)
+
+ def test_a_round_total_is_null_when_a_counted_request_has_no_token_count(self):
+ self.server.script["concurrency-r2-c3"] = {"usage": False}
+ doc, _ = self.run_standard(optional_concurrency=True)
+ second = doc["concurrency_rounds"][1]
+ self.assertIsNone(second["output_tokens"])
+ self.assertIsNone(second["aggregate_tps"])
+ self.assertIsNotNone(doc["concurrency_rounds"][0]["aggregate_tps"])
+ self.assertEqual(bs.validate(doc), [])
+
+ def test_the_plain_summary(self):
+ doc, _ = self.run_standard()
+ text = bs.format_plain(doc)
+ self.assertIn("Status: complete. 21 of 21 required requests finished, 0 failed, 0 skipped.", text)
+ self.assertIn("prompt-16k", text)
+ self.assertIn("Sanity checks: 12 of 12 passed", text)
+ self.assertIn("Prompt tok/s", text)
+ self.assertIn("1000.0 (1000.0-1000.0) n=3", text)
+
+ def test_the_server_is_not_asked_for_anything_outside_the_protocol(self):
+ self.run_standard()
+ self.assertTrue(all(body["messages"] == corpus_specs()[rid]["messages"] for body, rid in zip(self.server.bodies, self.server.order)))
+
+
+class ValidatorTests(StandardBase):
+ def test_the_validator_refuses_broken_results(self):
+ doc, _ = self.run_standard()
+ self.assertEqual(bs.validate(doc), [])
+ broken = copy.deepcopy(doc)
+ broken["schema_version"] = 2
+ self.assertTrue(bs.validate(broken))
+ broken = copy.deepcopy(doc)
+ broken["request_counts"]["performance"]["completed"] = 8
+ self.assertTrue(any("identity" in p for p in bs.validate(broken)))
+ broken = copy.deepcopy(doc)
+ broken["missing"].pop(next(iter(broken["missing"])))
+ self.assertTrue(any("missing must list" in p for p in bs.validate(broken)))
+ broken = copy.deepcopy(doc)
+ broken["summaries"][0]["median"] = 12345.0
+ self.assertTrue(any("rebuilt" in p for p in bs.validate(broken)))
+ broken = copy.deepcopy(doc)
+ broken["requests"][1]["timings"]["ttft_s"] = float("nan")
+ self.assertTrue(bs.validate(broken))
+ broken = copy.deepcopy(doc)
+ broken["surprise"] = 1
+ self.assertTrue(any("extensions" in p for p in bs.validate(broken)))
+ broken = copy.deepcopy(doc)
+ broken["artifacts"][0]["path"] = "../escape"
+ self.assertTrue(any("artifact" in p for p in bs.validate(broken)))
+
+ def test_load_reads_a_saved_result_and_refuses_other_files(self):
+ doc, out = self.run_standard()
+ self.assertEqual(bs.load(out)["run"]["id"], doc["run"]["id"])
+ self.assertIsNone(bs.load_if_standard(self.root / "nothing"))
+ other = self.root / "other.json"
+ other.write_text(json.dumps({"kind": "strata.serving.benchmark"}), encoding="utf-8")
+ self.assertIsNone(bs.load_if_standard(other))
+ bad = json.loads((out / "results.json").read_text(encoding="utf-8"))
+ bad["schema_version"] = 9
+ (self.root / "bad.json").write_text(json.dumps(bad), encoding="utf-8")
+ with self.assertRaises(ProfileError):
+ bs.load_if_standard(self.root / "bad.json")
+
+
+class CommandLineTests(StandardBase):
+ def invoke(self, *args, env=None):
+ output, errors = io.StringIO(), io.StringIO()
+ with mock.patch.dict(os.environ, env or {}), contextlib.redirect_stdout(output), contextlib.redirect_stderr(errors):
+ code = main(["bench", "run", *args])
+ return code, output.getvalue(), errors.getvalue()
+
+ def test_the_standard_suite_is_the_default_and_prints_a_plain_summary(self):
+ out = self.root / "cli"
+ code, text, errors = self.invoke("--url", self.server.url, "--out", str(out))
+ self.assertEqual(code, 0, errors)
+ self.assertIn("Strata standard benchmark (strata-qwen38-thinking-v1-draft.1)", text)
+ self.assertIn("Sanity checks: 12 of 12 passed", text)
+ self.assertIn("Report:", text)
+ self.assertEqual(bs.load(out)["run"]["status"], "complete")
+ self.assertIn("[1/22] warmup", errors)
+
+ def test_json_output(self):
+ code, text, _ = self.invoke("--url", self.server.url, "--out", str(self.root / "j"), "--output", "json")
+ self.assertEqual(code, 0)
+ value = json.loads(text)
+ self.assertTrue(value["ok"])
+ self.assertEqual(value["quality"]["passed"], 12)
+ self.assertEqual(len(value["summaries"]), 15)
+
+ def test_a_partial_run_exits_with_6(self):
+ self.server.script["warmup"] = {"http": 500}
+ code, text, _ = self.invoke("--url", self.server.url, "--out", str(self.root / "p"))
+ self.assertEqual(code, 6)
+ self.assertIn("Status: partial", text)
+ self.assertIn("Why this is not a complete standard result", text)
+
+ def test_serving_options_do_not_apply(self):
+ for option in (("--concurrency", "2"), ("--repeat", "2"), ("--request-timeout", "30"), ("--condition", "baseline")):
+ code, text, _ = self.invoke("--url", self.server.url, "--out", str(self.root / "x"), "--output", "json", *option)
+ self.assertEqual(code, 2, option)
+ self.assertIn("do not apply to the standard suite", text)
+ self.assertFalse((self.root / "x").exists())
+
+ def test_the_out_folder_must_be_new_and_the_default_goes_under_bench_results(self):
+ (self.root / "taken").mkdir()
+ code, text, _ = self.invoke("--url", self.server.url, "--out", str(self.root / "taken"), "--output", "json")
+ self.assertEqual(code, 2)
+ previous = Path.cwd()
+ os.chdir(self.root)
+ try:
+ code, text, _ = self.invoke("--url", self.server.url)
+ finally:
+ os.chdir(previous)
+ self.assertEqual(code, 0)
+ self.assertTrue(any((self.root / "bench-results").glob("*-standard-*/results.json")))
+
+ def test_the_serving_suites_still_need_out_and_keep_their_defaults(self):
+ code, text, _ = self.invoke("--url", self.server.url, "--suite", "serving-smoke", "--output", "json")
+ self.assertEqual(code, 2)
+ self.assertIn("--out is required for the serving suites", text)
+
+ def test_the_options_reach_the_report(self):
+ out = self.root / "opts"
+ code, _, errors = self.invoke("--url", self.server.url, "--out", str(out), "--track", "default", "--track-notes",
+ "0.2.0 release, setup choices", "--model-revision", "r1", "--source-revision", "r2",
+ "--tokenizer-revision", "r3", "--custom-pack", "none", "--background", "browser open",
+ "--build-options", "cuda,sm86", "--optional-concurrency")
+ self.assertEqual(code, 0, errors)
+ doc = bs.load(out)
+ self.assertEqual((doc["configuration"]["track"], doc["configuration"]["track_notes"]), ("default", "0.2.0 release, setup choices"))
+ self.assertEqual((doc["model"]["revision"], doc["model"]["source_revision"], doc["model"]["tokenizer_revision"]), ("r1", "r2", "r3"))
+ self.assertEqual(doc["hardware"]["background_workloads"], ["browser open"])
+ self.assertEqual(doc["software"]["build_options"], ["cuda", "sm86"])
+ self.assertTrue(doc["run"]["optional_concurrency"])
+ self.assertIn("--track default", doc["run"]["command"])
+ self.assertTrue(doc["run"]["command"].startswith("strata bench run"))
+
+ def test_report_reads_a_standard_result_in_three_forms(self):
+ out = self.root / "rep"
+ self.assertEqual(self.invoke("--url", self.server.url, "--out", str(out))[0], 0)
+ output = io.StringIO()
+ with contextlib.redirect_stdout(output):
+ self.assertEqual(main(["bench", "report", str(out)]), 0)
+ self.assertIn("Sanity checks: 12 of 12 passed", output.getvalue())
+ output = io.StringIO()
+ with contextlib.redirect_stdout(output):
+ self.assertEqual(main(["bench", "report", str(out), "--output", "markdown"]), 0)
+ self.assertIn("# Strata standard benchmark", output.getvalue())
+ output = io.StringIO()
+ with contextlib.redirect_stdout(output):
+ self.assertEqual(main(["bench", "report", str(out), "--output", "json"]), 0)
+ self.assertEqual(json.loads(output.getvalue())["summary"]["status"], "complete")
+
+ def test_a_profile_that_is_not_running_is_started_and_stopped(self):
+ out = self.root / "prof"
+ cfg = {"api_key": "", "model_name": "mock"}
+ plan = {"effective": {"host": "127.0.0.1", "port": int(self.server.url.rsplit(":", 1)[1])}, "engine_argv": ["--kv", "int8", "--expert-cache", "1500"]}
+ health = {"calls": 0}
+ real_get = bs.Target.get
+
+ def get(target, path, timeout=10):
+ if path == "/health" and health["calls"] < 1:
+ health["calls"] += 1
+ return None # nothing answers before the engine is started
+ return real_get(target, path, timeout)
+ started = []
+ with mock.patch("strata_cli.cli._source", return_value=(Path("p.json"), None, None, cfg)), \
+ mock.patch("strata_cli.cli._launch_plan", return_value=(plan, [])), \
+ mock.patch.object(bs.Target, "get", get), \
+ mock.patch.object(bs, "start_engine", side_effect=lambda ref, target, log, say, wait: started.append(ref) or "proc") as start, \
+ mock.patch.object(bs, "stop_engine") as stop:
+ code, text, errors = self.invoke("--profile", "p.json", "--out", str(out))
+ self.assertEqual(code, 0, errors)
+ self.assertEqual(started, ["p.json"])
+ stop.assert_called_once_with("proc")
+ doc = bs.load(out)
+ self.assertTrue(doc["extensions"]["started_engine"])
+ self.assertEqual(doc["configuration"]["kv_cache"], {"--kv": "int8"})
+ self.assertEqual(doc["configuration"]["expert_cache"], {"--expert-cache": "1500"})
+ self.assertNotIn("/configuration/kv_cache", doc["missing"])
+
+ def test_a_running_profile_is_used_and_left_running(self):
+ plan = {"effective": {"host": "127.0.0.1", "port": int(self.server.url.rsplit(":", 1)[1])}, "engine_argv": []}
+ with mock.patch("strata_cli.cli._source", return_value=(Path("p.json"), None, None, {})), \
+ mock.patch("strata_cli.cli._launch_plan", return_value=(plan, [])), \
+ mock.patch.object(bs, "start_engine") as start, mock.patch.object(bs, "stop_engine") as stop:
+ code, text, errors = self.invoke("--profile", "p.json", "--out", str(self.root / "run"))
+ self.assertEqual(code, 0, errors)
+ start.assert_not_called()
+ stop.assert_called_once_with(None)
+ self.assertIn("already running", errors)
+ self.assertFalse(bs.load(self.root / "run")["extensions"]["started_engine"])
+
+ def test_no_start_refuses_to_launch(self):
+ self.server.close()
+ plan = {"effective": {"host": "127.0.0.1", "port": int(self.server.url.rsplit(":", 1)[1])}, "engine_argv": []}
+ with mock.patch("strata_cli.cli._source", return_value=(Path("p.json"), None, None, {})), \
+ mock.patch("strata_cli.cli._launch_plan", return_value=(plan, [])), \
+ mock.patch.object(bs, "start_engine") as start:
+ code, text, _ = self.invoke("--profile", "p.json", "--out", str(self.root / "ns"), "--no-start", "--output", "json")
+ self.assertEqual(code, 5)
+ start.assert_not_called()
+
+
+class EngineProcessTests(unittest.TestCase):
+ def test_start_engine_runs_serve_with_the_profile_and_stop_ends_only_that_process(self):
+ class Fake:
+ pid = 4242
+
+ def poll(self):
+ return None
+
+ def wait(self, timeout=None):
+ return 0
+ seen = {}
+
+ def popen(command, **kwargs):
+ seen["command"] = command
+ return Fake()
+ target = mock.Mock()
+ target.get.return_value = {"service": "strata", "loaded": True}
+ with tempfile.TemporaryDirectory() as temp, mock.patch("subprocess.Popen", popen):
+ proc = bs.start_engine("run.json", target, Path(temp) / "engine.log", lambda m: None, 5)
+ self.assertEqual(seen["command"][1].replace("\\", "/").rsplit("/", 1)[-1], "strata.py")
+ self.assertEqual(seen["command"][2:5], ["serve", "--profile", "run.json"])
+ with mock.patch("subprocess.run") as run, mock.patch("os.killpg", create=True) as killpg:
+ bs.stop_engine(proc)
+ if os.name == "nt":
+ self.assertEqual(run.call_args[0][0][:4], ["taskkill", "/PID", "4242", "/T"])
+ else:
+ killpg.assert_called()
+
+ def test_an_engine_that_exits_before_it_is_ready_is_an_error(self):
+ class Dead:
+ pid = 1
+
+ def poll(self):
+ return 3
+
+ def wait(self, timeout=None):
+ return 3
+ target = mock.Mock()
+ target.get.return_value = None
+ with tempfile.TemporaryDirectory() as temp, mock.patch("subprocess.Popen", lambda *a, **k: Dead()):
+ with self.assertRaisesRegex(ProfileError, "did not become ready"):
+ bs.start_engine("run.json", target, Path(temp) / "engine.log", lambda m: None, 5)
+
+
+class RealServerTests(unittest.TestCase):
+ """The same request bodies against the real API server with its mock engine: every protocol setting is accepted."""
+
+ def test_the_real_server_accepts_the_protocol_requests(self):
+ from serve.frontend import ChatTemplate
+ from serve.server import ByteTokenizer, MockEngine, Service, serve
+ with tempfile.TemporaryDirectory() as temp:
+ tok = ByteTokenizer()
+ template = ChatTemplate(Path(__file__).resolve().parents[2] / "serve/chat_template.jinja")
+ service = Service(MockEngine(tok, "A hash table hashes the key.", max_context=400000), tok, template)
+ service.api_key = "inference-test-secret"
+ httpd = serve(service, port=0)
+ self.addCleanup(httpd.server_close)
+ self.addCleanup(httpd.shutdown)
+ target = bs.Target(f"http://127.0.0.1:{httpd.server_address[1]}", "inference-test-secret")
+ patches = [mock.patch.object(bench_facts, "git_commit", return_value=COMMIT)]
+ for patch in patches:
+ patch.start()
+ self.addCleanup(patch.stop)
+ doc, out = bs.run(target, Path(temp) / "real", hash_model=True)
+ self.assertEqual(bs.validate(doc), [])
+ failed = [(r["id"], r["error"]) for r in doc["requests"] if r["status"] == "failed"]
+ self.assertEqual(failed, [])
+ self.assertTrue(all(r["status"] == "completed" for r in doc["requests"]))
+ self.assertEqual(doc["configuration"]["template"]["sha256"], bench_facts.sha256_text(template.source))
+ self.assertIsNotNone(doc["configuration"]["context_tokens"])
+ # the mock engine has no clock: the rates stay null and say why instead of being estimated
+ record = next(r for r in doc["requests"] if r["id"] == "performance-short-r1")
+ self.assertIsNone(record["metrics"]["decode_tps"])
+ self.assertIsNotNone(record["tokens"]["input_tokens"])
+ self.assertIsNotNone(record["timings"]["ttft_s"])
+ self.assertEqual(doc["run"]["status"], "nonconforming") # no model file to hash, so the identity is incomplete
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/terminal/test_runtime.py b/tests/terminal/test_runtime.py
index 56a726a50..feef2dec2 100644
--- a/tests/terminal/test_runtime.py
+++ b/tests/terminal/test_runtime.py
@@ -571,7 +571,7 @@ class RuntimeTests(unittest.TestCase):
self.assertTrue(result["profile_running_check"]["after"]["matched"])
output_json = io.StringIO()
with redirect_stdout(output_json):
- code = main(["bench", "run", "--profile", "local", "--require-managed-match",
+ code = main(["bench", "run", "--suite", "serving-smoke", "--profile", "local", "--require-managed-match",
"--concurrency", "1", "--out", str(self.root / "cli-managed"),
"--output", "json"])
self.assertEqual(code, 0)