Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
strata bench run: the standard benchmark protocol (standard-v1 is the default suite); fixtures and COMMIT file in the Terminal archive; tests with a mock server
Niko12216d41b37e769c
diff --git a/packaging/package_terminal.py b/packaging/package_terminal.pyindex 9d0111c22..7d1221061 100644--- a/packaging/package_terminal.py+++ b/packaging/package_terminal.py@@ -99,6 +99,7 @@ PAYLOAD = ("docs/MANAGEMENT_PROTOCOL.md","docs/TERMINAL_LAUNCH.md", # the README below points at it (`strata launch`); it used to be missing from the archive"data/gguf_fingerprints.json", # `strata models import --plan` names a known file set from this table+ "bench/prompts/standard-v1", # `strata bench run` (the standard benchmark) sends these frozen requests; hashed byte for byte)#: `tools/` modules the terminal paths import. Adding one here is a deliberate act: this list is what keeps@@ -343,6 +344,10 @@ def main() -> int:shutil.copy2(src, tools_dir / item)installer = None if args.no_installer else add_installer(root)(root / "VERSION").write_text(version + "\n", encoding="utf-8")+ # the commit the archive was built from: `strata bench run` records it (an archive has no .git to ask)+ built = subprocess.run(["git", "-C", str(ROOT), "rev-parse", "HEAD"], capture_output=True, text=True).stdout.strip().lower()+ if re.fullmatch(r"[0-9a-f]{40}", built):+ (root / "COMMIT").write_text(built + "\n", encoding="utf-8")bin_dir = root / "bin"bin_dir.mkdir(exist_ok=True)diff --git a/strata_cli/bench_facts.py b/strata_cli/bench_facts.pynew file mode 100644index 000000000..7d3cc7d5d--- /dev/null+++ b/strata_cli/bench_facts.py@@ -0,0 +1,488 @@+"""What a standard benchmark report says about the machine, the build, the model and the settings (docs/BENCHMARK_PROTOCOL.md).++Every probe is best effort and local: it reads the inference host's own tools (nvidia-smi, rocm-smi, PowerShell, /proc, sysctl) and the+server's answers. A fact that cannot be read becomes `None` with a reason in `Why`, never a guess. Nothing here changes the machine.+"""++from __future__ import annotations++import getpass+import hashlib+import json+import os+import platform+import re+import socket+import subprocess+import sys+import tempfile+from pathlib import Path++ROOT = Path(__file__).resolve().parents[1]+LOOPBACK = frozenset({"localhost", "127.0.0.1", "::1"})+HEX40 = re.compile(r"^[0-9a-f]{40}$")+MODEL_SUFFIXES = (".gguf", ".safetensors", ".pack", ".bin")+SECRET_KEY = re.compile(r"(?i)key|token|secret|password|auth")+++class Why(dict):+ """JSON Pointer -> the reason a value is null. Built while the report is assembled and written as `missing`."""++ def add(self, pointer: str, reason: str) -> None:+ self[pointer] = reason+++def pointer(*parts) -> str:+ return "".join("/" + str(p).replace("~", "~0").replace("/", "~1") for p in parts)+++# ------------------------------------------------------------------------------------------------------------ redaction+def redactor(key: str = ""):+ """text -> text with this machine's home folders, user and host names, the Strata folder, private IP addresses and the API key replaced."""+ subs: list[tuple[re.Pattern, str]] = []++ def add(value, label, word=False):+ if value and len(str(value)) >= 4:+ text = str(value)+ for form in sorted({text, text.replace("\\", "/"), text.replace("/", "\\")}, key=len, reverse=True):+ pattern = re.escape(form)+ subs.append((re.compile(r"(?<![\w-])" + pattern + r"(?![\w-])" if word else pattern, re.I), label))++ add(ROOT, "<strata>")+ add(Path.home(), "<home>")+ add(tempfile.gettempdir(), "<tmp>")+ try:+ add(getpass.getuser(), "<user>", True)+ except Exception: # noqa: BLE001 - no user name is not an error+ pass+ add(socket.gethostname(), "<host>", True)+ add(platform.node(), "<host>", True)+ add(key, "<key>")+ subs.sort(key=lambda item: -len(item[0].pattern))+ path_user = re.compile(r"(?i)([A-Za-z]:)?([\\/])(Users|home)([\\/])(?!<)[^\\/\s\"':;,]+")+ private_ip = re.compile(r"\b(?:192\.168|10\.\d{1,3}|172\.(?:1[6-9]|2\d|3[01])|100\.(?:6[4-9]|[7-9]\d|1[01]\d|12[0-7])"+ r"|169\.254)\.\d{1,3}\.\d{1,3}\b")+ bearer = re.compile(r"(?i)(bearer\s+)[A-Za-z0-9._~+/=-]{8,}")++ def clean(text):+ if not isinstance(text, str):+ return text+ for pattern, label in subs:+ text = pattern.sub(label, text)+ text = path_user.sub(lambda m: f"{m.group(1) or ''}{m.group(2)}{m.group(3)}{m.group(4)}<user>", text)+ text = bearer.sub(r"\1<key>", text)+ return private_ip.sub("<ip>", text)+ return clean+++def redact_object(value, clean):+ if isinstance(value, dict):+ return {clean(k) if isinstance(k, str) else k: redact_object(v, clean) for k, v in value.items()}+ if isinstance(value, list):+ return [redact_object(v, clean) for v in value]+ return clean(value) if isinstance(value, str) else value+++# ------------------------------------------------------------------------------------------------------------ commands+def run_cmd(cmd: list[str], timeout: float = 15) -> str:+ try:+ done = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, encoding="utf-8", errors="replace")+ return done.stdout if done.returncode == 0 else ""+ except (OSError, subprocess.SubprocessError):+ return ""+++def powershell(script: str, timeout: float = 20) -> str:+ return run_cmd(["powershell", "-NoProfile", "-NonInteractive", "-Command", script], timeout)+++def sha256_file(path: Path, chunk: int = 8 << 20) -> str:+ digest = hashlib.sha256()+ with open(path, "rb") as stream:+ while block := stream.read(chunk):+ digest.update(block)+ return digest.hexdigest()+++def sha256_text(text: str) -> str:+ return hashlib.sha256(text.encode("utf-8")).hexdigest()+++# ------------------------------------------------------------------------------------------------------------ hardware+def ram_bytes() -> int | None:+ try:+ import psutil+ return int(psutil.virtual_memory().total)+ except Exception: # noqa: BLE001+ pass+ try:+ for line in Path("/proc/meminfo").read_text().splitlines():+ if line.startswith("MemTotal:"):+ return int(line.split()[1]) * 1024+ except OSError:+ pass+ if sys.platform == "darwin":+ out = run_cmd(["sysctl", "-n", "hw.memsize"]).strip()+ return int(out) if out.isdigit() else None+ return None+++def cpu_name() -> str | None:+ try:+ for line in Path("/proc/cpuinfo").read_text().splitlines():+ if line.lower().startswith("model name"):+ return line.split(":", 1)[1].strip()+ except OSError:+ pass+ if sys.platform == "darwin":+ out = run_cmd(["sysctl", "-n", "machdep.cpu.brand_string"]).strip()+ if out:+ return out+ if os.name == "nt":+ out = powershell("(Get-CimInstance Win32_Processor | Select-Object -First 1).Name").strip()+ if out:+ return out+ return platform.processor() or None+++def nvidia_gpus() -> tuple[list[dict], str | None, str | None]:+ """-> (cards, driver version, CUDA version) from nvidia-smi."""+ out = run_cmd(["nvidia-smi", "--query-gpu=index,name,memory.total,power.limit,driver_version",+ "--format=csv,noheader,nounits"])+ cards = []+ for line in out.splitlines():+ part = [x.strip() for x in line.split(",")]+ if len(part) < 5 or not part[0].isdigit():+ continue++ def number(text):+ try:+ return float(text)+ except ValueError:+ return None+ mib, watts = number(part[2]), number(part[3])+ cards.append({"device_index": int(part[0]), "name": part[1],+ "memory_bytes": int(mib * 1024 * 1024) if mib else None,+ "memory_kind": "dedicated", "power_limit_w": watts, "driver": part[4]})+ cuda = None+ if cards:+ match = re.search(r"CUDA Version:\s*([\d.]+)", run_cmd(["nvidia-smi"]))+ cuda = match.group(1) if match else None+ return cards, (cards[0]["driver"] if cards else None), cuda+++def other_gpus() -> tuple[list[dict], str | None]:+ """AMD (rocm-smi, or the Windows registry), Apple silicon: -> (cards, driver version)."""+ cards, driver = [], None+ rocm = run_cmd(["rocm-smi", "--showproductname"])+ names = re.findall(r"Card [Ss]eries:\s*(.+)", rocm) or re.findall(r"Card [Mm]odel:\s*(.+)", rocm)+ if names:+ match = re.search(r"Driver version:\s*(\S+)", run_cmd(["rocm-smi", "--showdriverversion"]))+ driver = match.group(1) if match else None+ cards = [{"device_index": i, "name": n.strip(), "memory_bytes": None, "memory_kind": "dedicated",+ "power_limit_w": None} for i, n in enumerate(names)]+ elif os.name == "nt":+ try:+ if str(ROOT) not in sys.path:+ sys.path.insert(0, str(ROOT))+ from serve.telemetry import windows_amd_cards+ found = windows_amd_cards()+ cards = [{"device_index": i, "name": c["name"], "memory_bytes": c.get("vram_bytes") or None,+ "memory_kind": "dedicated", "power_limit_w": None} for i, c in enumerate(found)]+ driver = next((c.get("driver") for c in found if c.get("driver")), None)+ except Exception: # noqa: BLE001 - a report without the card's name is still a report+ pass+ elif sys.platform == "darwin":+ chip = cpu_name()+ if chip:+ cards = [{"device_index": 0, "name": chip, "memory_bytes": ram_bytes(), "memory_kind": "shared",+ "power_limit_w": None}]+ return cards, driver+++def storage_of(path: Path | None) -> list[dict] | None:+ """The drive that holds `path`: [{kind, model, bus}], or None when it cannot be told."""+ if path is None:+ return None+ try:+ path = path.resolve()+ except OSError:+ return None+ if os.name == "nt":+ letter = path.drive[:1]+ if not letter.isalpha():+ return None+ out = powershell(f"Get-Partition -DriveLetter {letter} | Get-Disk | Get-PhysicalDisk | "+ "Select-Object MediaType,FriendlyName,BusType | ConvertTo-Json -Compress")+ try:+ data = json.loads(out)+ except ValueError:+ return None+ rows = data if isinstance(data, list) else [data]+ return [{"kind": str(r.get("MediaType") or "unknown").lower(), "model": str(r.get("FriendlyName") or "unknown"),+ "bus": str(r.get("BusType") or "unknown").lower()} for r in rows if isinstance(r, dict)] or None+ if sys.platform.startswith("linux"):+ source = run_cmd(["findmnt", "-n", "-o", "SOURCE", "--target", str(path)]).strip()+ if not source.startswith("/dev/"):+ return None+ try:+ data = json.loads(run_cmd(["lsblk", "-J", "-s", "-o", "NAME,TYPE,ROTA,MODEL,TRAN", source]))+ except ValueError:+ return None+ for device in data.get("blockdevices") or []:+ chain = [device, *(device.get("children") or [])]+ for item in chain:+ if item.get("type") == "disk":+ rotational = str(item.get("rota")).lower() in ("1", "true")+ bus = str(item.get("tran") or "unknown").lower()+ return [{"kind": "hdd" if rotational else ("nvme" if bus == "nvme" else "ssd"),+ "model": str(item.get("model") or "unknown").strip(), "bus": bus}]+ return None+++def gpu_selection(config: dict | None, argv: list[str] | None) -> list[int]:+ """The card numbers the engine is told to use, from the run config's "gpu" or a `--gpu` in its arguments."""+ def parse(value):+ if value is None or value == "":+ return []+ items = value if isinstance(value, (list, tuple)) else str(value).split(",")+ return [int(x) for x in (str(i).strip() for i in items) if x.isdigit()]+ chosen = parse((config or {}).get("gpu"))+ if not chosen and argv and "--gpu" in argv and argv.index("--gpu") + 1 < len(argv):+ chosen = parse(argv[argv.index("--gpu") + 1])+ return chosen+++def hardware(why: Why, *, selected: list[int], served_gpu: str | None, model_dir: Path | None,+ declared_background: list[str]) -> tuple[dict, dict]:+ """-> (the report's `hardware`, the driver and runtime facts the `software` block uses)."""+ cards, driver, cuda = nvidia_gpus()+ if not cards:+ cards, driver = other_gpus()+ if selected:+ cards = [c for c in cards if c["device_index"] in selected] or cards+ elif len(cards) > 1 and served_gpu:+ named = [c for c in cards if c["name"] == served_gpu]+ cards = named[:1] or cards[:1]+ served = [n for n in (served_gpu or "").split(" + ") if n and n != "?"]+ if served and cards and not all(n in {c["name"] for c in cards} for n in served):+ # the server reads cards the local tools do not list (another vendor's, or a different numbering)+ cards = [{"device_index": i, "name": n, "memory_bytes": None, "memory_kind": "dedicated", "power_limit_w": None}+ for i, n in enumerate(served)]+ driver = cuda = None+ if not cards and served:+ cards = [{"device_index": i, "name": n, "memory_bytes": None, "memory_kind": "dedicated", "power_limit_w": None}+ for i, n in enumerate(served)]+ gpus = []+ for i, card in enumerate(cards):+ gpus.append({k: card.get(k) for k in ("device_index", "name", "memory_bytes", "memory_kind", "power_limit_w")})+ if gpus[-1]["memory_bytes"] is None:+ why.add(pointer("hardware", "gpus", i, "memory_bytes"), "the local tools did not report this card's memory")+ if gpus[-1]["power_limit_w"] is None:+ why.add(pointer("hardware", "gpus", i, "power_limit_w"), "no power limit available for this card from the local tools")+ ram = ram_bytes()+ cpu = cpu_name()+ drive = storage_of(model_dir)+ block = {"gpus": gpus, "cpu": cpu, "ram_bytes": ram, "storage": drive,+ "background_workloads": declared_background or None}+ if not gpus:+ why.add(pointer("hardware", "gpus"), "no GPU was found by the local tools or reported by the server")+ if cpu is None:+ why.add(pointer("hardware", "cpu"), "the CPU model could not be read")+ if ram is None:+ why.add(pointer("hardware", "ram_bytes"), "installed memory could not be read")+ if drive is None:+ why.add(pointer("hardware", "storage"), "the drive holding the model files could not be identified")+ if not declared_background:+ why.add(pointer("hardware", "background_workloads"),+ "the runner does not inspect other programs; none were declared with --background")+ return block, {"driver": driver, "cuda": cuda}+++# ------------------------------------------------------------------------------------------------------------ software+def os_info() -> dict:+ if os.name == "nt":+ return {"name": "Windows", "version": platform.version()}+ if sys.platform == "darwin":+ return {"name": "macOS", "version": platform.mac_ver()[0] or platform.release()}+ name = None+ try:+ for line in Path("/etc/os-release").read_text().splitlines():+ if line.startswith("PRETTY_NAME="):+ name = line.split("=", 1)[1].strip().strip('"')+ except OSError:+ pass+ return {"name": name or platform.system(), "version": platform.release()}+++def git_commit(folder: Path = ROOT) -> str | None:+ """The full commit of this checkout, or the COMMIT file a release archive carries; None for neither."""+ if (folder / ".git").exists():+ out = run_cmd(["git", "-C", str(folder), "rev-parse", "HEAD"]).strip().lower()+ if HEX40.match(out):+ return out+ try:+ text = (folder / "COMMIT").read_text(encoding="utf-8").strip().lower()+ except OSError:+ return None+ return text if HEX40.match(text) else None+++def software(why: Why, machine: dict, *, engine_version: str | None, build_options: list[str] | None) -> dict:+ commit = git_commit()+ runtime = []+ if machine.get("cuda"):+ runtime.append({"name": "cuda", "version": machine["cuda"]})+ if sys.platform == "darwin":+ runtime.append({"name": "metal", "version": platform.mac_ver()[0] or platform.release()})+ drivers = [{"name": "gpu-driver", "version": machine["driver"]}] if machine.get("driver") else []+ block = {"os": os_info(), "drivers": drivers or None, "runtime": runtime or None,+ "strata_commit": commit, "engine_version": engine_version,+ "build_kind": "source" if (ROOT / ".git").exists() else "release",+ "runner_commit": commit, "build_options": build_options}+ if not drivers:+ why.add("/software/drivers", "no GPU driver version was reported by the local tools")+ if not runtime:+ why.add("/software/runtime", "no CUDA, ROCm or Metal runtime version was reported by the local tools")+ if commit is None:+ for field in ("strata_commit", "runner_commit"):+ why.add(f"/software/{field}", "not a git checkout and no COMMIT file in this install; a release must carry its commit")+ if engine_version is None:+ why.add("/software/engine_version", "the server reported no engine version (/props build_info, /v1/status engine)")+ if build_options is None:+ why.add("/software/build_options", "the server does not report build options; pass --build-options for a source build")+ return block+++# ------------------------------------------------------------------------------------------------------------ model+QUANT = re.compile(r"(?i)(?:^|[-_.])((?:UD-)?(?:IQ\d_[A-Z]+|Q\d_[A-Z0-9_]+|Q\d_K(?:_[A-Z]+)?|IQ\d_?[A-Z]*|BF16|F16|NVFP4|Q\d))(?=[-_.]|$)")+SHARD = re.compile(r"^(.*)-(\d{5})-of-(\d{5})(\.gguf)$")+++def model_files_from(paths: list[str]) -> list[Path]:+ """The model files behind these paths: each existing file, and the other shards of a split GGUF."""+ found: dict[str, Path] = {}+ for raw in paths:+ try:+ path = Path(raw)+ if not path.is_file() or path.suffix.lower() not in MODEL_SUFFIXES:+ continue+ except (OSError, ValueError):+ continue+ found[str(path.resolve())] = path+ match = SHARD.match(path.name)+ if match:+ total = int(match.group(3))+ for index in range(1, total + 1):+ sibling = path.with_name(f"{match.group(1)}-{index:05d}-of-{match.group(3)}{match.group(4)}")+ if sibling.is_file():+ found[str(sibling.resolve())] = sibling+ return sorted(found.values(), key=lambda p: p.name)+++def detect_quantization(names: list[str]) -> str | None:+ for name in names:+ match = QUANT.search(name)+ if match:+ return match.group(1).upper()+ return None+++def model(why: Why, *, props: dict, status: dict, files: list[Path], declared: dict, log) -> dict:+ served = props.get("model_alias") or status.get("model")+ rows = []+ for index, path in enumerate(files):+ log(f"hashing model file {index + 1}/{len(files)}: {path.name}")+ rows.append({"path": path.name, "sha256": sha256_file(path)})+ quant = declared.get("quantization") or detect_quantization([p.name for p in files] + [str(served or "")])+ block = {"id": str(served) if served else None,+ "revision": declared.get("model_revision"), "source_revision": declared.get("source_revision"),+ "quantization": quant, "files": rows or None, "tokenizer_revision": declared.get("tokenizer_revision"),+ "custom_pack": declared.get("custom_pack")}+ reasons = {"id": "the server reported no model name", "quantization": "no quantization name found in the model file names; pass --quantization",+ "files": "no local model file could be found to hash (the server reports no model_path or it is not on this host)",+ "revision": "not given: the quantization repository's immutable revision cannot be read from the engine; pass --model-revision",+ "source_revision": "not given: pass --source-revision",+ "tokenizer_revision": "not given: pass --tokenizer-revision",+ "custom_pack": "no custom pack or expert profile was declared by the operator (--custom-pack)"}+ for field, reason in reasons.items():+ if block[field] is None:+ why.add(f"/model/{field}", reason)+ return block+++# ------------------------------------------------------------------------------------------------------------ configuration+GROUPS = (+ ("kv_cache", ("--kv",)),+ ("expert_cache", ("--expert",)),+ ("offloading", ("--gpu", "--layer-split", "--low-ram", "--offload", "--split", "--vram", "--pcie", "--host-core",+ "--pool", "--numa", "--unmap", "--cpu-expert", "--tail-skip")),+ ("prefill", ("--prefill",)),+ ("mtp", ("--mtp", "--spec", "--draft", "--batch-mtp")),+ ("calibration", ("--calib",)),+ ("experimental_options", ("--experimental", "--exp-")),+)+++def base_if_path(value):+ if isinstance(value, str) and (("/" in value or "\\" in value) or re.search(r"\.(gguf|bin|json|pack|txt|jinja|so|dll|exe)$", value, re.I)):+ return re.split(r"[\\/]", value.rstrip("\\/"))[-1]+ return value+++def parse_options(argv: list[str]) -> dict:+ """['--kv', 'int8', '--low-ram'] -> {'--kv': 'int8', '--low-ram': True}; a repeated option becomes a list."""+ options: dict = {}+ i = 0+ while i < len(argv):+ token = str(argv[i])+ if token.startswith("--"):+ name, value = (token.split("=", 1) + [None])[:2] if "=" in token else (token, None)+ if value is None:+ nxt = argv[i + 1] if i + 1 < len(argv) else None+ if nxt is not None and not (str(nxt).startswith("--") and not re.fullmatch(r"-?\d+(\.\d+)?", str(nxt))):+ value, i = str(nxt), i + 1+ else:+ value = True+ value = base_if_path(value)+ if name in options:+ options[name] = [*(options[name] if isinstance(options[name], list) else [options[name]]), value]+ else:+ options[name] = value+ i += 1+ return options+++def configuration(why: Why, *, options: dict | None, env: dict | None, props: dict, declared: dict,+ sampling: dict, template_ref: dict | None) -> tuple[dict, dict]:+ """-> (the report's `configuration`, the engine options no group takes: they go to extensions)."""+ grouped = {name: {} for name, _ in GROUPS}+ other = {}+ for name, value in (options or {}).items():+ for group, prefixes in GROUPS:+ if name.startswith(prefixes):+ grouped[group][name] = value+ break+ else:+ other[name] = value+ for name, value in sorted((env or {}).items()):+ if name.startswith("STRATA_") and not SECRET_KEY.search(name):+ grouped["experimental_options"][name] = base_if_path(str(value))+ n_ctx = ((props.get("default_generation_settings") or {}).get("n_ctx"))+ slots = props.get("total_slots")+ block = {"track": declared["track"], "track_notes": declared["track_notes"],+ "context_tokens": n_ctx if isinstance(n_ctx, int) and not isinstance(n_ctx, bool) else None,+ "server_capacity": slots if isinstance(slots, int) and not isinstance(slots, bool) else None,+ **({name: grouped[name] for name, _ in GROUPS} if options is not None else {name: None for name, _ in GROUPS}),+ "template": template_ref, "sampling": dict(sampling["sampling"]),+ "chat_template_kwargs": dict(sampling["chat_template_kwargs"])}+ if block["context_tokens"] is None:+ why.add("/configuration/context_tokens", "the server's /props reported no context size")+ if block["server_capacity"] is None:+ why.add("/configuration/server_capacity", "the server's /props reported no slot count")+ if options is None:+ for name, _ in GROUPS:+ why.add(f"/configuration/{name}", "engine options are not exposed by a running server; start it from a profile or pass --config")+ if template_ref is None:+ why.add("/configuration/template", "the server's /props returned no chat template")+ return block, otherdiff --git a/strata_cli/bench_standard.py b/strata_cli/bench_standard.pynew file mode 100644index 000000000..c1552793f--- /dev/null+++ b/strata_cli/bench_standard.py@@ -0,0 +1,1208 @@+"""`strata bench run` for the standard benchmark protocol (docs/BENCHMARK_PROTOCOL.md).++The runner sends the frozen requests of bench/prompts/standard-v1 to a Strata server through streaming chat completions, in the+protocol's order, and writes a `strata-standard-benchmark` result that other people's results can be compared with:++ warm-up (1, excluded) -> performance (short, prompt-4k, prompt-16k, three times) -> sanity (I1-I4, R1-R4, T1-T4)+ -> optional concurrency (four short requests released together, three rounds)++Speeds are the engine's own timings (the `timings` of the last stream chunk); first-token times are the client's monotonic clock. A value+that cannot be read stays `None` with a reason in `missing`; nothing is estimated from chunks, characters or words. Prompts, answers and+timings are saved without home folders, host names, private addresses or keys.+"""++from __future__ import annotations++import hashlib+import http.client+import json+import math+import os+import re+import socket+import statistics+import subprocess+import sys+import threading+import time+import urllib.parse+import uuid+from concurrent.futures import ThreadPoolExecutor+from datetime import datetime, timezone+from pathlib import Path++from strata_cli import bench_facts as facts+from strata_cli.bench import _private_mkdir, _write_new+from strata_control.profiles import ProfileError++FORMAT = "strata-standard-benchmark"+SCHEMA_VERSION = 1+PROTOCOL_ID = "strata-qwen38-thinking-v1-draft.1"+PROFILE = "qwen3.8-flash-next-thinking"+SUITE = "standard-v1"+#: The model-card revision the profile was defined from. The owner has not pinned one yet, so every result says so.+MODEL_CARD_REVISION: str | None = None+MODEL_CARD_URL = "https://huggingface.co/Qwen/Qwen3.8-Flash-Next"++REQUEST_TIMEOUT_S = 1800.0+#: Prompt tokens the cache may report beyond the longest prefix a request shares with the rest of the bundle before the+#: user payload counts as reused. The slack covers token merges at the boundary.+REUSE_SLACK_TOKENS = 8+LENGTH_TOLERANCE = 0.02++SAMPLING = {"temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0,+ "repetition_penalty": 1.0, "reasoning_effort": "xhigh"}+TEMPLATE_KWARGS = {"enable_thinking": True, "preserve_thinking": True}+PHASES = ("warmup", "performance", "sanity", "concurrency")+PLANNED = {"warmup": 1, "performance": 9, "sanity": 12, "concurrency": 12}+PERFORMANCE_CASES = ("short", "prompt-4k", "prompt-16k")+METRICS = (("prompt_tps", "tokens/s"), ("decode_tps", "tokens/s"), ("ttft_s", "s"),+ ("first_answer_s", "s"), ("total_s", "s"))+HEX64 = re.compile(r"^[0-9a-f]{64}$")+++class CorpusError(ProfileError):+ pass+++def _now() -> str:+ return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")+++def _number(value) -> float | None:+ if isinstance(value, bool) or not isinstance(value, (int, float)):+ return None+ return float(value) if math.isfinite(value) and value >= 0 else None+++def _count(value) -> int | None:+ if isinstance(value, bool) or not isinstance(value, int) or value < 0:+ if isinstance(value, float) and value.is_integer() and value >= 0 and math.isfinite(value):+ return int(value)+ return None+ return value+++# ------------------------------------------------------------------------------------------------------------ the fixtures+class Corpus:+ def __init__(self, sha256: str, specs: list[dict], build: dict, manifest: bytes, directory: Path):+ self.sha256, self.specs, self.build, self.manifest, self.directory = sha256, specs, build, manifest, directory+++def corpus_dir() -> Path:+ return Path(__file__).resolve().parents[1] / "bench" / "prompts" / SUITE+++def load_corpus(directory: Path | None = None) -> Corpus:+ """Read the fixture bundle and check every file against its manifest. Any difference is an error: the run would not be the protocol's."""+ root = Path(directory) if directory else corpus_dir()+ try:+ manifest = (root / "MANIFEST.sha256").read_bytes()+ lines = manifest.decode("utf-8").splitlines()+ except (OSError, UnicodeError):+ raise CorpusError("the standard benchmark fixtures are missing (bench/prompts/standard-v1/MANIFEST.sha256)", 5) from None+ entries: dict[str, str] = {}+ for line in lines:+ digest, _, name = line.partition(" ")+ if not HEX64.match(digest) or not name or name.startswith(("/", "\\")) or ".." in Path(name).parts:+ raise CorpusError("the fixture manifest is malformed", 5)+ entries[name] = digest+ for name, digest in entries.items():+ try:+ data = (root / name).read_bytes()+ except OSError:+ raise CorpusError(f"fixture file missing: {name}", 5) from None+ if hashlib.sha256(data).hexdigest() != digest:+ raise CorpusError(f"fixture file changed: {name} (line endings converted? bench/prompts is hashed byte for byte)", 5)+ extra = {p.relative_to(root).as_posix() for p in (root / "requests").glob("*.json")} - set(entries)+ if extra:+ raise CorpusError(f"fixture file not in the manifest: {sorted(extra)[0]}", 5)+ try:+ build = json.loads((root / "build.json").read_text(encoding="utf-8"))+ specs = [json.loads((root / "requests" / f"{row['id']}.json").read_text(encoding="utf-8")) for row in build["requests"]]+ except (OSError, ValueError, KeyError, TypeError):+ raise CorpusError("the fixture bundle is unreadable", 5) from None+ counts = {p: sum(1 for s in specs if s.get("phase") == p) for p in PHASES}+ if counts != PLANNED or len({s["id"] for s in specs}) != len(specs) or build.get("protocol_id") != PROTOCOL_ID:+ raise CorpusError("the fixture bundle does not match this protocol", 5)+ return Corpus(hashlib.sha256(manifest).hexdigest(), specs, build, manifest, root)+++# ------------------------------------------------------------------------------------------------------------ HTTP+class Target:+ def __init__(self, root: str, key: str | None):+ parts = urllib.parse.urlsplit(root)+ self.scheme, self.host = parts.scheme, parts.hostname or ""+ self.port = parts.port or (443 if parts.scheme == "https" else 80)+ self.key = key+ self.root = root.rstrip("/")++ @property+ def loopback(self) -> bool:+ return self.host in facts.LOOPBACK++ def connection(self, timeout: float) -> http.client.HTTPConnection:+ cls = http.client.HTTPSConnection if self.scheme == "https" else http.client.HTTPConnection+ return cls(self.host, self.port, timeout=timeout)++ def headers(self, extra: dict | None = None) -> dict:+ out = {"User-Agent": "Strata-Bench/1", "Accept": "application/json", "Connection": "close", **(extra or {})}+ if self.key:+ out["Authorization"] = "Bearer " + self.key+ return out++ def get(self, path: str, timeout: float = 10) -> dict | None:+ conn = self.connection(timeout)+ try:+ conn.request("GET", path, headers=self.headers())+ response = conn.getresponse()+ body = response.read(8 << 20)+ if response.status != 200:+ return None+ value = json.loads(body)+ return value if isinstance(value, dict) else None+ except (OSError, http.client.HTTPException, ValueError):+ return None+ finally:+ conn.close()+++def payload_of(spec: dict, model: str | None) -> dict:+ """The request body: the protocol's sampling and thinking settings, every one explicit, plus the fixture's messages and tools."""+ body = {"model": model or "strata", "messages": spec["messages"], "stream": True,+ "stream_options": {"include_usage": True}, "max_tokens": spec["max_output_tokens"], "seed": spec["seed"],+ **SAMPLING, "reasoning_budget_tokens": 0, "chat_template_kwargs": dict(TEMPLATE_KWARGS)}+ if spec.get("tools"):+ body["tools"] = spec["tools"]+ body["tool_choice"] = spec.get("tool_choice") or "auto"+ return body+++def stream_chat(target: Target, spec: dict, model: str | None, *, timeout_s: float,+ gate: threading.Event | None = None) -> dict:+ """Send one streamed request and keep what the client saw, with monotonic offsets from the moment of dispatch."""+ body = json.dumps(payload_of(spec, model)).encode("utf-8")+ raw: dict = {"http_status": None, "error": None, "finish_reason": None, "usage": None, "timings": None,+ "events": [], "reasoning": [], "content": [], "tool_calls": {}, "done": False,+ "timed_out": False, "interrupted": False}+ if gate is not None:+ gate.wait()+ raw["t0"] = start = time.monotonic_ns()+ conn = target.connection(timeout_s)++ def expire():+ raw["timed_out"] = True+ sock = getattr(conn, "sock", None)+ if sock is not None:+ try:+ sock.shutdown(socket.SHUT_RDWR)+ except OSError:+ pass+ try:+ sock.close()+ except OSError:+ pass+ timer = threading.Timer(timeout_s, expire)+ timer.daemon = True+ timer.start()++ def offset() -> float:+ return round((time.monotonic_ns() - start) / 1e9, 6)++ def mark(kind: str) -> None:+ raw["events"].append([offset(), kind])++ try:+ conn.request("POST", "/v1/chat/completions", body,+ target.headers({"Content-Type": "application/json", "Accept": "text/event-stream"}))+ response = conn.getresponse()+ raw["http_status"] = response.status+ if response.status != 200:+ text = response.read(4096).decode("utf-8", "replace")+ raw["error"] = {"kind": f"http_{response.status}", "message": text.strip()[:300] or f"HTTP {response.status}"}+ else:+ while True:+ line = response.readline(1 << 20)+ if not line:+ break+ if not line.startswith(b"data:"):+ continue+ data = line[5:].strip()+ if data == b"[DONE]":+ raw["done"] = True+ break+ try:+ event = json.loads(data)+ except ValueError:+ raw["error"] = {"kind": "malformed_stream", "message": "a stream event was not JSON"}+ break+ if not isinstance(event, dict):+ raw["error"] = {"kind": "malformed_stream", "message": "a stream event was not an object"}+ break+ if "error" in event:+ err = event["error"]+ kind = (err.get("code") or err.get("type")) if isinstance(err, dict) else None+ raw["error"] = {"kind": str(kind or "stream_error")[:60],+ "message": str(err.get("message") if isinstance(err, dict) else err)[:300]}+ break+ if isinstance(event.get("usage"), dict):+ raw["usage"] = event["usage"]+ if isinstance(event.get("timings"), dict):+ raw["timings"] = event["timings"]+ for choice in event.get("choices") or []:+ if not isinstance(choice, dict):+ continue+ delta = choice.get("delta") if isinstance(choice.get("delta"), dict) else {}+ reasoning, text = delta.get("reasoning_content"), delta.get("content")+ if isinstance(reasoning, str) and reasoning:+ raw["reasoning"].append(reasoning)+ mark("reasoning")+ if isinstance(text, str) and text:+ raw["content"].append(text)+ mark("answer")+ calls = delta.get("tool_calls")+ if isinstance(calls, list):+ qualifies = False+ for call in calls:+ if not isinstance(call, dict):+ continue+ slot = raw["tool_calls"].setdefault(call.get("index", len(raw["tool_calls"])),+ {"id": "", "name": "", "arguments": ""})+ function = call.get("function") if isinstance(call.get("function"), dict) else {}+ for field, value in (("id", call.get("id")), ("name", function.get("name")),+ ("arguments", function.get("arguments"))):+ if isinstance(value, str) and value:+ slot[field] += value+ qualifies = True+ if qualifies:+ mark("tool_call")+ if choice.get("finish_reason"):+ raw["finish_reason"] = str(choice["finish_reason"])+ if raw["error"] is None:+ if not raw["done"]:+ raw["error"] = {"kind": "malformed_stream", "message": "the stream ended without [DONE]"}+ elif raw["finish_reason"] is None:+ raw["error"] = {"kind": "malformed_stream", "message": "the stream carried no finish_reason"}+ except KeyboardInterrupt:+ raw["interrupted"] = True+ raw["error"] = {"kind": "cancelled", "message": "cancelled by the operator"}+ except (OSError, http.client.HTTPException, ValueError) as exc:+ if raw["timed_out"]:+ raw["error"] = {"kind": "timeout", "message": f"no complete answer within {timeout_s:g} seconds"}+ else:+ raw["error"] = {"kind": "connection_" + type(exc).__name__, "message": str(exc)[:200] or type(exc).__name__}+ finally:+ timer.cancel()+ try:+ conn.close()+ except OSError:+ pass+ if raw["timed_out"] and raw["error"] is None:+ raw["error"] = {"kind": "timeout", "message": f"no complete answer within {timeout_s:g} seconds"}+ raw["t_end"] = time.monotonic_ns()+ return raw+++# ------------------------------------------------------------------------------------------------------------ scoring+def strict_json(text: str):+ """One complete JSON value: no duplicate keys, no NaN or Infinity, no prose or fence around it."""+ def pairs(items):+ out = {}+ for key, value in items:+ if key in out:+ raise ValueError(f"duplicate key {key!r}")+ out[key] = value+ return out++ def constant(name):+ raise ValueError(f"{name} is not JSON")+ return json.loads(text.strip(" \t\r\n\x0b\x0c"), object_pairs_hook=pairs, parse_constant=constant)+++def same_json(a, b) -> bool:+ """Equal values of equal JSON types (true is not 1, 3 is not 3.0); object key order does not matter."""+ if isinstance(a, bool) or isinstance(b, bool):+ return isinstance(a, bool) and isinstance(b, bool) and a == b+ if isinstance(a, dict) and isinstance(b, dict):+ return a.keys() == b.keys() and all(same_json(a[k], b[k]) for k in a)+ if isinstance(a, list) and isinstance(b, list):+ return len(a) == len(b) and all(same_json(x, y) for x, y in zip(a, b))+ return type(a) is type(b) and a == b+++def evaluate(spec: dict, status: str, answer: str, calls: list[dict]) -> dict:+ expected = spec["expected"]+ category = spec["category"]++ def verdict(passed: bool, reason: str) -> dict:+ return {"category": category, "passed": passed, "reason": reason}+ if status != "completed":+ return verdict(False, {"truncated": "the output cap was reached before the answer was complete",+ "failed": "the request failed", "skipped": "the request was skipped"}[status])+ kind = expected["kind"]+ if kind == "tool_call":+ if len(calls) != 1:+ return verdict(False, f"expected one structured tool call, got {len(calls)}")+ call = calls[0]+ if call["name"] != expected["name"]:+ return verdict(False, f"wrong function {call['name']!r}")+ try:+ arguments = strict_json(call["arguments"] or "{}")+ except ValueError as exc:+ return verdict(False, f"the tool arguments are not strict JSON ({exc})")+ if not isinstance(arguments, dict) or not same_json(arguments, expected["arguments"]):+ return verdict(False, "the tool arguments differ from the expected values")+ return verdict(True, "the expected tool call")+ if kind == "text":+ text = answer.strip(" \t\r\n\x0b\x0c")+ return verdict(text == expected["value"],+ "the exact expected text" if text == expected["value"] else "the answer text differs from the expected text")+ try:+ value = strict_json(answer)+ except ValueError as exc:+ return verdict(False, f"the answer is not one strict JSON value ({exc})")+ return verdict(same_json(value, expected["value"]),+ "the expected JSON value" if same_json(value, expected["value"]) else "the JSON value differs from the expected value")+++# ------------------------------------------------------------------------------------------------------------ one request+def _round(value: float | None) -> float | None:+ return None if value is None else round(value, 6)+++def record_of(spec: dict, raw: dict, meta: dict, why: facts.Why | None = None) -> tuple[dict, dict, dict]:+ """-> (the report's request record, evidence for outputs.jsonl, extras for the report's `extensions`)."""+ usage = raw["usage"] if isinstance(raw.get("usage"), dict) else {}+ timings = raw["timings"] if isinstance(raw.get("timings"), dict) else {}+ details = usage.get("prompt_tokens_details") if isinstance(usage.get("prompt_tokens_details"), dict) else {}+ error = raw["error"]+ finish = raw["finish_reason"]+ if error is not None:+ status = "failed"+ elif finish == "length":+ status = "truncated"+ elif finish in ("stop", "tool_calls"):+ status = "completed"+ else:+ status = "failed"+ error = {"kind": "unexpected_finish_reason", "message": f"finish_reason {finish!r}"}+ t0 = raw["t0"]+ events = raw["events"]+ first = events[0] if events else None+ answer = next((e for e in events if e[1] in ("answer", "tool_call")), None)++ input_tokens, output_tokens = _count(usage.get("prompt_tokens")), _count(usage.get("completion_tokens"))+ cached, cached_source = _count(details.get("cached_tokens")), "usage.prompt_tokens_details.cached_tokens"+ if cached is None:+ cached, cached_source = _count(timings.get("cache_n")), "timings.cache_n"+ read, read_source = _count(timings.get("prompt_n")), "timings.prompt_n"+ if read is None and input_tokens is not None and cached is not None and cached <= input_tokens:+ read, read_source = input_tokens - cached, "derived: usage.prompt_tokens - cached tokens"+ prompt_s = _number(timings.get("prompt_ms"))+ prompt_s = prompt_s / 1000 if prompt_s is not None else None+ decode_s = _number(timings.get("predicted_ms"))+ decode_s = decode_s / 1000 if decode_s is not None else None+ rate, predicted_n = _number(timings.get("predicted_per_second")), _count(timings.get("predicted_n"))+ decode_tokens, decode_source, note = None, None, None+ if rate is not None and decode_s:+ implied = round(rate * decode_s)+ # the engine rounds its rate to 0.1 tokens/s, so the implied count is good to about 0.05 * decode_s tokens+ if predicted_n is not None and abs(predicted_n - implied) <= 0.05 * decode_s + 1:+ decode_tokens, decode_source = predicted_n, "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)"+ else:+ decode_tokens = implied+ decode_source = "derived: timings.predicted_per_second x timings.predicted_ms / 1000 (the count the engine's rate used)"+ if predicted_n is not None:+ note = (f"timings.predicted_n is {predicted_n} but the engine's decode rate and time imply {implied} tokens; "+ "the implied count is used because it is the one the rate was computed from")+ elif predicted_n is not None:+ decode_tokens, decode_source = predicted_n, "timings.predicted_n (no decode rate to check it against)"+ if note is None and output_tokens is not None and decode_tokens is not None and output_tokens != decode_tokens:+ note = (f"API usage.completion_tokens is {output_tokens}; the engine's decode count is {decode_tokens}. The engine count "+ "feeds decode_tps; the API count feeds the concurrent aggregate")+ prompt_tps = read / prompt_s if read and prompt_s else None+ decode_tps = decode_tokens / decode_s if decode_tokens and decode_s else None++ text = "".join(raw["content"])+ calls = [raw["tool_calls"][k] for k in sorted(raw["tool_calls"], key=lambda k: (not isinstance(k, int), k))]+ ttft, first_answer = (first[0] if first else None), (answer[0] if answer else None)+ total = round((raw["t_end"] - t0) / 1e9, 6)+ kind_names = {"reasoning": "reasoning", "answer": "answer", "tool_call": "tool_call"}+ record = {+ "id": spec["id"], "phase": spec["phase"], "case_id": spec["case_id"], "repetition": spec["repetition"],+ "seed": spec["seed"], "max_output_tokens": spec["max_output_tokens"], "concurrency": spec["concurrency"],+ "round": spec["round"], "status": status, "finish_reason": finish,+ "error": None if status in ("completed", "truncated") else error,+ "tokens": {"input_tokens": input_tokens, "cached_tokens": cached, "read_tokens": read,+ "output_tokens": output_tokens, "decode_tokens": decode_tokens},+ "timings": {"prompt_s": _round(prompt_s), "decode_s": _round(decode_s), "ttft_s": _round(ttft),+ "first_answer_s": _round(first_answer), "total_s": total},+ "metrics": {"prompt_tps": _round(prompt_tps), "decode_tps": _round(decode_tps)},+ "sources": {"input_tokens": "usage.prompt_tokens of the final stream chunk" if input_tokens is not None else None,+ "cached_tokens": cached_source if cached is not None else None,+ "read_tokens": read_source if read is not None else None,+ "output_tokens": "usage.completion_tokens of the final stream chunk" if output_tokens is not None else None,+ "decode_tokens": decode_source,+ "engine_timings": ("timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream "+ "chunk (raw values in outputs.jsonl)") if timings else None,+ "client_timings": ("client monotonic clock from dispatch; event offsets in outputs.jsonl")},+ "first_delta_kind": kind_names.get(first[1]) if first else None,+ "token_accounting_note": note, "evaluation": None}+ if spec["phase"] == "sanity":+ record["evaluation"] = evaluate(spec, status, text, [c for c in calls if c["name"] or c["arguments"]])+ reuse_limit = meta["max_shared_prefix_tokens"] + REUSE_SLACK_TOKENS+ extras = {"max_shared_prefix_tokens": meta["max_shared_prefix_tokens"],+ "frozen_input_tokens": meta["input_tokens"],+ "unexpected_payload_reuse": bool(cached is not None and cached > reuse_limit),+ "length_outside_tolerance": bool(spec.get("target_input_tokens") and input_tokens is not None and+ abs(input_tokens - spec["target_input_tokens"]) >+ LENGTH_TOLERANCE * spec["target_input_tokens"])}+ evidence = {"request_id": spec["id"], "phase": spec["phase"], "case_id": spec["case_id"],+ "request": {k: v for k, v in payload_of(spec, None).items() if k not in ("messages", "tools", "model")},+ "messages": spec["messages"], "tools": spec.get("tools"),+ "final_answer": text, "reasoning": "".join(raw["reasoning"]),+ "tool_calls": [{"id": c["id"], "name": c["name"], "arguments": c["arguments"]} for c in calls],+ "finish_reason": finish, "usage": raw.get("usage"), "timings": raw.get("timings"), "error": error,+ "http_status": raw["http_status"], "dispatch_offset_s": 0.0, "end_offset_s": total,+ "first_delta": {"offset_s": first[0], "kind": first[1]} if first else None,+ "first_answer_offset_s": first_answer, "client_events": events}+ return record, evidence, extras+++def skipped_record(spec: dict, reason: str) -> tuple[dict, dict, dict]:+ record = {"id": spec["id"], "phase": spec["phase"], "case_id": spec["case_id"], "repetition": spec["repetition"],+ "seed": spec["seed"], "max_output_tokens": spec["max_output_tokens"], "concurrency": spec["concurrency"],+ "round": spec["round"], "status": "skipped", "finish_reason": None,+ "error": {"kind": "skipped", "message": reason},+ "tokens": {k: None for k in ("input_tokens", "cached_tokens", "read_tokens", "output_tokens", "decode_tokens")},+ "timings": {k: None for k in ("prompt_s", "decode_s", "ttft_s", "first_answer_s", "total_s")},+ "metrics": {"prompt_tps": None, "decode_tps": None},+ "sources": {k: None for k in ("input_tokens", "cached_tokens", "read_tokens", "output_tokens", "decode_tokens",+ "engine_timings", "client_timings")},+ "first_delta_kind": None, "token_accounting_note": None, "evaluation": None}+ if spec["phase"] == "sanity":+ record["evaluation"] = {"category": spec["category"], "passed": False, "reason": "the request was skipped: " + reason}+ return record, {"request_id": spec["id"], "skipped": reason}, {"unexpected_payload_reuse": False,+ "length_outside_tolerance": False}+++# ------------------------------------------------------------------------------------------------------------ the run+class Collector:+ """Runs the scheduled requests in order and keeps their records."""++ def __init__(self, target: Target, corpus: Corpus, model: str | None, log, timeout_s: float):+ self.target, self.corpus, self.model, self.log, self.timeout_s = target, corpus, model, log, timeout_s+ self.meta = {row["id"]: row for row in corpus.build["requests"]}+ self.results: dict[str, tuple[dict, dict, dict]] = {}+ self.abort: str | None = None+ self.number = 0+ self.total = sum(PLANNED[p] for p in ("warmup", "performance", "sanity"))++ def finish(self, spec: dict, raw: dict) -> dict:+ result = record_of(spec, raw, self.meta[spec["id"]])+ self.results[spec["id"]] = result+ return result[0]++ def alive(self) -> bool:+ health = self.target.get("/health", timeout=5)+ return bool(health and health.get("service") == "strata")++ def one(self, spec: dict) -> None:+ if self.abort:+ self.results[spec["id"]] = skipped_record(spec, self.abort)+ return+ self.number += 1+ self.log(f"[{self.number}/{self.total}] {spec['id']} (up to {spec['max_output_tokens']} tokens)")+ raw = stream_chat(self.target, spec, self.model, timeout_s=self.timeout_s)+ record = self.finish(spec, raw)+ timing = record["timings"]+ speed = record["metrics"]+ self.log(f" {record['status']}: prompt {speed['prompt_tps'] or 'n/a'} tok/s, decode {speed['decode_tps'] or 'n/a'} tok/s, "+ f"first token {timing['ttft_s'] if timing['ttft_s'] is not None else 'n/a'} s"+ + (f"; {record['error']['kind']}" if record["error"] else ""))+ if raw["interrupted"]:+ self.abort = "the operator cancelled the run"+ elif record["status"] == "failed":+ if spec["phase"] == "warmup":+ self.abort = f"the warm-up request failed ({record['error']['kind']}); the measured stages were not run"+ elif not self.alive():+ self.abort = f"the server stopped answering after {spec['id']} ({record['error']['kind']})"++ def concurrent_round(self, specs: list[dict]) -> dict:+ gate = threading.Event()+ raws: dict[str, dict] = {}++ def work(spec):+ raws[spec["id"]] = stream_chat(self.target, spec, self.model, timeout_s=self.timeout_s, gate=gate)+ with ThreadPoolExecutor(max_workers=len(specs)) as pool:+ futures = [pool.submit(work, spec) for spec in specs]+ time.sleep(0.05)+ gate.set()+ for future in futures:+ future.result()+ records = [self.finish(spec, raws[spec["id"]]) for spec in specs]+ start = min(raws[s["id"]]["t0"] for s in specs)+ end = max(raws[s["id"]]["t_end"] for s in specs)+ wall = round((end - start) / 1e9, 6)+ counted = [r for r in records if r["status"] in ("completed", "truncated")]+ outputs = [r["tokens"]["output_tokens"] for r in counted]+ total = sum(outputs) if outputs and all(o is not None for o in outputs) else None+ failed = sum(r["status"] == "failed" for r in records)+ self.log(f" round {specs[0]['round']}: {len(counted)} completed or capped, {failed} failed, {wall} s")+ if any(raws[s["id"]]["interrupted"] for s in specs):+ self.abort = "the operator cancelled the run"+ return {"round": specs[0]["round"], "request_ids": [s["id"] for s in specs], "wall_s": wall,+ "output_tokens": total, "aggregate_tps": _round(total / wall) if total is not None and wall > 0 else None,+ "failed": failed}+++def summaries_of(records: list[dict], extras: dict, phase: str, cases: tuple[str, ...], concurrency: int) -> list[dict]:+ rows = []+ for case in cases:+ group = [r for r in records if r["phase"] == phase and r["case_id"] == case]+ for metric, unit in METRICS:+ included, excluded, values = [], [], []+ for r in group:+ reason = None+ if r["status"] == "skipped":+ reason = "skipped: " + r["error"]["message"]+ elif r["status"] == "failed":+ reason = "failed: " + r["error"]["kind"]+ elif extras[r["id"]]["unexpected_payload_reuse"]:+ reason = (f"unexpected cache reuse: {r['tokens']['cached_tokens']} cached tokens exceed the "+ f"{extras[r['id']]['max_shared_prefix_tokens']}-token shared template prefix")+ else:+ value = r["metrics"].get(metric) if metric in ("prompt_tps", "decode_tps") else r["timings"].get(metric)+ if value is None:+ reason = f"{metric} is not available for this request"+ else:+ values.append(value)+ included.append(r["id"])+ if reason:+ excluded.append({"request_id": r["id"], "reason": reason})+ rows.append({"phase": phase, "case_id": case, "concurrency": concurrency, "metric": metric, "unit": unit,+ "median": _round(statistics.median(values)) if values else None,+ "min": _round(min(values)) if values else None, "max": _round(max(values)) if values else None,+ "n": len(values), "included_request_ids": included, "excluded": excluded})+ return rows+++def counts_of(records: list[dict], phase: str, planned: int) -> dict:+ rows = [r for r in records if r["phase"] == phase]+ tally = {s: sum(r["status"] == s for r in rows) for s in ("completed", "truncated", "failed", "skipped")}+ attempted = tally["completed"] + tally["truncated"] + tally["failed"]+ return {"planned": planned, "attempted": attempted, **tally}+++def quality_of(records: list[dict]) -> dict:+ sanity = [r for r in records if r["phase"] == "sanity"]+ cats = {name: {"passed": sum(r["evaluation"]["passed"] for r in sanity if r["evaluation"]["category"] == name),+ "total": 4} for name in ("instruction", "retrieval", "tool_call")}+ return {"passed": sum(c["passed"] for c in cats.values()), "total": 12, "categories": cats}+++def scan_nulls(value, path: str = ""):+ if value is None:+ yield path+ elif isinstance(value, dict):+ for key, item in value.items():+ if path == "" and key in ("missing", "extensions"):+ continue+ yield from scan_nulls(item, path + facts.pointer(key))+ elif isinstance(value, list):+ for index, item in enumerate(value):+ yield from scan_nulls(item, path + facts.pointer(index))+++def default_reason(doc: dict, pointer: str) -> str:+ parts = [p.replace("~1", "/").replace("~0", "~") for p in pointer.split("/")[1:]]+ if parts[0] == "requests" and len(parts) >= 3:+ record = doc["requests"][int(parts[1])]+ field = ".".join(parts[2:])+ if record["status"] == "skipped":+ return "not measured: the request was skipped (" + record["error"]["message"] + ")"+ if field == "error":+ return "none: the request ended normally"+ if field == "round":+ return "not applicable outside the concurrency stage"+ if field == "evaluation":+ return "not applicable outside the sanity stage"+ if field == "token_accounting_note":+ return "none: the API and engine counts agree or only one exists"+ if field == "finish_reason":+ return "the server sent no finish_reason"+ if field == "first_delta_kind":+ return "no nonempty reasoning, answer or tool-call delta arrived"+ if field in ("timings.ttft_s", "timings.first_answer_s"):+ return ("no nonempty delta arrived" if field == "timings.ttft_s"+ else "no answer text or tool call arrived (reasoning alone does not count)")+ if field.startswith("metrics."):+ return ("the rate needs a nonzero freshly read count (or generated count) and an engine duration, and one is missing or zero")+ if field.startswith("sources."):+ return "no source: the value was not reported"+ if record["status"] == "failed":+ return f"the request failed ({record['error']['kind']}) before this value was reported"+ return "the server's response did not include this value (usage or timings)"+ if parts[0] == "summaries" and len(parts) >= 3:+ row = doc["summaries"][int(parts[1])]+ return f"no valid samples of {row['metric']} for {row['case_id']} ({len(row['excluded'])} excluded)"+ if parts[0] == "concurrency_rounds":+ return "an included request has no output token count, or the round wall time is zero"+ if parts[0] == "run" and parts[1] == "optional_duration_s":+ return "the optional concurrency stage was not selected or did not run"+ if parts[0] == "protocol" and parts[1] == "model_card_revision":+ return "the profile's model-card revision is not pinned yet (protocol draft)"+ return "not available"+++def _start_wait(target: Target, proc, timeout_s: float, log, log_path: Path | None) -> bool:+ began, announced = time.monotonic(), 0.0+ while time.monotonic() - began < timeout_s:+ if proc is not None and proc.poll() is not None:+ return False+ health = target.get("/health", timeout=5)+ if health and health.get("service") == "strata" and health.get("loaded"):+ return True+ if time.monotonic() - announced > 30:+ log(f"waiting for the model to load ({int(time.monotonic() - began)} s) ...")+ announced = time.monotonic()+ time.sleep(2)+ return False+++def start_engine(ref: str, target: Target, log_path: Path, log, timeout_s: float):+ """Start `strata serve --profile REF` as a child of this run. -> the process."""+ root = Path(__file__).resolve().parents[1]+ command = [sys.executable, str(root / "strata.py"), "serve", "--profile", ref, "--quiet"]+ handle = open(log_path, "wb")+ kwargs = {}+ if os.name == "nt":+ kwargs["creationflags"] = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)+ else:+ kwargs["start_new_session"] = True+ proc = subprocess.Popen(command, cwd=str(root), stdout=handle, stderr=subprocess.STDOUT, **kwargs)+ proc._strata_log = handle # closed by stop_engine+ log("starting the engine (the model load is not part of the benchmark time) ...")+ if not _start_wait(target, proc, timeout_s, log, log_path):+ stop_engine(proc)+ tail = ""+ try:+ tail = log_path.read_text(encoding="utf-8", errors="replace")[-600:]+ except OSError:+ pass+ raise ProfileError("the engine did not become ready" + (": " + tail.strip().splitlines()[-1] if tail.strip() else ""), 6)+ return proc+++def stop_engine(proc) -> None:+ """Stop only the process this run started."""+ if proc is None:+ return+ try:+ if proc.poll() is None:+ if os.name == "nt":+ subprocess.run(["taskkill", "/PID", str(proc.pid), "/T", "/F"], capture_output=True, timeout=60)+ else:+ import signal+ try:+ os.killpg(proc.pid, signal.SIGTERM)+ except OSError:+ proc.terminate()+ try:+ proc.wait(90)+ except subprocess.TimeoutExpired:+ try:+ os.killpg(proc.pid, signal.SIGKILL)+ except OSError:+ proc.kill()+ try:+ proc.wait(30)+ except subprocess.TimeoutExpired:+ pass+ finally:+ handle = getattr(proc, "_strata_log", None)+ if handle is not None:+ handle.close()+++def slug(text: str, limit: int = 40) -> str:+ cleaned = re.sub(r"(?i)\b(nvidia|geforce|amd|radeon|intel|graphics|corporation)\b", " ", text or "")+ return re.sub(r"[^a-z0-9]+", "-", cleaned.lower()).strip("-")[:limit].strip("-") or "unknown"+++def run(target: Target, out: Path | None, *, out_root: Path | None = None, optional_concurrency: bool = False,+ corpus: Corpus | None = None, declared: dict | None = None, plan: dict | None = None,+ engine_log: Path | None = None, command: str = "strata bench run --suite standard-v1",+ log=lambda message: None, timeout_s: float = REQUEST_TIMEOUT_S, started_engine: bool = False,+ hash_model: bool = True) -> tuple[dict, Path]:+ """Run the protocol against `target` and write the report into the new directory `out` (or a new folder named+ <date>-standard-<gpu>-<run id> under `out_root`). -> (the result, the folder)."""+ declared = {"track": "custom", "track_notes": "no notes given by the operator", **(declared or {})}+ if declared["track"] not in ("default", "custom"):+ raise ProfileError("--track must be default or custom", 2)+ corpus = corpus or load_corpus()+ if out is not None and (out.exists() or out.is_symlink()):+ raise ProfileError("benchmark output directory already exists", 2)+ if out is None and out_root is None:+ raise ProfileError("benchmark needs an output directory", 2)+ health = target.get("/health", timeout=10)+ if not health or health.get("service") != "strata":+ raise ProfileError("no Strata server answers at that address", 5)+ if health.get("loaded") is not True:+ raise ProfileError("the server has no model loaded; the benchmark needs a loaded model", 5)+ props = target.get("/props", timeout=15)+ status = target.get("/v1/status", timeout=15) or {}+ if props is None:+ raise ProfileError("the server did not answer /props (an API key may be needed: STRATA_API_KEY)", 5)+ redact = facts.redactor(target.key or "")+ why = facts.Why()+ reasons_nonconforming: list[str] = []+ if not target.loopback:+ reasons_nonconforming.append("the server is not on loopback: standard latency is measured over loopback")++ served_model = props.get("model_alias") or status.get("model")+ prior = ((status.get("activity") or {}).get("requests"))+ # the server's own view of its hardware and build+ machine = target.get("/metrics", timeout=20) or {}+ static = machine.get("hardware_static") or {}+ served_gpu = ((status.get("machine") or {}).get("gpu") or {}).get("name") or static.get("gpu_name")+ engine_version = ((machine.get("engine") or {}).get("version") or status.get("engine")+ or (str(props.get("build_info", "")).replace("Strata ", "") or None))++ plan = plan or {}+ options = facts.parse_options(plan["engine_argv"]) if plan.get("engine_argv") else None+ model_paths = [str(props.get("model_path") or "")] + [str(v) for v in (plan.get("engine_argv") or [])]+ files = facts.model_files_from([p for p in model_paths if p]) if hash_model else []+ model_dir = files[0].parent if files else None+ selected = facts.gpu_selection(plan.get("config"), plan.get("engine_argv"))+ template_text = props.get("chat_template") if isinstance(props.get("chat_template"), str) else None++ hardware_block, machine_facts = facts.hardware(why, selected=selected, served_gpu=served_gpu, model_dir=model_dir,+ declared_background=list(declared.get("background") or []))+ software_block = facts.software(why, machine_facts, engine_version=engine_version,+ build_options=declared.get("build_options"))+ model_block = facts.model(why, props=props, status=status, files=files, declared=declared, log=log)+ template_ref = {"path": "provenance/chat_template.jinja", "sha256": facts.sha256_text(template_text)} if template_text else None+ configuration_block, other_options = facts.configuration(+ why, options=options, env=(plan.get("config") or {}).get("env") if isinstance(plan.get("config"), dict) else None,+ props=props, declared=declared,+ sampling={"sampling": SAMPLING, "chat_template_kwargs": TEMPLATE_KWARGS}, template_ref=template_ref)++ if software_block["strata_commit"] is None:+ reasons_nonconforming.append("the Strata commit is unknown (not a git checkout and no COMMIT file)")+ if not model_block["files"]:+ reasons_nonconforming.append("no model file could be hashed")+ if template_ref is None:+ reasons_nonconforming.append("the effective chat template could not be read from the server")+ if props.get("is_sleeping"):+ reasons_nonconforming.append("the server reported its model as sleeping")++ run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid.uuid4().hex[:8]+ if out is None:+ names = "-plus-".join(slug(g["name"], 30) for g in hardware_block["gpus"]) or "cpu"+ out = out_root / (datetime.now(timezone.utc).strftime("%Y-%m-%d") + "-standard-" + names[:60].strip("-") + "-" + run_id[-8:])+ out_root.mkdir(parents=True, exist_ok=True)+ _private_mkdir(out, parents=False, exist_ok=False)+ started_at = _now()+ collector = Collector(target, corpus, served_model, log, timeout_s)+ by_phase = {p: [s for s in corpus.specs if s["phase"] == p] for p in PHASES}+ required_start = time.monotonic()+ interrupted = False+ try:+ for phase in ("warmup", "performance", "sanity"):+ for spec in by_phase[phase]:+ collector.one(spec)+ required_duration = round(time.monotonic() - required_start, 3)+ optional_duration = None+ rounds: list[dict] = []+ if optional_concurrency:+ optional_start = time.monotonic()+ specs = by_phase["concurrency"]+ for round_no in (1, 2, 3):+ group = [s for s in specs if s["round"] == round_no]+ if collector.abort:+ for spec in group:+ collector.results[spec["id"]] = skipped_record(spec, collector.abort)+ rounds.append({"round": round_no, "request_ids": [s["id"] for s in group], "wall_s": None,+ "output_tokens": None, "aggregate_tps": None, "failed": 0})+ continue+ log(f"concurrency round {round_no}/3: four requests released together")+ rounds.append(collector.concurrent_round(group))+ if not collector.abort and any(collector.results[s["id"]][0]["status"] == "failed" for s in group) \+ and not collector.alive():+ collector.abort = "the server stopped answering during the concurrency stage"+ optional_duration = round(time.monotonic() - optional_start, 3)+ except KeyboardInterrupt:+ interrupted = True+ required_duration = round(time.monotonic() - required_start, 3)+ optional_duration, rounds = None, []+ collector.abort = "the operator cancelled the run"+ for spec in corpus.specs:+ if spec["id"] not in collector.results and (spec["phase"] != "concurrency" or optional_concurrency):+ collector.results[spec["id"]] = skipped_record(spec, collector.abort)+ for spec in corpus.specs: # a stage that never started is skipped, not missing+ if spec["id"] not in collector.results and (spec["phase"] != "concurrency" or optional_concurrency):+ collector.results[spec["id"]] = skipped_record(spec, collector.abort or "not run")++ selected_specs = [s for s in corpus.specs if s["id"] in collector.results]+ records = [collector.results[s["id"]][0] for s in selected_specs]+ evidence = [collector.results[s["id"]][1] for s in selected_specs]+ extras = {s["id"]: collector.results[s["id"]][2] for s in selected_specs}++ for r in records:+ if r["phase"] == "performance" and extras[r["id"]]["unexpected_payload_reuse"]:+ reasons_nonconforming.append(f"unexpected reuse of the user payload in {r['id']}")+ if extras[r["id"]]["length_outside_tolerance"]:+ reasons_nonconforming.append(f"{r['id']} measured {r['tokens']['input_tokens']} prompt tokens, outside 2% of its target")+ if r["status"] == "failed" and r["error"]["kind"] in ("http_400", "http_422"):+ reasons_nonconforming.append(f"the server refused the protocol's required settings in {r['id']} ({r['error']['kind']})")+ skipped_required = any(r["status"] == "skipped" for r in records if r["phase"] != "concurrency")+ skipped_optional = any(r["status"] == "skipped" for r in records if r["phase"] == "concurrency")+ run_status = ("nonconforming" if reasons_nonconforming else+ "partial" if skipped_required or interrupted else "complete")+ reasons_nonconforming = list(dict.fromkeys(reasons_nonconforming))++ # the artifacts first, so the report can hash their final bytes+ artifacts: list[dict] = []++ def put(kind: str, relative: str, data: bytes) -> None:+ path = out / relative+ path.parent.mkdir(parents=True, exist_ok=True)+ _write_new(path, data)+ artifacts.append({"kind": kind, "path": relative, "sha256": hashlib.sha256(data).hexdigest()})+ lines = [json.dumps(facts.redact_object(row, redact), ensure_ascii=False, allow_nan=False) for row in evidence]+ put("outputs", "outputs.jsonl", ("\n".join(lines) + "\n").encode("utf-8"))+ config_doc = facts.redact_object({"engine_options": options, "other_engine_options": other_options,+ "server_defaults": (props.get("default_generation_settings") or {}).get("params"),+ "total_slots": props.get("total_slots"), "context": configuration_block["context_tokens"],+ "source": "effective engine arguments of the profile" if options is not None+ else "the server was already running; its engine options are not exposed"}, redact)+ put("config", "config.json", (json.dumps(config_doc, ensure_ascii=False, indent=2, allow_nan=False) + "\n").encode("utf-8"))+ if engine_log is not None and engine_log.is_file():+ text = engine_log.read_text(encoding="utf-8", errors="replace")[-2_000_000:]+ put("engine_log", "engine.log", redact(text).encode("utf-8"))+ if template_text:+ put("provenance", "provenance/chat_template.jinja", template_text.encode("utf-8"))+ put("provenance", "provenance/corpus-manifest.sha256", corpus.manifest)+ put("provenance", "provenance/corpus-build.json", (corpus.directory / "build.json").read_bytes())++ summaries = summaries_of(records, extras, "performance", PERFORMANCE_CASES, 1)+ if optional_concurrency:+ summaries += summaries_of(records, extras, "concurrency", ("short-burst",), 4)+ doc = {+ "format": FORMAT, "schema_version": SCHEMA_VERSION,+ "protocol": {"id": PROTOCOL_ID, "git_commit": software_block["runner_commit"], "profile": PROFILE,+ "corpus_sha256": corpus.sha256, "model_card_revision": MODEL_CARD_REVISION},+ "run": {"id": run_id, "started_at": started_at, "status": run_status, "required_duration_s": required_duration,+ "optional_duration_s": optional_duration, "optional_concurrency": bool(optional_concurrency),+ "command": command},+ "hardware": hardware_block, "software": software_block, "model": model_block,+ "configuration": configuration_block,+ "request_counts": {p: counts_of(records, p, PLANNED[p] if p != "concurrency" or optional_concurrency else 0)+ for p in PHASES},+ "requests": records, "summaries": summaries, "concurrency_rounds": rounds if optional_concurrency else [],+ "quality": quality_of(records), "artifacts": artifacts, "missing": {},+ "extensions": {"suite": SUITE, "runner": "strata_cli.bench_standard",+ "status_reasons": reasons_nonconforming or (+ [collector.abort] if collector.abort else []),+ "started_engine": started_engine,+ "server_requests_before_run": prior if isinstance(prior, int) else None,+ "other_engine_options": other_options,+ "request_checks": {rid: v for rid, v in extras.items()},+ "optional_stage_skipped": skipped_optional,+ "api_key_sent": bool(target.key)}}+ if doc["run"]["optional_duration_s"] is None:+ why.add("/run/optional_duration_s", "the optional concurrency stage was not selected" if not optional_concurrency+ else "the optional concurrency stage did not run")+ for pointer_text in scan_nulls(doc):+ doc["missing"][pointer_text] = why.get(pointer_text) or default_reason(doc, pointer_text)+ problems = validate(doc)+ if problems:+ raise ProfileError("the benchmark report failed its own validation: " + problems[0], 6)+ encoded = (json.dumps(doc, ensure_ascii=False, indent=2, allow_nan=False) + "\n").encode("utf-8")+ _write_new(out / "results.json", encoded)+ _write_new(out / "README.md", report_markdown(doc, redact).encode("utf-8"))+ return doc, out+++# ------------------------------------------------------------------------------------------------------------ validation+def validate(doc: dict) -> list[str]:+ """A strict check of the result contract: shape, types, count identities, null reasons and summaries rebuilt from the requests."""+ bad: list[str] = []++ def need(condition: bool, message: str) -> None:+ if not condition:+ bad.append(message)++ if not isinstance(doc, dict):+ return ["the result is not an object"]+ need(doc.get("format") == FORMAT and doc.get("schema_version") == SCHEMA_VERSION, "unknown format or schema version")+ required = ("protocol", "run", "hardware", "software", "model", "configuration", "request_counts", "requests", "summaries",+ "concurrency_rounds", "quality", "artifacts", "missing")+ for key in required:+ need(key in doc, f"missing top-level field {key}")+ if bad:+ return bad+ allowed = {"format", "schema_version", *required, "extensions"}+ need(set(doc) <= allowed, "unknown top-level fields must be under extensions")+ protocol, run_block = doc["protocol"], doc["run"]+ need(all(k in protocol for k in ("id", "git_commit", "profile", "corpus_sha256", "model_card_revision")), "protocol fields")+ need(isinstance(protocol.get("corpus_sha256"), str) and bool(HEX64.match(protocol["corpus_sha256"])), "corpus_sha256 is not a SHA-256")+ for field in ("id", "started_at", "status", "required_duration_s", "optional_duration_s", "optional_concurrency", "command"):+ need(field in run_block, f"run.{field} missing")+ need(run_block.get("status") in ("complete", "partial", "nonconforming"), "run.status")+ need(bool(re.fullmatch(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ", str(run_block.get("started_at")))), "run.started_at is not UTC ISO 8601")+ for field in ("gpus", "cpu", "ram_bytes", "storage", "background_workloads"):+ need(field in doc["hardware"], f"hardware.{field} missing")+ for gpu in doc["hardware"].get("gpus") or []:+ need(gpu.get("memory_kind") in ("dedicated", "shared"), "gpu memory_kind")+ need(isinstance(gpu.get("device_index"), int), "gpu device_index")+ for field in ("os", "drivers", "runtime", "strata_commit", "engine_version", "build_kind", "runner_commit", "build_options"):+ need(field in doc["software"], f"software.{field} missing")+ need(doc["software"].get("build_kind") in ("release", "source"), "software.build_kind")+ for field in ("id", "revision", "source_revision", "quantization", "files", "tokenizer_revision", "custom_pack"):+ need(field in doc["model"], f"model.{field} missing")+ for entry in doc["model"].get("files") or []:+ need(bool(HEX64.match(str(entry.get("sha256")))) and "/" not in str(entry.get("path")) and "\\" not in str(entry.get("path")),+ "model file entry")+ cfg = doc["configuration"]+ for field in ("track", "track_notes", "context_tokens", "server_capacity", "kv_cache", "expert_cache", "offloading", "prefill",+ "mtp", "calibration", "experimental_options", "template", "sampling", "chat_template_kwargs"):+ need(field in cfg, f"configuration.{field} missing")+ need(cfg.get("track") in ("default", "custom"), "configuration.track")+ need(all(k in cfg.get("sampling", {}) for k in SAMPLING), "configuration.sampling")++ records = doc["requests"]+ ids = [r.get("id") for r in records]+ need(len(set(ids)) == len(ids), "duplicate request ids")+ need(len(records) in (22, 34), "the requests array has neither 22 nor 34 records")+ sanity_ids = {f"sanity-{n}" for n in ("I1", "I2", "I3", "I4", "R1", "R2", "R3", "R4", "T1", "T2", "T3", "T4")}+ need(sanity_ids <= set(ids), "a sanity case is missing")+ for r in records:+ need(r.get("status") in ("completed", "truncated", "failed", "skipped"), f"{r.get('id')}: status")+ need(r.get("phase") in PHASES, f"{r.get('id')}: phase")+ if r.get("status") in ("completed", "truncated"):+ need(r.get("error") is None, f"{r.get('id')}: error on a finished request")+ else:+ need(isinstance(r.get("error"), dict) and "kind" in r["error"] and "message" in r["error"], f"{r.get('id')}: error object")+ for group, keys in (("tokens", ("input_tokens", "cached_tokens", "read_tokens", "output_tokens", "decode_tokens")),+ ("timings", ("prompt_s", "decode_s", "ttft_s", "first_answer_s", "total_s")),+ ("metrics", ("prompt_tps", "decode_tps")),+ ("sources", ("input_tokens", "cached_tokens", "read_tokens", "output_tokens", "decode_tokens",+ "engine_timings", "client_timings"))):+ need(isinstance(r.get(group), dict) and all(k in r[group] for k in keys), f"{r.get('id')}: {group}")+ for k, v in (r.get("tokens") or {}).items():+ need(v is None or (isinstance(v, int) and not isinstance(v, bool) and v >= 0), f"{r.get('id')}: tokens.{k}")+ for group in ("timings", "metrics"):+ for k, v in (r.get(group) or {}).items():+ need(v is None or (isinstance(v, (int, float)) and not isinstance(v, bool) and math.isfinite(v) and v >= 0),+ f"{r.get('id')}: {group}.{k}")+ if r.get("phase") == "sanity":+ need(isinstance(r.get("evaluation"), dict) and isinstance(r["evaluation"].get("passed"), bool), f"{r.get('id')}: evaluation")+ else:+ need(r.get("evaluation") is None, f"{r.get('id')}: evaluation outside sanity")+ need((r.get("round") is not None) == (r.get("phase") == "concurrency"), f"{r.get('id')}: round")+ counts = doc["request_counts"]+ for phase in PHASES:+ rows = [r for r in records if r.get("phase") == phase]+ c = counts.get(phase) or {}+ need(c.get("attempted") == c.get("completed", 0) + c.get("truncated", 0) + c.get("failed", 0), f"{phase}: attempted identity")+ need(c.get("planned") == c.get("attempted", 0) + c.get("skipped", 0), f"{phase}: planned identity")+ need(len(rows) == c.get("planned"), f"{phase}: record count")+ need(doc["quality"] == quality_of(records) if len(records) >= 22 else False, "quality does not match the sanity verdicts")+ need(isinstance(run_block.get("optional_concurrency"), bool), "run.optional_concurrency")+ need((counts["concurrency"]["planned"] == 12) == bool(run_block.get("optional_concurrency")), "concurrency count")++ extras = {k: v for k, v in (doc.get("extensions") or {}).get("request_checks", {}).items()}+ if extras:+ rebuilt = summaries_of(records, extras, "performance", PERFORMANCE_CASES, 1)+ if run_block.get("optional_concurrency"):+ rebuilt += summaries_of(records, extras, "concurrency", ("short-burst",), 4)+ need(rebuilt == doc["summaries"], "summaries cannot be rebuilt from the request records")+ for row in doc["summaries"]:+ need(row["n"] == len(row["included_request_ids"]), "summary n")+ if row["n"] == 0:+ need(row["median"] is None and row["min"] is None and row["max"] is None, "empty summary must be null")+ for artifact in doc["artifacts"]:+ need(artifact.get("kind") in ("config", "engine_log", "outputs", "provenance") and bool(HEX64.match(str(artifact.get("sha256"))))+ and not str(artifact.get("path")).startswith(("/", "\\")) and ".." not in Path(str(artifact.get("path"))).parts+ and not re.match(r"^[A-Za-z]:", str(artifact.get("path"))), "artifact entry")+ nulls = set(scan_nulls(doc))+ need(nulls == set(doc["missing"]), "missing must list exactly the null values: "+ f"{sorted(nulls ^ set(doc['missing']))[:3]}")+ need(all(isinstance(v, str) and v for v in doc["missing"].values()), "every missing entry needs a reason")+ try:+ json.dumps(doc, allow_nan=False)+ except ValueError:+ bad.append("the result contains NaN or infinity")+ return bad+++# ------------------------------------------------------------------------------------------------------------ reading it out+def _span(row: dict | None, digits: int = 1) -> str:+ if not row or row["n"] == 0:+ return "not measured"+ return f"{row['median']:.{digits}f} ({row['min']:.{digits}f}-{row['max']:.{digits}f}) n={row['n']}"+++def _find(doc: dict, phase: str, case: str, metric: str) -> dict | None:+ return next((s for s in doc["summaries"] if s["phase"] == phase and s["case_id"] == case and s["metric"] == metric), None)+++def _minutes(seconds: float | None) -> str:+ if seconds is None:+ return "not measured"+ return f"{int(seconds // 60)} min {int(seconds % 60)} s"+++def table_lines(doc: dict, phase: str, cases: tuple[str, ...]) -> list[str]:+ head = ["Case", "Prompt tok/s", "Decode tok/s", "TTFT s", "Total s"]+ rows = [head]+ for case in cases:+ rows.append([case, _span(_find(doc, phase, case, "prompt_tps")), _span(_find(doc, phase, case, "decode_tps")),+ _span(_find(doc, phase, case, "ttft_s"), 2), _span(_find(doc, phase, case, "total_s"), 1)])+ widths = [max(len(r[i]) for r in rows) for i in range(len(head))]+ return [" " + " ".join(cell.ljust(widths[i]) for i, cell in enumerate(row)).rstrip() for row in rows]+++def format_plain(doc: dict) -> str:+ """The summary printed at the end of a run: what ran, how fast, what passed, what to be careful about."""+ hw, sw, run_block = doc["hardware"], doc["software"], doc["run"]+ gpu = ", ".join(g["name"] for g in hw["gpus"]) or "no GPU found"+ counts = doc["request_counts"]+ done = sum(counts[p]["completed"] + counts[p]["truncated"] for p in ("performance", "sanity"))+ quality = doc["quality"]+ lines = [f"Strata standard benchmark ({doc['protocol']['id']})",+ f"Status: {run_block['status']}. {done} of 21 required requests finished, "+ f"{sum(counts[p]['failed'] for p in PHASES)} failed, {sum(counts[p]['skipped'] for p in PHASES)} skipped. "+ f"Time after the model loaded: {_minutes(run_block['required_duration_s'])}.",+ f"Machine: {gpu}; {hw['cpu'] or 'CPU not read'}; "+ f"{round(hw['ram_bytes'] / 2 ** 30) if hw['ram_bytes'] else 'unknown'} GB RAM; {sw['os']['name']} {sw['os']['version']}.",+ f"Model: {doc['model']['id'] or 'unknown'}, {doc['model']['quantization'] or 'quantization not read'}, "+ f"context {doc['configuration']['context_tokens'] or 'unknown'}; engine {sw['engine_version'] or 'unknown'}; "+ f"track {doc['configuration']['track']}.", "",+ "Speed (median, min-max, n; the engine's own prompt and decode timings, the client's first-token time):"]+ lines += table_lines(doc, "performance", PERFORMANCE_CASES)+ lines += ["", f"Sanity checks: {quality['passed']} of 12 passed "+ f"(instruction {quality['categories']['instruction']['passed']}/4, retrieval "+ f"{quality['categories']['retrieval']['passed']}/4, tool calls {quality['categories']['tool_call']['passed']}/4)."]+ failed = [r["evaluation"] and f"{r['case_id']}: {r['evaluation']['reason']}" for r in doc["requests"]+ if r["phase"] == "sanity" and not r["evaluation"]["passed"]]+ lines += [" Not passed: " + "; ".join(failed)] if failed else []+ if doc["run"]["optional_concurrency"]:+ rounds = [r["aggregate_tps"] for r in doc["concurrency_rounds"] if r["aggregate_tps"] is not None]+ lines += ["", "Four requests at once (server capacity "+ f"{doc['configuration']['server_capacity'] or 'unknown'}): "+ + (", ".join(f"{v:.1f}" for v in rounds) + " tok/s in total per round" if rounds else "no round finished")]+ reasons = doc["extensions"]["status_reasons"]+ if run_block["status"] != "complete" and reasons:+ lines += ["", "Why this is not a complete standard result:"] + [f" - {r}" for r in reasons]+ failures = [f"{r['id']}: {r['error']['kind']}" for r in doc["requests"] if r["status"] == "failed"]+ if failures:+ lines += ["", "Failed requests: " + "; ".join(failures)]+ return "\n".join(lines)+++def report_markdown(doc: dict, redact=lambda text: text) -> str:+ hw, sw, run_block, model, cfg = doc["hardware"], doc["software"], doc["run"], doc["model"], doc["configuration"]+ counts = doc["request_counts"]+ quality = doc["quality"]+ gpu = "; ".join(("%s (%.1f GiB %s)" % (g["name"], g["memory_bytes"] / 2 ** 30, g["memory_kind"])) if g["memory_bytes"]+ else g["name"] for g in hw["gpus"]) or "not measured"+ nm = "not measured"+ ram = "%.1f GiB" % (hw["ram_bytes"] / 2 ** 30) if hw["ram_bytes"] else nm+ storage = ", ".join("%s %s (%s)" % (s["kind"], s["model"], s["bus"]) for s in hw["storage"]) if hw["storage"] else nm+ drivers = ", ".join("%s %s" % (d["name"], d["version"]) for d in sw["drivers"]) if sw["drivers"] else nm+ runtime = ", ".join("%s %s" % (d["name"], d["version"]) for d in sw["runtime"]) if sw["runtime"] else nm+ lines = [+ "# Strata standard benchmark: %s" % (", ".join(g["name"] for g in hw["gpus"]) or "unknown GPU"), "",+ "Protocol `%s` (draft), run `%s`, started %s. **Status: %s.**" % (doc["protocol"]["id"], run_block["id"],+ run_block["started_at"], run_block["status"]), "",+ "Written by `strata bench run`. The machine-readable result is [results.json](results.json); every request, its raw usage "+ "and timings and the client event offsets are in [outputs.jsonl](outputs.jsonl).", ""]+ reasons = doc["extensions"]["status_reasons"]+ if run_block["status"] != "complete":+ lines += ["This run is not a complete standard result:", ""] + ["- " + r for r in reasons] + [""]+ lines += [+ "## Hardware and software", "",+ "- GPU: " + gpu,+ "- CPU: %s; RAM: %s" % (hw["cpu"] or nm, ram),+ "- Storage of the model files: " + storage,+ "- OS: %s %s; drivers: %s; runtime: %s" % (sw["os"]["name"], sw["os"]["version"], drivers, runtime),+ "- Strata commit: `%s`; engine %s; %s build" % (sw["strata_commit"] or nm, sw["engine_version"] or nm, sw["build_kind"]),+ "- Background work: " + ("; ".join(hw["background_workloads"]) if hw["background_workloads"] else nm), "",+ "## Model and configuration", "",+ f"- Model: {model['id'] or nm}; quantization {model['quantization'] or nm}; revision {model['revision'] or nm}; "+ f"source revision {model['source_revision'] or nm}",+ f"- Files: " + ("; ".join(f"`{f['path']}` `{f['sha256'][:16]}...`" for f in model['files']) if model['files'] else nm),+ f"- Track: **{cfg['track']}** ({cfg['track_notes']})",+ f"- Context {cfg['context_tokens'] or nm} tokens; server capacity {cfg['server_capacity'] or nm} slot(s)",+ f"- Sampling: " + ", ".join(f"{k} {v}" for k, v in cfg["sampling"].items()) + "; thinking on, preserved",+ f"- Engine options: `config.json`; chat template: `provenance/chat_template.jinja`", "",+ "## Method", "",+ f"The frozen requests of `bench/prompts/standard-v1` (corpus `{doc['protocol']['corpus_sha256'][:16]}...`) were sent over "+ "loopback HTTP, streaming, one at a time, in this order: 1 warm-up (excluded), 3 repetitions of the short, 4K and 16K prompts, "+ "and the twelve sanity checks. Prompt and decode speed are the engine's own `timings`; time to first token is the client's "+ "monotonic clock from dispatch to the first nonempty reasoning, answer or tool-call delta. No request was retried. "+ f"Time after the model loaded: {_minutes(run_block['required_duration_s'])}.", "",+ "## Results", ""]+ lines += ["| Case | Prompt tok/s | Decode tok/s | TTFT s | First answer s | Total s |", "| --- | --- | --- | --- | --- | --- |"]+ for case in PERFORMANCE_CASES:+ lines.append(f"| {case} | " + " | ".join(+ _span(_find(doc, "performance", case, m), d) for m, d in (("prompt_tps", 1), ("decode_tps", 1), ("ttft_s", 2),+ ("first_answer_s", 2), ("total_s", 1))) + " |")+ lines += ["", "Cells are median (min-max) n=samples.", "",+ "| Phase | Planned | Completed | Truncated | Failed | Skipped |", "| --- | ---: | ---: | ---: | ---: | ---: |"]+ for phase in PHASES:+ c = counts[phase]+ lines.append(f"| {phase} | {c['planned']} | {c['completed']} | {c['truncated']} | {c['failed']} | {c['skipped']} |")+ lines += ["", f"Sanity checks: **{quality['passed']} of 12**.", "", "| Case | Result | Reason |", "| --- | --- | --- |"]+ for r in doc["requests"]:+ if r["phase"] == "sanity":+ lines.append(f"| {r['case_id']} | {'pass' if r['evaluation']['passed'] else 'fail'} | {r['evaluation']['reason']} |")+ if run_block["optional_concurrency"]:+ lines += ["", "Concurrency rounds (four requests released together):", "",+ "| Round | Wall s | Output tokens | Aggregate tok/s | Failed |", "| ---: | ---: | ---: | ---: | ---: |"]+ for r in doc["concurrency_rounds"]:+ lines.append(f"| {r['round']} | {r['wall_s']} | {r['output_tokens']} | {r['aggregate_tps']} | {r['failed']} |")+ failures = [r for r in doc["requests"] if r["status"] == "failed"]+ lines += ["", "## Failures, exclusions and limitations", ""]+ lines += [f"- {r['id']} failed: {r['error']['kind']} ({r['error']['message']})" for r in failures] or ["- No request failed."]+ excluded = {(e["request_id"], e["reason"]) for s in doc["summaries"] for e in s["excluded"]}+ lines += [f"- Excluded from speed summaries: {rid}: {reason}" for rid, reason in sorted(excluded)]+ lines += ["- Prompt-prefix reuse is the shared template prefix only; the cache was not emptied between requests, and the expert "+ "cache stays warm and evolves. These are not cold-model measurements.",+ "- Output caps include reasoning; a request that reaches its cap is `truncated` and fails its sanity check.",+ "- The runner cannot see other programs on the machine, the repeat-stop and loop-recovery switches of the server, or "+ "the quantization repository's revision; see `missing` in results.json for every value it could not read.",+ f"- The server had answered {doc['extensions']['server_requests_before_run']} request(s) before this run."+ if doc["extensions"]["server_requests_before_run"] else+ "- The runner could not tell whether the server was freshly loaded.", ""]+ return redact("\n".join(lines))+++def load(path: Path) -> dict:+ """Read a saved standard result and validate it. A result in another format or schema version is refused, never merged."""+ target = path / "results.json" if path.is_dir() else path+ try:+ if target.stat().st_size > 64 << 20:+ raise ProfileError("benchmark result is larger than 64 MiB", 5)+ doc = json.loads(target.read_text(encoding="utf-8"))+ except (OSError, UnicodeError, ValueError):+ raise ProfileError("cannot read the benchmark result", 5) from None+ problems = validate(doc)+ if problems:+ raise ProfileError("not a valid standard benchmark result: " + problems[0], 5)+ return doc+++def load_if_standard(path: Path) -> dict | None:+ """The validated result at `path` when it is a standard benchmark result; None when it is another kind of report."""+ target = path / "results.json" if path.is_dir() else path+ try:+ head = json.loads(target.read_text(encoding="utf-8"))+ except (OSError, UnicodeError, ValueError):+ return None+ if not isinstance(head, dict) or head.get("format") != FORMAT:+ return None+ return load(path)diff --git a/strata_cli/cli.py b/strata_cli/cli.pyindex 276d1f72f..caf5c960b 100644--- a/strata_cli/cli.py+++ b/strata_cli/cli.py@@ -278,24 +278,48 @@ def parser() -> argparse.ArgumentParser:help="require the candidate run-config bytes shown in a restart recipe plan")bench = sp.add_parser("bench", help="explicit serving benchmark with local JSON evidence")bench_sp = bench.add_subparsers(dest="bench_command", required=True)- bench_run = bench_sp.add_parser("run", help="measure streaming requests and sample serving telemetry")+ bench_run = bench_sp.add_parser("run", help="the standard benchmark (docs/BENCHMARK_PROTOCOL.md), or a serving suite")bench_source = bench_run.add_mutually_exclusive_group(required=True)bench_source.add_argument("--profile", help="saved serving profile or installed run config")bench_source.add_argument("--url", help="server root; remote connections require HTTPS")- bench_run.add_argument("--suite", choices=("serving-smoke", "parallel-mixed-v1", "parallel-overload-v1",+ bench_run.add_argument("--suite", choices=("standard-v1", "serving-smoke", "parallel-mixed-v1", "parallel-overload-v1","parallel-cancel-v1", "parallel-prefill-cancel-v1"),- default="serving-smoke")- bench_run.add_argument("--concurrency", type=int, default=4,- help="simultaneous requests (1..16, or 1..64 for parallel-overload-v1)")- bench_run.add_argument("--repeat", type=int, default=1, help="repeat measured phases (1..5)")- bench_run.add_argument("--condition", help="operator label for this run, e.g. baseline or monitor-default")+ default="standard-v1",+ help="standard-v1 (default) follows docs/BENCHMARK_PROTOCOL.md; the others are serving suites for your own comparisons")+ bench_run.add_argument("--concurrency", type=int, default=None,+ help="serving suites: simultaneous requests (1..16, or 1..64 for parallel-overload-v1; default 4)")+ bench_run.add_argument("--repeat", type=int, default=None, help="serving suites: repeat measured phases (1..5; default 1)")+ bench_run.add_argument("--condition", help="serving suites: operator label for this run, e.g. baseline or monitor-default")bench_run.add_argument("--require-managed-match", action="store_true",- help="require the local owner host and current profile to match the serving instance")- bench_run.add_argument("--request-timeout", type=float, default=120)- bench_run.add_argument("--out", required=True, metavar="NEW_DIRECTORY")+ help="serving suites: require the local owner host and current profile to match the serving instance")+ bench_run.add_argument("--request-timeout", type=float, default=None,+ help="serving suites: seconds per request (1..600; default 120). The standard suite fixes it at 1800")+ bench_run.add_argument("--out", metavar="NEW_DIRECTORY",+ help="new output directory (required for serving suites). Standard suite: a new folder "+ "<date>-standard-<gpu>-<run id> under ./bench/results or ./bench-results")bench_run.add_argument("--credential-env", default="STRATA_API_KEY")bench_run.add_argument("--telemetry-credential-env", default=None)bench_run.add_argument("--output", choices=("human", "json"), default="human")+ standard = bench_run.add_argument_group("standard suite (docs/BENCHMARK_PROTOCOL.md)")+ standard.add_argument("--optional-concurrency", action="store_true",+ help="also run the optional stage: four short requests released together, three rounds")+ standard.add_argument("--no-start", action="store_true",+ help="with --profile: never start the engine; use only a server that is already running")+ standard.add_argument("--load-timeout", type=float, default=1800, metavar="SECONDS",+ help="with --profile: how long to wait for the engine to load the model (default 1800)")+ standard.add_argument("--track", choices=("default", "custom"), default="custom",+ help="default: an unmodified release's setup-generated settings; custom (the default here): anything else")+ standard.add_argument("--track-notes", default=None, metavar="TEXT",+ help="the release and setup choices (default track) or the changes made (custom track)")+ standard.add_argument("--model-revision", metavar="REV", help="immutable revision of the quantization repository")+ standard.add_argument("--source-revision", metavar="REV", help="revision of the original Qwen/Qwen3.8-Flash-Next weights")+ standard.add_argument("--tokenizer-revision", metavar="REV")+ standard.add_argument("--quantization", metavar="NAME", help="when it cannot be read from the model file names")+ standard.add_argument("--custom-pack", metavar="TEXT", help="describe a prepared pack or custom expert profile")+ standard.add_argument("--background", action="append", metavar="TEXT", help="a program that ran during the benchmark (repeatable)")+ standard.add_argument("--build-options", metavar="LIST", help="comma-separated build options of a source build")+ standard.add_argument("--skip-model-hash", action="store_true",+ help="do not hash the model files (the run is then marked nonconforming)")bench_cancel = bench_sp.add_parser("cancel", help="stop the exact active saved-profile benchmark")bench_cancel.add_argument("--profile", required=True, help="saved serving profile")bench_cancel.add_argument("--path", required=True, help="absolute run directory or results.json")@@ -1000,6 +1024,22 @@ def _bench(a: argparse.Namespace) -> int:if a.bench_command in ("report", "compare") and a.out and a.output != "markdown":return _error("--out requires --output markdown", 2, machine)if a.bench_command == "report":+ from strata_cli import bench_standard+ standard_report = bench_standard.load_if_standard(Path(a.path))+ if standard_report is not None:+ if a.output == "markdown":+ rendered = bench_standard.report_markdown(standard_report)+ if a.out:+ bench.write_markdown(Path(a.out), rendered)+ print(f"Markdown report: {Path(a.out).resolve()}")+ else:+ print(rendered, end="")+ return 0+ return _success({"summary": {"format": standard_report["format"], "run_id": standard_report["run"]["id"],+ "status": standard_report["run"]["status"],+ "quality": standard_report["quality"],+ "summaries": standard_report["summaries"]}}, machine=machine,+ human=bench_standard.format_plain(standard_report))report = bench.load(Path(a.path))result = bench.summary(report)if a.output == "markdown":@@ -1035,6 +1075,13 @@ def _bench(a: argparse.Namespace) -> int:if not a.credential_env.isidentifier() or (a.telemetry_credential_env andnot a.telemetry_credential_env.isidentifier()):return _error("credential options must name environment variables", 2, machine)+ if a.suite == "standard-v1":+ return _bench_standard(a, machine)+ if a.out is None:+ return _error("--out is required for the serving suites", 2, machine)+ a.concurrency = 4 if a.concurrency is None else a.concurrency+ a.repeat = 1 if a.repeat is None else a.repeat+ a.request_timeout = 120 if a.request_timeout is None else a.request_timeoutmaximum = 64 if a.suite == "parallel-overload-v1" else 16if not 1 <= a.concurrency <= maximum or not 1 <= a.repeat <= 5 or not 1 <= a.request_timeout <= 600:return _error(f"concurrency must be 1..{maximum}, repeat 1..5 and request timeout 1..600 seconds", 2,@@ -1079,6 +1126,86 @@ def _bench(a: argparse.Namespace) -> int:return 0 if report["successful"] else 6+def _bench_standard(a: argparse.Namespace, machine: bool) -> int:+ """`strata bench run` for docs/BENCHMARK_PROTOCOL.md: against a running server (--url, or a profile that is already serving),+ or by starting the profile's engine for the run and stopping it afterwards."""+ from strata_cli import bench_facts, bench_standard+ fixed = [name for name, value in (("--concurrency", a.concurrency), ("--repeat", a.repeat),+ ("--request-timeout", a.request_timeout), ("--condition", a.condition),+ ("--require-managed-match", a.require_managed_match or None),+ ("--telemetry-credential-env", a.telemetry_credential_env))+ if value is not None]+ if fixed:+ return _error(f"{', '.join(fixed)} do not apply to the standard suite: the protocol fixes the workload "+ "(use --suite serving-smoke for a serving benchmark)", 2, machine)+ if not a.credential_env.isidentifier():+ return _error("credential options must name environment variables", 2, machine)+ if not 1 <= a.load_timeout <= 86400:+ return _error("--load-timeout must be 1..86400 seconds", 2, machine)+ plan = None+ saved = None+ key = os.environ.get(a.credential_env)+ if a.profile:+ _, saved, _, cfg = _source(a.profile)+ plan, _ = _launch_plan(a.profile)+ plan = {**plan, "config": cfg}+ host = plan["effective"]["host"]+ host = "127.0.0.1" if host in ("0.0.0.0", "::") else host+ authority = f"[{host}]" if ":" in host and not host.startswith("[") else host+ url = f"http://{authority}:{plan['effective']['port']}"+ if not key and cfg.get("api_key"):+ from serve.server import api_key_of, key_list+ keys = key_list(api_key_of(cfg["api_key"]))+ key = keys[0] if keys else None+ else:+ url = a.url+ from strata_cli.client import server_url+ target = bench_standard.Target(server_url(url), key)+ redact = bench_facts.redactor(key or "")+ command = redact("strata " + " ".join(str(part) for part in (a.raw_argv or ["bench", "run", "--suite", "standard-v1"])))+ declared = {"track": a.track, "track_notes": a.track_notes or "no notes given by the operator",+ "model_revision": a.model_revision, "source_revision": a.source_revision,+ "tokenizer_revision": a.tokenizer_revision, "quantization": a.quantization,+ "custom_pack": a.custom_pack, "background": a.background or [],+ "build_options": [x.strip() for x in a.build_options.split(",") if x.strip()] if a.build_options else None}+ out = Path(a.out) if a.out else None+ out_root = None if out else (Path.cwd() / "bench" / "results" if (Path.cwd() / "bench" / "results").is_dir()+ else Path.cwd() / "bench-results")+ from strata_control.benchmarking import benchmark_lease+ import shutil+ import tempfile+ lease = benchmark_lease(saved["id"], out) if saved else contextlib.nullcontext()+ proc, scratch, engine_log = None, None, None+ try:+ with lease, contextlib.redirect_stdout(sys.stderr):+ def log(message: str) -> None:+ print(message, file=sys.stderr, flush=True)+ if a.profile and target.get("/health", timeout=5) is None:+ if a.no_start:+ raise ProfileError("no server is answering at the profile's address and --no-start was given", 5)+ scratch = Path(tempfile.mkdtemp(prefix="strata-bench-"))+ engine_log = scratch / "engine.log"+ proc = bench_standard.start_engine(a.profile, target, engine_log, log, a.load_timeout)+ elif a.profile:+ log("using the server that is already running at this profile's address; it will not be stopped")+ doc, out_dir = bench_standard.run(target, out, out_root=out_root, optional_concurrency=a.optional_concurrency,+ declared=declared, plan=plan, engine_log=engine_log, command=command, log=log,+ started_engine=proc is not None, hash_model=not a.skip_model_hash)+ finally:+ bench_standard.stop_engine(proc)+ if scratch is not None:+ shutil.rmtree(scratch, ignore_errors=True)+ ok = doc["run"]["status"] == "complete"+ path = out_dir / "results.json"+ if machine:+ print(json.dumps({"schema_version": 1, "ok": ok, "path": str(path.resolve()), "status": doc["run"]["status"],+ "quality": doc["quality"], "summaries": doc["summaries"]}, ensure_ascii=False, allow_nan=False))+ else:+ print(bench_standard.format_plain(doc))+ print(f"\nReport: {out_dir.resolve() / 'README.md'}\nResult: {path.resolve()}")+ return 0 if ok else 6++def _runtime(a: argparse.Namespace) -> int:from strata_control import managementmachine = a.output == "json"@@ -1965,6 +2092,7 @@ def main(argv: list[str] | None = None) -> int:passap = parser()a, extras = ap.parse_known_args(argv)+ a.raw_argv = list(sys.argv[1:] if argv is None else argv)if extras:if getattr(a, "command", None) in ("launch", "connect"):from strata_cli.launch import cmd as launch_cmddiff --git a/tests/terminal/test_bench.py b/tests/terminal/test_bench.pyindex e066ea86b..b90d751d1 100644--- a/tests/terminal/test_bench.py+++ b/tests/terminal/test_bench.py@@ -394,7 +394,7 @@ class BenchTests(unittest.TestCase):out = self.root / "cli-baseline"output = io.StringIO()with mock.patch.dict(os.environ, {"STRATA_API_KEY": self.service.api_key}), redirect_stdout(output):- code = main(["bench", "run", "--url", self.url, "--condition", "baseline",+ code = main(["bench", "run", "--suite", "serving-smoke", "--url", self.url, "--condition", "baseline","--concurrency", "1", "--out", str(out), "--output", "json"])self.assertEqual(code, 0)self.assertEqual(json.loads(output.getvalue())["summary"]["condition"], "baseline")@@ -454,7 +454,7 @@ class BenchTests(unittest.TestCase):def test_managed_match_flag_requires_a_saved_profile(self):output = io.StringIO()with redirect_stdout(output):- code = main(["bench", "run", "--url", self.url, "--require-managed-match",+ code = main(["bench", "run", "--suite", "serving-smoke", "--url", self.url, "--require-managed-match","--out", str(self.root / "unused"), "--output", "json"])self.assertEqual(code, 2)self.assertFalse(json.loads(output.getvalue())["ok"])diff --git a/tests/terminal/test_bench_standard.py b/tests/terminal/test_bench_standard.pynew file mode 100644index 000000000..b966f2cb6--- /dev/null+++ b/tests/terminal/test_bench_standard.py@@ -0,0 +1,972 @@+"""The standard benchmark (docs/BENCHMARK_PROTOCOL.md) against a scripted mock server: no engine, no model weights."""++from __future__ import annotations++import contextlib+import copy+import http.client+import io+import json+import os+import re+import shutil+import tempfile+import threading+import time+import unittest+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from pathlib import Path+from unittest import mock++from strata_cli import bench, bench_facts, bench_standard as bs+from strata_cli.cli import main+from strata_control.profiles import ProfileError++COMMIT = "a" * 40+TEMPLATE = "{{ messages }} mock chat template"+++def corpus_specs() -> dict[str, dict]:+ return {spec["id"]: spec for spec in bs.load_corpus().specs}+++class MockStrata:+ """A server that speaks enough of Strata's API: /health, /props, /v1/status, /metrics and streamed chat completions."""++ def __init__(self, model_file: Path):+ self.specs = corpus_specs()+ self.model_file = model_file+ self.script: dict[str, dict] = {}+ self.bodies: list[dict] = []+ self.order: list[str] = []+ self.arrivals: list[float] = []+ self.active = 0+ self.max_active = 0+ self.dead = False+ self.lock = threading.Lock()+ self.requests_before = 0+ self.gpu_name = "Mock GPU 12 GB"+ owner = self++ class Handler(BaseHTTPRequestHandler):+ def log_message(self, *args):+ pass++ def _json(self, code, value):+ data = json.dumps(value).encode()+ self.send_response(code)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(data)))+ self.end_headers()+ self.wfile.write(data)++ def do_GET(self):+ if owner.dead and self.path == "/health":+ return self._json(503, {"error": "down"})+ if self.path == "/health":+ return self._json(200, {"service": "strata", "loaded": True})+ if self.path == "/props":+ return self._json(200, {"model_alias": "qwen3.8-flash-next", "chat_template": TEMPLATE,+ "default_generation_settings": {"n_ctx": 131072, "params": {"temperature": 0.7}},+ "total_slots": 1, "build_info": "Strata 0.1.45.3",+ "model_path": str(owner.model_file), "is_sleeping": False})+ if self.path == "/v1/status":+ return self._json(200, {"service": "strata", "model": "qwen3.8-flash-next", "engine": "0.1.45.3",+ "machine": {"gpu": {"name": owner.gpu_name}},+ "activity": {"requests": owner.requests_before}})+ if self.path == "/metrics":+ return self._json(200, {"engine": {"version": "0.1.45.3"}, "hardware_static": {"gpu_name": owner.gpu_name}})+ self._json(404, {})++ def do_POST(self):+ body = json.loads(self.rfile.read(int(self.headers.get("Content-Length", "0"))))+ rid = re.match(r"\[case ([^\]]+)\]", body["messages"][0]["content"]).group(1)+ with owner.lock:+ owner.bodies.append(body)+ owner.order.append(rid)+ owner.arrivals.append(time.monotonic())+ owner.active += 1+ owner.max_active = max(owner.max_active, owner.active)+ try:+ self.respond(rid, body)+ finally:+ with owner.lock:+ owner.active -= 1++ def respond(self, rid, body):+ spec = owner.specs[rid]+ plan = owner.script.get(rid, {})+ if plan.get("hold_for_arrivals"):+ deadline = time.monotonic() + 3+ while time.monotonic() < deadline and owner.active < plan["hold_for_arrivals"]:+ time.sleep(0.01)+ if plan.get("die"):+ owner.dead = True+ self.connection.close()+ return+ if plan.get("http"):+ return self._json(plan["http"], {"error": {"message": "refused"}})+ self.send_response(200)+ self.send_header("Content-Type", "text/event-stream")+ self.end_headers()++ def send(event):+ self.wfile.write(b"data: " + json.dumps(event).encode() + b"\n\n")+ self.wfile.flush()+ chunk = lambda delta, finish=None: {"choices": [{"index": 0, "delta": delta, "finish_reason": finish}]} # noqa: E731+ send(chunk({"role": "assistant", "content": ""}))+ self.wfile.write(b": keep-alive\n\n")+ send(chunk({"content": ""}))+ if plan.get("hang"):+ time.sleep(plan["hang"])+ return+ time.sleep(plan.get("first_delay", 0))+ send(chunk({"reasoning_content": "Thinking about it. "}))+ expected = spec.get("expected") or {}+ finish = plan.get("finish", "stop")+ if "tool_calls" in plan:+ calls = plan["tool_calls"]+ finish = plan.get("finish", "tool_calls" if calls else "stop")+ elif expected.get("kind") == "tool_call":+ calls = [{"name": expected["name"], "arguments": json.dumps(expected["arguments"])}]+ finish = plan.get("finish", "tool_calls")+ else:+ calls = []+ if "content" in plan:+ text = plan["content"]+ elif expected.get("kind") == "text":+ text = expected["value"]+ elif expected.get("kind") == "json":+ text = json.dumps(expected["value"], separators=(",", ":"))+ elif expected.get("kind") == "tool_call":+ text = ""+ else:+ text = "A hash table hashes the key."+ if text:+ send(chunk({"content": text}))+ for index, call in enumerate(calls):+ send(chunk({"tool_calls": [{"index": index, "id": f"call_{index}", "type": "function",+ "function": {"name": call["name"], "arguments": ""}}]}))+ send(chunk({"tool_calls": [{"index": index, "function": {"arguments": call["arguments"]}}]}))+ final = chunk({}, finish)+ prompt = spec["input_tokens_frozen"]+ output = plan.get("completion_tokens", 50)+ if plan.get("usage", True):+ final["usage"] = {"prompt_tokens": prompt, "completion_tokens": output, "total_tokens": prompt + output,+ "prompt_tokens_details": {"cached_tokens": plan.get("cached", 0)}}+ if plan.get("timings", True):+ cached = plan.get("cached", 0)+ read = prompt - cached+ final["timings"] = {"cache_n": cached, "prompt_n": read, "prompt_ms": float(read), "prompt_per_second": 1000.0,+ "predicted_n": plan.get("predicted_n", output), "predicted_ms": 500.0,+ "predicted_per_second": plan.get("rate", 100.0)}+ send(final)+ if not plan.get("no_done"):+ self.wfile.write(b"data: [DONE]\n\n")+ self.wfile.flush()++ self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler)+ self.httpd.daemon_threads = True+ self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True)+ self.thread.start()+ self.url = f"http://127.0.0.1:{self.httpd.server_address[1]}"++ def close(self):+ self.httpd.shutdown()+ self.httpd.server_close()+++class StandardBase(unittest.TestCase):+ def setUp(self):+ self.temp = tempfile.TemporaryDirectory()+ self.addCleanup(self.temp.cleanup)+ self.root = Path(self.temp.name)+ self.model = self.root / "Qwen3.8-Flash-Next-IQ2_XS-00001-of-00001.gguf"+ self.model.write_bytes(b"model bytes")+ self.server = MockStrata(self.model)+ self.addCleanup(self.server.close)+ for name, value in (("nvidia_gpus", ([], None, None)), ("other_gpus", ([], None)), ("cpu_name", lambda: "Mock CPU"),+ ("ram_bytes", lambda: 64 * 2 ** 30), ("storage_of", lambda path: [{"kind": "nvme", "model": "Mock SSD", "bus": "nvme"}]),+ ("git_commit", lambda folder=None: COMMIT)):+ patcher = mock.patch.object(bench_facts, name, **({"return_value": value} if isinstance(value, tuple) else {"side_effect": value}))+ patcher.start()+ self.addCleanup(patcher.stop)+ self.target = bs.Target(self.server.url, None)++ def run_standard(self, name="out", **kwargs):+ kwargs.setdefault("hash_model", True)+ doc, out = bs.run(self.target, self.root / name, **kwargs)+ return doc, out++ def record(self, doc, rid):+ return next(r for r in doc["requests"] if r["id"] == rid)++ def summary(self, doc, case, metric, phase="performance"):+ return next(s for s in doc["summaries"] if s["phase"] == phase and s["case_id"] == case and s["metric"] == metric)+++class FixtureTests(unittest.TestCase):+ def test_bundle_follows_the_protocol_tables(self):+ corpus = bs.load_corpus()+ specs = {s["id"]: s for s in corpus.specs}+ self.assertEqual([s["id"] for s in corpus.specs][:10],+ ["warmup", "performance-short-r1", "performance-prompt-4k-r1", "performance-prompt-16k-r1",+ "performance-short-r2", "performance-prompt-4k-r2", "performance-prompt-16k-r2",+ "performance-short-r3", "performance-prompt-4k-r3", "performance-prompt-16k-r3"])+ self.assertEqual([s["id"] for s in corpus.specs if s["phase"] == "sanity"],+ [f"sanity-{n}" for n in ("I1", "I2", "I3", "I4", "R1", "R2", "R3", "R4", "T1", "T2", "T3", "T4")])+ self.assertEqual(len(corpus.specs), 34)+ self.assertEqual((specs["warmup"]["seed"], specs["warmup"]["max_output_tokens"]), (1, 128))+ for repetition in (1, 2, 3):+ for case in ("short", "prompt-4k", "prompt-16k"):+ spec = specs[f"performance-{case}-r{repetition}"]+ self.assertEqual((spec["seed"], spec["max_output_tokens"], spec["concurrency"]), (repetition, 512, 1))+ for spec in corpus.specs:+ if spec["phase"] == "sanity":+ self.assertEqual((spec["seed"], spec["max_output_tokens"]), (1, 2048))+ if spec["phase"] == "concurrency":+ self.assertEqual(spec["seed"], 4 * (spec["round"] - 1) + int(spec["id"][-1]))+ self.assertEqual((spec["max_output_tokens"], spec["concurrency"]), (512, 4))+ self.assertTrue(spec["messages"][0]["content"].startswith(f"[case {spec['id']}]\n"))+ self.assertEqual(len(spec["messages"]), 1)+ self.assertEqual(specs["performance-short-r1"]["messages"][0]["content"].split("\n", 1)[1],+ "Explain how a hash table finds a value, including collisions and one small example.")++ def test_records_and_retrieval_placement(self):+ specs = corpus_specs()+ body = specs["performance-prompt-4k-r1"]["messages"][0]["content"].split("\n", 1)[1]+ lines = body.split("\n")+ self.assertEqual(lines[0], "record000001: alpha beta gamma delta epsilon zeta eta theta.")+ self.assertEqual(lines[-1], "Summarize the structure of these records and explain how to look up a record by its ID.")+ count = len(lines) - 1+ for rid, key, fraction in (("sanity-R1", "access_code = ORBIT-731", 0.10), ("sanity-R2", "storage_label = CEDAR-284", 0.50)):+ rows = specs[rid]["messages"][0]["content"].split("\n", 1)[1].split("\n")+ self.assertEqual(rows.index(key), int(fraction * count)) # inserted before record index floor(f * count)+ self.assertEqual(sum("record" not in r for r in rows), 2) # the inserted line and the question+ long_count = len(specs["performance-prompt-16k-r1"]["messages"][0]["content"].split("\n")) - 2+ rows = specs["sanity-R4"]["messages"][0]["content"].split("\n", 1)[1].split("\n")+ self.assertEqual(rows.index("north = N-124"), int(0.10 * long_count))+ self.assertEqual(rows.index("south = S-875") - 1, int(0.90 * long_count)) # one earlier insert shifts it by one line+ self.assertEqual(specs["sanity-R4"]["expected"]["value"], {"north": "N-124", "south": "S-875"})+ for rid, name, props in (("sanity-T1", "lookup_record", {"id": "string"}),+ ("sanity-T3", "convert_units", {"value": "number", "from_unit": "string", "to_unit": "string"}),+ ("sanity-T4", "get_build_status", {"project": "string", "build": "integer"})):+ function = specs[rid]["tools"][0]["function"]+ self.assertEqual(function["name"], name)+ self.assertEqual({k: v["type"] for k, v in function["parameters"]["properties"].items()}, props)+ self.assertEqual(function["parameters"]["required"], list(props))+ self.assertIs(function["parameters"]["additionalProperties"], False)+ self.assertEqual(function["description"], "Synthetic benchmark lookup; no external action is performed.")+ self.assertEqual(specs[rid]["tool_choice"], "auto")+ self.assertIsNone(specs["sanity-I1"]["tools"])++ def test_prompt_lengths_are_within_two_percent_of_their_targets(self):+ for spec in bs.load_corpus().specs:+ if spec["target_input_tokens"]:+ self.assertLessEqual(abs(spec["input_tokens_frozen"] - spec["target_input_tokens"]) / spec["target_input_tokens"], 0.02)++ def test_a_changed_or_converted_file_is_refused(self):+ with tempfile.TemporaryDirectory() as temp:+ copy_of = Path(temp) / "bundle"+ shutil.copytree(bs.corpus_dir(), copy_of)+ self.assertEqual(bs.load_corpus(copy_of).sha256, bs.load_corpus().sha256)+ target = copy_of / "requests" / "warmup.json"+ target.write_bytes(target.read_bytes().replace(b"\n", b"\r\n"))+ with self.assertRaisesRegex(ProfileError, "fixture file changed: requests/warmup.json"):+ bs.load_corpus(copy_of)+ target.write_bytes(bs.load_corpus().manifest[:0] + (bs.corpus_dir() / "requests" / "warmup.json").read_bytes())+ (copy_of / "requests" / "extra.json").write_text("{}", encoding="utf-8")+ with self.assertRaisesRegex(ProfileError, "not in the manifest"):+ bs.load_corpus(copy_of)++ def test_gitattributes_keeps_fixture_bytes(self):+ text = (Path(__file__).resolve().parents[2] / ".gitattributes").read_text(encoding="utf-8")+ self.assertIn("bench/prompts/** -text", text)+++class DocsAndPackagingTests(unittest.TestCase):+ ROOT = Path(__file__).resolve().parents[2]++ def test_the_protocol_names_the_frozen_bundle(self):+ text = (self.ROOT / "docs" / "BENCHMARK_PROTOCOL.md").read_text(encoding="utf-8")+ self.assertIn(bs.load_corpus().sha256, text)+ self.assertIn(bs.PROTOCOL_ID, text)+ for heading in ("## Run it", "## Submit and review", "## Activation checklist", "### Where the numbers come from"):+ self.assertIn(heading, text)+ self.assertNotIn("do not implement this protocol", text)++ def test_the_terminal_archive_carries_the_fixtures(self):+ import sys+ sys.path.insert(0, str(self.ROOT / "packaging"))+ import package_terminal+ self.assertIn("bench/prompts/standard-v1", package_terminal.PAYLOAD)++ def test_the_launchers_point_at_the_standard_benchmark(self):+ for name in ("BENCH.bat", "bench.sh"):+ self.assertIn("strata.py bench run", (self.ROOT / name).read_text(encoding="utf-8"))+++class ScoringTests(unittest.TestCase):+ def test_strict_json(self):+ self.assertEqual(bs.strict_json(' {"a": 1}\n'), {"a": 1})+ for bad in ('{"a":1,"a":2}', "```json\n{}\n```", '{"a":1} extra', "NaN", "Infinity", "", '{"a":1}{"b":2}'):+ with self.assertRaises(ValueError, msg=bad):+ bs.strict_json(bad)++ def test_json_equality_checks_types(self):+ self.assertTrue(bs.same_json({"a": [1, 2], "b": True}, {"b": True, "a": [1, 2]}))+ self.assertFalse(bs.same_json({"ok": True}, {"ok": 1}))+ self.assertFalse(bs.same_json(3, 3.0))+ self.assertFalse(bs.same_json([1, 2], [2, 1]))+ self.assertFalse(bs.same_json({"a": 1}, {"a": 1, "b": 2}))++ def test_text_check_strips_only_ascii_whitespace(self):+ spec = {"expected": {"kind": "text", "value": "READY"}, "category": "instruction"}+ self.assertTrue(bs.evaluate(spec, "completed", " \nREADY\t", [])["passed"])+ self.assertFalse(bs.evaluate(spec, "completed", "READY.", [])["passed"])+ self.assertFalse(bs.evaluate(spec, "completed", "ready", [])["passed"])+ self.assertFalse(bs.evaluate(spec, "truncated", "READY", [])["passed"])++ def test_tool_call_needs_one_exact_structured_call(self):+ spec = {"expected": {"kind": "tool_call", "name": "get_weather", "arguments": {"city": "Zagreb", "unit": "celsius"}},+ "category": "tool_call"}+ good = {"name": "get_weather", "arguments": '{"unit":"celsius","city":"Zagreb"}'}+ self.assertTrue(bs.evaluate(spec, "completed", "", [good])["passed"])+ self.assertFalse(bs.evaluate(spec, "completed", "get_weather(...)", [])["passed"])+ self.assertFalse(bs.evaluate(spec, "completed", "", [good, good])["passed"])+ self.assertFalse(bs.evaluate(spec, "completed", "", [{**good, "name": "other"}])["passed"])+ self.assertFalse(bs.evaluate(spec, "completed", "", [{**good, "arguments": '{"city":"Zagreb"}'}])["passed"])+ self.assertFalse(bs.evaluate(spec, "completed", "", [{**good, "arguments": '{"city":"Zagreb","unit":"celsius","x":1}'}])["passed"])+ self.assertFalse(bs.evaluate(spec, "completed", "", [{**good, "arguments": '{"city":"a","city":"Zagreb","unit":"celsius"}'}])["passed"])+++class RunTests(StandardBase):+ def test_a_full_run_follows_the_protocol(self):+ doc, out = self.run_standard()+ self.assertEqual(bs.validate(doc), [])+ run = doc["run"]+ self.assertEqual((doc["format"], doc["schema_version"], run["status"]), ("strata-standard-benchmark", 1, "complete"))+ self.assertEqual(doc["protocol"]["id"], "strata-qwen38-thinking-v1-draft.1")+ self.assertEqual(doc["protocol"]["corpus_sha256"], bs.load_corpus().sha256)+ self.assertEqual(doc["protocol"]["git_commit"], COMMIT)+ self.assertEqual(len(doc["requests"]), 22)+ self.assertEqual(self.server.order, [r["id"] for r in doc["requests"]])+ self.assertEqual(doc["request_counts"]["warmup"], {"planned": 1, "attempted": 1, "completed": 1, "truncated": 0, "failed": 0, "skipped": 0})+ self.assertEqual(doc["request_counts"]["performance"]["completed"], 9)+ self.assertEqual(doc["request_counts"]["concurrency"]["planned"], 0)+ self.assertEqual(doc["quality"], {"passed": 12, "total": 12, "categories": {+ "instruction": {"passed": 4, "total": 4}, "retrieval": {"passed": 4, "total": 4}, "tool_call": {"passed": 4, "total": 4}}})+ self.assertEqual(len(doc["summaries"]), 15)+ self.assertEqual(doc["concurrency_rounds"], [])+ self.assertEqual(self.summary(doc, "prompt-4k", "prompt_tps")["median"], 1000.0)+ self.assertEqual(self.summary(doc, "short", "decode_tps")["median"], 100.0)+ for case in ("short", "prompt-4k", "prompt-16k"):+ for metric in ("prompt_tps", "decode_tps", "ttft_s", "first_answer_s", "total_s"):+ row = self.summary(doc, case, metric)+ self.assertEqual(row["n"], 3)+ self.assertLessEqual(row["min"], row["median"])+ self.assertLessEqual(row["median"], row["max"])+ # the warm-up and the sanity cases do not enter the speed summaries+ self.assertNotIn("warmup", {i for s in doc["summaries"] for i in s["included_request_ids"]})+ self.assertFalse(any(i.startswith("sanity") for s in doc["summaries"] for i in s["included_request_ids"]))+ self.assertTrue(run["required_duration_s"] > 0)+ self.assertIsNone(run["optional_duration_s"])+ self.assertIn("optional_duration_s", "".join(doc["missing"]))++ def test_every_request_carries_the_required_settings(self):+ self.run_standard()+ self.assertEqual(len(self.server.bodies), 22)+ specs = corpus_specs()+ for body, rid in zip(self.server.bodies, self.server.order):+ spec = specs[rid]+ self.assertIs(body["stream"], True)+ self.assertEqual(body["stream_options"], {"include_usage": True})+ self.assertEqual((body["temperature"], body["top_p"], body["top_k"], body["min_p"]), (1.0, 0.95, 20, 0.0))+ self.assertEqual((body["presence_penalty"], body["repetition_penalty"]), (0.0, 1.0))+ self.assertEqual(body["reasoning_effort"], "xhigh")+ self.assertEqual(body["chat_template_kwargs"], {"enable_thinking": True, "preserve_thinking": True})+ self.assertEqual(body["reasoning_budget_tokens"], 0)+ self.assertEqual((body["seed"], body["max_tokens"]), (spec["seed"], spec["max_output_tokens"]))+ self.assertNotIn("stop", body)+ self.assertEqual(body["messages"], spec["messages"])+ self.assertEqual(body.get("tools"), spec["tools"])+ if spec["tools"]:+ self.assertEqual(body["tool_choice"], "auto")++ def test_the_report_files_and_their_hashes(self):+ doc, out = self.run_standard()+ for name in ("results.json", "README.md", "outputs.jsonl", "config.json", "provenance/chat_template.jinja",+ "provenance/corpus-manifest.sha256", "provenance/corpus-build.json"):+ self.assertTrue((out / name).is_file(), name)+ saved = json.loads((out / "results.json").read_text(encoding="utf-8"))+ self.assertEqual(saved, doc)+ for artifact in doc["artifacts"]:+ self.assertEqual(bench_facts.sha256_file(out / artifact["path"]), artifact["sha256"])+ self.assertNotIn("..", artifact["path"])+ self.assertEqual({a["kind"] for a in doc["artifacts"]}, {"outputs", "config", "provenance"})+ lines = (out / "outputs.jsonl").read_text(encoding="utf-8").splitlines()+ self.assertEqual([json.loads(line)["request_id"] for line in lines], [r["id"] for r in doc["requests"]])+ first = json.loads(lines[0])+ self.assertEqual(first["dispatch_offset_s"], 0.0)+ self.assertEqual(first["first_delta"]["kind"], "reasoning")+ self.assertEqual(doc["configuration"]["template"], {"path": "provenance/chat_template.jinja",+ "sha256": bench_facts.sha256_text(TEMPLATE)})+ self.assertEqual(doc["model"]["files"], [{"path": self.model.name, "sha256": bench_facts.sha256_file(self.model)}])+ self.assertEqual(doc["model"]["quantization"], "IQ2_XS")+ readme = (out / "README.md").read_text(encoding="utf-8")+ self.assertIn("Status: complete", readme)+ self.assertIn("12 of 12", readme)+ self.assertNotIn(str(self.root), readme)++ def test_every_null_has_a_reason_and_the_request_shape_is_complete(self):+ doc, _ = self.run_standard()+ self.assertEqual(set(doc["missing"]), set(bs.scan_nulls(doc)))+ self.assertIn("/hardware/gpus/0/power_limit_w", doc["missing"])+ self.assertIn("/protocol/model_card_revision", doc["missing"])+ self.assertIn("/requests/0/round", doc["missing"])+ record = self.record(doc, "performance-short-r1")+ self.assertEqual(set(record), {"id", "phase", "case_id", "repetition", "seed", "max_output_tokens", "concurrency", "round",+ "status", "finish_reason", "error", "tokens", "timings", "metrics", "sources",+ "first_delta_kind", "token_accounting_note", "evaluation"})+ self.assertEqual(record["tokens"], {"input_tokens": record["tokens"]["input_tokens"], "cached_tokens": 0,+ "read_tokens": record["tokens"]["input_tokens"], "output_tokens": 50, "decode_tokens": 50})+ self.assertEqual(record["timings"]["decode_s"], 0.5)+ self.assertEqual(record["first_delta_kind"], "reasoning")+ self.assertIsNone(record["token_accounting_note"])++ def test_the_run_names_its_default_folder_after_the_gpu(self):+ self.server.gpu_name = "NVIDIA GeForce RTX 3060"+ with mock.patch.object(bench_facts, "nvidia_gpus", return_value=([{+ "device_index": 0, "name": "NVIDIA GeForce RTX 3060", "memory_bytes": 12 * 2 ** 30, "memory_kind": "dedicated",+ "power_limit_w": 170.0, "driver": "560.1"}], "560.1", "12.6")):+ doc, out = bs.run(self.target, None, out_root=self.root / "results", hash_model=True)+ self.assertRegex(out.name, r"^\d{4}-\d\d-\d\d-standard-rtx-3060-[0-9a-f]{8}$")+ self.assertEqual(doc["hardware"]["gpus"][0]["power_limit_w"], 170.0)+ self.assertEqual(doc["software"]["runtime"], [{"name": "cuda", "version": "12.6"}])++ def test_early_eos_and_a_reached_cap_stay_in_the_speed_summaries(self):+ self.server.script["performance-short-r1"] = {"completion_tokens": 5, "predicted_n": 5, "rate": 10.0}+ self.server.script["performance-short-r2"] = {"finish": "length", "completion_tokens": 512, "predicted_n": 512, "rate": 1024.0}+ doc, _ = self.run_standard()+ self.assertEqual(self.record(doc, "performance-short-r2")["status"], "truncated")+ self.assertEqual(self.record(doc, "performance-short-r2")["finish_reason"], "length")+ row = self.summary(doc, "short", "decode_tps")+ self.assertEqual((row["n"], row["min"], row["max"]), (3, 10.0, 1024.0))+ self.assertEqual(doc["request_counts"]["performance"]["truncated"], 1)++ def test_a_failed_request_is_excluded_and_listed_but_the_run_goes_on(self):+ self.server.script["performance-prompt-4k-r2"] = {"http": 500}+ doc, out = self.run_standard()+ self.assertEqual(doc["run"]["status"], "complete")+ record = self.record(doc, "performance-prompt-4k-r2")+ self.assertEqual((record["status"], record["error"]["kind"]), ("failed", "http_500"))+ row = self.summary(doc, "prompt-4k", "decode_tps")+ self.assertEqual(row["n"], 2)+ self.assertEqual(row["excluded"], [{"request_id": "performance-prompt-4k-r2", "reason": "failed: http_500"}])+ self.assertEqual(doc["request_counts"]["performance"]["failed"], 1)+ self.assertEqual(len(self.server.order), 22)+ self.assertIn("performance-prompt-4k-r2 failed", (out / "README.md").read_text(encoding="utf-8"))+ self.assertEqual(bs.validate(doc), [])++ def test_a_failed_warmup_skips_everything_else(self):+ self.server.script["warmup"] = {"http": 500}+ doc, _ = self.run_standard()+ self.assertEqual(doc["run"]["status"], "partial")+ self.assertEqual(len(self.server.order), 1)+ self.assertEqual(len(doc["requests"]), 22)+ self.assertEqual(doc["request_counts"]["performance"]["skipped"], 9)+ self.assertEqual(doc["request_counts"]["sanity"], {"planned": 12, "attempted": 0, "completed": 0, "truncated": 0, "failed": 0, "skipped": 12})+ self.assertEqual(doc["quality"]["passed"], 0)+ skipped = self.record(doc, "sanity-T4")+ self.assertEqual((skipped["status"], skipped["error"]["kind"]), ("skipped", "skipped"))+ self.assertFalse(skipped["evaluation"]["passed"])+ self.assertTrue(all(s["n"] == 0 and s["median"] is None for s in doc["summaries"]))+ self.assertEqual(bs.validate(doc), [])++ def test_a_dead_server_marks_the_rest_skipped(self):+ self.server.script["performance-prompt-4k-r1"] = {"die": True}+ doc, _ = self.run_standard()+ self.assertEqual(doc["run"]["status"], "partial")+ self.assertEqual(self.record(doc, "performance-prompt-4k-r1")["status"], "failed")+ self.assertEqual(self.record(doc, "performance-prompt-16k-r1")["status"], "skipped")+ self.assertIn("server stopped answering", self.record(doc, "sanity-I1")["error"]["message"])+ self.assertEqual(len(doc["requests"]), 22)+ self.assertEqual(bs.validate(doc), [])++ def test_a_request_that_never_finishes_times_out(self):+ self.server.script["performance-short-r1"] = {"hang": 3}+ started = time.monotonic()+ doc, _ = self.run_standard(timeout_s=0.6)+ record = self.record(doc, "performance-short-r1")+ self.assertEqual((record["status"], record["error"]["kind"]), ("failed", "timeout"))+ self.assertLess(record["timings"]["total_s"], 2.0)+ self.assertGreaterEqual(record["timings"]["total_s"], 0.5)+ self.assertEqual(doc["run"]["status"], "complete")+ self.assertLess(time.monotonic() - started, 30)++ def test_a_stream_without_done_or_finish_reason_is_a_failure(self):+ self.server.script["performance-short-r1"] = {"no_done": True}+ self.server.script["performance-short-r2"] = {"finish": None}+ doc, _ = self.run_standard()+ self.assertEqual(self.record(doc, "performance-short-r1")["error"]["kind"], "malformed_stream")+ self.assertEqual(self.record(doc, "performance-short-r2")["error"]["kind"], "malformed_stream")+ self.assertEqual(self.summary(doc, "short", "decode_tps")["n"], 1)++ def test_empty_deltas_and_keepalives_do_not_count_as_the_first_token(self):+ self.server.script["performance-short-r1"] = {"first_delay": 0.4}+ doc, out = self.run_standard()+ record = self.record(doc, "performance-short-r1")+ self.assertGreaterEqual(record["timings"]["ttft_s"], 0.35)+ self.assertEqual(record["first_delta_kind"], "reasoning")+ self.assertGreaterEqual(record["timings"]["first_answer_s"], record["timings"]["ttft_s"])+ tool = self.record(doc, "sanity-T1")+ self.assertIsNotNone(tool["timings"]["first_answer_s"]) # a tool-call delta is an answer+ self.assertIsNone(self.record(doc, "sanity-T1")["error"])++ def test_missing_usage_and_timings_leave_the_rates_null_with_reasons(self):+ self.server.script["performance-short-r1"] = {"usage": False, "timings": False}+ doc, _ = self.run_standard()+ record = self.record(doc, "performance-short-r1")+ self.assertEqual(record["status"], "completed")+ self.assertIsNone(record["metrics"]["prompt_tps"])+ self.assertIsNone(record["metrics"]["decode_tps"])+ self.assertIsNone(record["tokens"]["output_tokens"])+ self.assertIsNotNone(record["timings"]["ttft_s"])+ index = doc["requests"].index(record)+ self.assertIn(f"/requests/{index}/metrics/decode_tps", doc["missing"])+ row = self.summary(doc, "short", "decode_tps")+ self.assertEqual(row["n"], 2)+ self.assertIn("decode_tps is not available", row["excluded"][0]["reason"])+ self.assertEqual(bs.validate(doc), [])++ def test_a_decode_count_that_disagrees_with_the_rate_is_explained(self):+ # the engine's rate and time imply 50 tokens (100/s over 0.5 s) while predicted_n says 60 and the API says 61+ self.server.script["performance-short-r1"] = {"predicted_n": 60, "completion_tokens": 61}+ doc, _ = self.run_standard()+ record = self.record(doc, "performance-short-r1")+ self.assertEqual(record["tokens"]["decode_tokens"], 50)+ self.assertEqual(record["tokens"]["output_tokens"], 61)+ self.assertIn("implied", record["token_accounting_note"])+ self.assertEqual(record["metrics"]["decode_tps"], 100.0)+ self.assertIn("derived", record["sources"]["decode_tokens"])++ def test_wrong_answers_are_kept_and_scored_without_changing_the_run_status(self):+ specs = corpus_specs()+ script = self.server.script+ script["sanity-I1"] = {"content": '```json\n{"ok":true,"count":3}\n```'}+ script["sanity-I2"] = {"content": "Here you go: [-4,1,2,9]"}+ script["sanity-I3"] = {"content": '{"project":"Orion","project":"Orion","build":17}'}+ script["sanity-R1"] = {"finish": "length", "content": "ORB", "completion_tokens": 2048, "predicted_n": 2048}+ script["sanity-R2"] = {"content": "cedar-284"}+ script["sanity-T1"] = {"tool_calls": [], "content": "lookup_record({\"id\":\"A-17\"})"}+ script["sanity-T2"] = {"tool_calls": [{"name": "get_weather", "arguments": '{"city":"Zagreb","unit":"celsius"}'}] * 2}+ script["sanity-T3"] = {"tool_calls": [{"name": "convert_units", "arguments": '{"value":3.0,"from_unit":"km","to_unit":"m"}'}]}+ doc, _ = self.run_standard()+ verdicts = {r["case_id"]: r["evaluation"]["passed"] for r in doc["requests"] if r["phase"] == "sanity"}+ self.assertEqual(verdicts, {"I1": False, "I2": False, "I3": False, "I4": True, "R1": False, "R2": False, "R3": True, "R4": True,+ "T1": False, "T2": False, "T3": False, "T4": True})+ self.assertEqual(doc["quality"]["passed"], 4)+ self.assertEqual(doc["quality"]["categories"]["instruction"]["passed"], 1)+ self.assertEqual(self.record(doc, "sanity-R1")["status"], "truncated")+ self.assertIn("output cap", self.record(doc, "sanity-R1")["evaluation"]["reason"])+ self.assertEqual(doc["run"]["status"], "complete")+ self.assertEqual(specs["sanity-I1"]["expected"]["value"], {"ok": True, "count": 3})+ text = bs.format_plain(doc)+ self.assertIn("Sanity checks: 4 of 12 passed", text)+ self.assertIn("I1:", text)++ def test_unexpected_reuse_of_the_user_payload_is_excluded_and_nonconforming(self):+ spec = corpus_specs()["performance-prompt-4k-r2"]+ self.server.script["performance-prompt-4k-r2"] = {"cached": spec["input_tokens_frozen"] - 10}+ # the shared template prefix alone is allowed+ self.server.script["performance-prompt-4k-r1"] = {"cached": 40}+ doc, _ = self.run_standard()+ self.assertEqual(doc["run"]["status"], "nonconforming")+ row = self.summary(doc, "prompt-4k", "prompt_tps")+ self.assertEqual(row["n"], 2)+ self.assertIn("unexpected cache reuse", row["excluded"][0]["reason"])+ self.assertTrue(any("unexpected reuse of the user payload in performance-prompt-4k-r2" in r+ for r in doc["extensions"]["status_reasons"]))+ self.assertEqual(self.record(doc, "performance-prompt-4k-r1")["tokens"]["cached_tokens"], 40)+ self.assertEqual(bs.validate(doc), [])++ def test_a_prompt_length_off_its_target_is_nonconforming(self):+ original = bs.load_corpus()+ specs = copy.deepcopy(original.specs)+ for spec in specs:+ if spec["id"] == "performance-prompt-4k-r1":+ spec["target_input_tokens"] = 3000+ patched = bs.Corpus(original.sha256, specs, original.build, original.manifest, original.directory)+ doc, _ = self.run_standard(corpus=patched)+ self.assertEqual(doc["run"]["status"], "nonconforming")+ self.assertTrue(any("outside 2%" in r for r in doc["extensions"]["status_reasons"]))++ def test_a_refused_setting_is_nonconforming(self):+ self.server.script["warmup"] = {"http": 400}+ doc, _ = self.run_standard()+ self.assertEqual(doc["run"]["status"], "nonconforming")+ self.assertTrue(any("refused the protocol's required settings" in r for r in doc["extensions"]["status_reasons"]))+ self.assertEqual(self.record(doc, "sanity-I1")["status"], "skipped")++ def test_missing_identity_makes_the_run_nonconforming(self):+ with mock.patch.object(bench_facts, "git_commit", return_value=None):+ doc, _ = self.run_standard("nocommit")+ self.assertEqual(doc["run"]["status"], "nonconforming")+ self.assertIn("/software/strata_commit", doc["missing"])+ self.assertIsNone(doc["protocol"]["git_commit"])+ doc2, _ = self.run_standard("nohash", hash_model=False)+ self.assertEqual(doc2["run"]["status"], "nonconforming")+ self.assertIsNone(doc2["model"]["files"])+ self.assertIn("/model/files", doc2["missing"])++ def test_a_remote_server_is_nonconforming(self):+ with mock.patch.object(bs.Target, "loopback", new_callable=mock.PropertyMock, return_value=False):+ doc, _ = self.run_standard()+ self.assertEqual(doc["run"]["status"], "nonconforming")+ self.assertTrue(any("not on loopback" in r for r in doc["extensions"]["status_reasons"]))++ def test_cancelling_keeps_the_finished_requests_and_skips_the_rest(self):+ real = bs.stream_chat+ calls = []++ def interrupted(target, spec, model, **kwargs):+ calls.append(spec["id"])+ if len(calls) == 6:+ class Boom:+ sock = None++ def request(self, *args, **kw):+ raise KeyboardInterrupt++ def close(self):+ pass+ with mock.patch.object(bs.Target, "connection", return_value=Boom()):+ return real(target, spec, model, **kwargs)+ return real(target, spec, model, **kwargs)+ with mock.patch.object(bs, "stream_chat", interrupted):+ doc, _ = self.run_standard()+ self.assertEqual(doc["run"]["status"], "partial")+ self.assertEqual(self.record(doc, "performance-prompt-4k-r2")["error"]["kind"], "cancelled")+ self.assertEqual(self.record(doc, "performance-prompt-4k-r2")["status"], "failed")+ self.assertEqual(self.record(doc, "performance-prompt-16k-r3")["status"], "skipped")+ self.assertEqual(self.record(doc, "sanity-T4")["error"]["message"], "the operator cancelled the run")+ self.assertEqual(self.summary(doc, "short", "decode_tps")["n"], 3 - 1)+ self.assertEqual(bs.validate(doc), [])++ def test_the_optional_concurrency_stage_releases_four_together(self):+ for r in (1, 2, 3):+ for c in (1, 2, 3, 4):+ self.server.script[f"concurrency-r{r}-c{c}"] = {"hold_for_arrivals": 4}+ doc, _ = self.run_standard(optional_concurrency=True)+ self.assertEqual(doc["run"]["status"], "complete")+ self.assertEqual(len(doc["requests"]), 34)+ self.assertEqual(self.server.max_active, 4)+ self.assertEqual(doc["request_counts"]["concurrency"]["planned"], 12)+ self.assertEqual(len(doc["concurrency_rounds"]), 3)+ for round_ in doc["concurrency_rounds"]:+ self.assertEqual(round_["output_tokens"], 200)+ self.assertEqual(round_["failed"], 0)+ self.assertAlmostEqual(round_["aggregate_tps"], 200 / round_["wall_s"], places=3)+ self.assertEqual(len(round_["request_ids"]), 4)+ rows = [s for s in doc["summaries"] if s["phase"] == "concurrency"]+ self.assertEqual(len(rows), 5)+ self.assertEqual({(s["case_id"], s["concurrency"]) for s in rows}, {("short-burst", 4)})+ self.assertEqual(rows[0]["n"], 12)+ self.assertIsNotNone(doc["run"]["optional_duration_s"])+ for round_no in (1, 2, 3): # a round finishes before the next one is released++ self.assertEqual({n for n in self.server.order[22 + 4 * (round_no - 1): 22 + 4 * round_no]},+ {f"concurrency-r{round_no}-c{c}" for c in (1, 2, 3, 4)})+ self.assertEqual(bs.validate(doc), [])++ def test_a_failed_concurrent_request_keeps_its_time_in_the_round(self):+ self.server.script["concurrency-r1-c2"] = {"http": 500}+ doc, _ = self.run_standard(optional_concurrency=True)+ first = doc["concurrency_rounds"][0]+ self.assertEqual((first["failed"], first["output_tokens"]), (1, 150))+ self.assertEqual(doc["concurrency_rounds"][1]["failed"], 0)+ self.assertEqual(doc["request_counts"]["concurrency"]["failed"], 1)++ def test_a_round_total_is_null_when_a_counted_request_has_no_token_count(self):+ self.server.script["concurrency-r2-c3"] = {"usage": False}+ doc, _ = self.run_standard(optional_concurrency=True)+ second = doc["concurrency_rounds"][1]+ self.assertIsNone(second["output_tokens"])+ self.assertIsNone(second["aggregate_tps"])+ self.assertIsNotNone(doc["concurrency_rounds"][0]["aggregate_tps"])+ self.assertEqual(bs.validate(doc), [])++ def test_the_plain_summary(self):+ doc, _ = self.run_standard()+ text = bs.format_plain(doc)+ self.assertIn("Status: complete. 21 of 21 required requests finished, 0 failed, 0 skipped.", text)+ self.assertIn("prompt-16k", text)+ self.assertIn("Sanity checks: 12 of 12 passed", text)+ self.assertIn("Prompt tok/s", text)+ self.assertIn("1000.0 (1000.0-1000.0) n=3", text)++ def test_the_server_is_not_asked_for_anything_outside_the_protocol(self):+ self.run_standard()+ self.assertTrue(all(body["messages"] == corpus_specs()[rid]["messages"] for body, rid in zip(self.server.bodies, self.server.order)))+++class ValidatorTests(StandardBase):+ def test_the_validator_refuses_broken_results(self):+ doc, _ = self.run_standard()+ self.assertEqual(bs.validate(doc), [])+ broken = copy.deepcopy(doc)+ broken["schema_version"] = 2+ self.assertTrue(bs.validate(broken))+ broken = copy.deepcopy(doc)+ broken["request_counts"]["performance"]["completed"] = 8+ self.assertTrue(any("identity" in p for p in bs.validate(broken)))+ broken = copy.deepcopy(doc)+ broken["missing"].pop(next(iter(broken["missing"])))+ self.assertTrue(any("missing must list" in p for p in bs.validate(broken)))+ broken = copy.deepcopy(doc)+ broken["summaries"][0]["median"] = 12345.0+ self.assertTrue(any("rebuilt" in p for p in bs.validate(broken)))+ broken = copy.deepcopy(doc)+ broken["requests"][1]["timings"]["ttft_s"] = float("nan")+ self.assertTrue(bs.validate(broken))+ broken = copy.deepcopy(doc)+ broken["surprise"] = 1+ self.assertTrue(any("extensions" in p for p in bs.validate(broken)))+ broken = copy.deepcopy(doc)+ broken["artifacts"][0]["path"] = "../escape"+ self.assertTrue(any("artifact" in p for p in bs.validate(broken)))++ def test_load_reads_a_saved_result_and_refuses_other_files(self):+ doc, out = self.run_standard()+ self.assertEqual(bs.load(out)["run"]["id"], doc["run"]["id"])+ self.assertIsNone(bs.load_if_standard(self.root / "nothing"))+ other = self.root / "other.json"+ other.write_text(json.dumps({"kind": "strata.serving.benchmark"}), encoding="utf-8")+ self.assertIsNone(bs.load_if_standard(other))+ bad = json.loads((out / "results.json").read_text(encoding="utf-8"))+ bad["schema_version"] = 9+ (self.root / "bad.json").write_text(json.dumps(bad), encoding="utf-8")+ with self.assertRaises(ProfileError):+ bs.load_if_standard(self.root / "bad.json")+++class CommandLineTests(StandardBase):+ def invoke(self, *args, env=None):+ output, errors = io.StringIO(), io.StringIO()+ with mock.patch.dict(os.environ, env or {}), contextlib.redirect_stdout(output), contextlib.redirect_stderr(errors):+ code = main(["bench", "run", *args])+ return code, output.getvalue(), errors.getvalue()++ def test_the_standard_suite_is_the_default_and_prints_a_plain_summary(self):+ out = self.root / "cli"+ code, text, errors = self.invoke("--url", self.server.url, "--out", str(out))+ self.assertEqual(code, 0, errors)+ self.assertIn("Strata standard benchmark (strata-qwen38-thinking-v1-draft.1)", text)+ self.assertIn("Sanity checks: 12 of 12 passed", text)+ self.assertIn("Report:", text)+ self.assertEqual(bs.load(out)["run"]["status"], "complete")+ self.assertIn("[1/22] warmup", errors)++ def test_json_output(self):+ code, text, _ = self.invoke("--url", self.server.url, "--out", str(self.root / "j"), "--output", "json")+ self.assertEqual(code, 0)+ value = json.loads(text)+ self.assertTrue(value["ok"])+ self.assertEqual(value["quality"]["passed"], 12)+ self.assertEqual(len(value["summaries"]), 15)++ def test_a_partial_run_exits_with_6(self):+ self.server.script["warmup"] = {"http": 500}+ code, text, _ = self.invoke("--url", self.server.url, "--out", str(self.root / "p"))+ self.assertEqual(code, 6)+ self.assertIn("Status: partial", text)+ self.assertIn("Why this is not a complete standard result", text)++ def test_serving_options_do_not_apply(self):+ for option in (("--concurrency", "2"), ("--repeat", "2"), ("--request-timeout", "30"), ("--condition", "baseline")):+ code, text, _ = self.invoke("--url", self.server.url, "--out", str(self.root / "x"), "--output", "json", *option)+ self.assertEqual(code, 2, option)+ self.assertIn("do not apply to the standard suite", text)+ self.assertFalse((self.root / "x").exists())++ def test_the_out_folder_must_be_new_and_the_default_goes_under_bench_results(self):+ (self.root / "taken").mkdir()+ code, text, _ = self.invoke("--url", self.server.url, "--out", str(self.root / "taken"), "--output", "json")+ self.assertEqual(code, 2)+ previous = Path.cwd()+ os.chdir(self.root)+ try:+ code, text, _ = self.invoke("--url", self.server.url)+ finally:+ os.chdir(previous)+ self.assertEqual(code, 0)+ self.assertTrue(any((self.root / "bench-results").glob("*-standard-*/results.json")))++ def test_the_serving_suites_still_need_out_and_keep_their_defaults(self):+ code, text, _ = self.invoke("--url", self.server.url, "--suite", "serving-smoke", "--output", "json")+ self.assertEqual(code, 2)+ self.assertIn("--out is required for the serving suites", text)++ def test_the_options_reach_the_report(self):+ out = self.root / "opts"+ code, _, errors = self.invoke("--url", self.server.url, "--out", str(out), "--track", "default", "--track-notes",+ "0.2.0 release, setup choices", "--model-revision", "r1", "--source-revision", "r2",+ "--tokenizer-revision", "r3", "--custom-pack", "none", "--background", "browser open",+ "--build-options", "cuda,sm86", "--optional-concurrency")+ self.assertEqual(code, 0, errors)+ doc = bs.load(out)+ self.assertEqual((doc["configuration"]["track"], doc["configuration"]["track_notes"]), ("default", "0.2.0 release, setup choices"))+ self.assertEqual((doc["model"]["revision"], doc["model"]["source_revision"], doc["model"]["tokenizer_revision"]), ("r1", "r2", "r3"))+ self.assertEqual(doc["hardware"]["background_workloads"], ["browser open"])+ self.assertEqual(doc["software"]["build_options"], ["cuda", "sm86"])+ self.assertTrue(doc["run"]["optional_concurrency"])+ self.assertIn("--track default", doc["run"]["command"])+ self.assertTrue(doc["run"]["command"].startswith("strata bench run"))++ def test_report_reads_a_standard_result_in_three_forms(self):+ out = self.root / "rep"+ self.assertEqual(self.invoke("--url", self.server.url, "--out", str(out))[0], 0)+ output = io.StringIO()+ with contextlib.redirect_stdout(output):+ self.assertEqual(main(["bench", "report", str(out)]), 0)+ self.assertIn("Sanity checks: 12 of 12 passed", output.getvalue())+ output = io.StringIO()+ with contextlib.redirect_stdout(output):+ self.assertEqual(main(["bench", "report", str(out), "--output", "markdown"]), 0)+ self.assertIn("# Strata standard benchmark", output.getvalue())+ output = io.StringIO()+ with contextlib.redirect_stdout(output):+ self.assertEqual(main(["bench", "report", str(out), "--output", "json"]), 0)+ self.assertEqual(json.loads(output.getvalue())["summary"]["status"], "complete")++ def test_a_profile_that_is_not_running_is_started_and_stopped(self):+ out = self.root / "prof"+ cfg = {"api_key": "", "model_name": "mock"}+ plan = {"effective": {"host": "127.0.0.1", "port": int(self.server.url.rsplit(":", 1)[1])}, "engine_argv": ["--kv", "int8", "--expert-cache", "1500"]}+ health = {"calls": 0}+ real_get = bs.Target.get++ def get(target, path, timeout=10):+ if path == "/health" and health["calls"] < 1:+ health["calls"] += 1+ return None # nothing answers before the engine is started+ return real_get(target, path, timeout)+ started = []+ with mock.patch("strata_cli.cli._source", return_value=(Path("p.json"), None, None, cfg)), \+ mock.patch("strata_cli.cli._launch_plan", return_value=(plan, [])), \+ mock.patch.object(bs.Target, "get", get), \+ mock.patch.object(bs, "start_engine", side_effect=lambda ref, target, log, say, wait: started.append(ref) or "proc") as start, \+ mock.patch.object(bs, "stop_engine") as stop:+ code, text, errors = self.invoke("--profile", "p.json", "--out", str(out))+ self.assertEqual(code, 0, errors)+ self.assertEqual(started, ["p.json"])+ stop.assert_called_once_with("proc")+ doc = bs.load(out)+ self.assertTrue(doc["extensions"]["started_engine"])+ self.assertEqual(doc["configuration"]["kv_cache"], {"--kv": "int8"})+ self.assertEqual(doc["configuration"]["expert_cache"], {"--expert-cache": "1500"})+ self.assertNotIn("/configuration/kv_cache", doc["missing"])++ def test_a_running_profile_is_used_and_left_running(self):+ plan = {"effective": {"host": "127.0.0.1", "port": int(self.server.url.rsplit(":", 1)[1])}, "engine_argv": []}+ with mock.patch("strata_cli.cli._source", return_value=(Path("p.json"), None, None, {})), \+ mock.patch("strata_cli.cli._launch_plan", return_value=(plan, [])), \+ mock.patch.object(bs, "start_engine") as start, mock.patch.object(bs, "stop_engine") as stop:+ code, text, errors = self.invoke("--profile", "p.json", "--out", str(self.root / "run"))+ self.assertEqual(code, 0, errors)+ start.assert_not_called()+ stop.assert_called_once_with(None)+ self.assertIn("already running", errors)+ self.assertFalse(bs.load(self.root / "run")["extensions"]["started_engine"])++ def test_no_start_refuses_to_launch(self):+ self.server.close()+ plan = {"effective": {"host": "127.0.0.1", "port": int(self.server.url.rsplit(":", 1)[1])}, "engine_argv": []}+ with mock.patch("strata_cli.cli._source", return_value=(Path("p.json"), None, None, {})), \+ mock.patch("strata_cli.cli._launch_plan", return_value=(plan, [])), \+ mock.patch.object(bs, "start_engine") as start:+ code, text, _ = self.invoke("--profile", "p.json", "--out", str(self.root / "ns"), "--no-start", "--output", "json")+ self.assertEqual(code, 5)+ start.assert_not_called()+++class EngineProcessTests(unittest.TestCase):+ def test_start_engine_runs_serve_with_the_profile_and_stop_ends_only_that_process(self):+ class Fake:+ pid = 4242++ def poll(self):+ return None++ def wait(self, timeout=None):+ return 0+ seen = {}++ def popen(command, **kwargs):+ seen["command"] = command+ return Fake()+ target = mock.Mock()+ target.get.return_value = {"service": "strata", "loaded": True}+ with tempfile.TemporaryDirectory() as temp, mock.patch("subprocess.Popen", popen):+ proc = bs.start_engine("run.json", target, Path(temp) / "engine.log", lambda m: None, 5)+ self.assertEqual(seen["command"][1].replace("\\", "/").rsplit("/", 1)[-1], "strata.py")+ self.assertEqual(seen["command"][2:5], ["serve", "--profile", "run.json"])+ with mock.patch("subprocess.run") as run, mock.patch("os.killpg", create=True) as killpg:+ bs.stop_engine(proc)+ if os.name == "nt":+ self.assertEqual(run.call_args[0][0][:4], ["taskkill", "/PID", "4242", "/T"])+ else:+ killpg.assert_called()++ def test_an_engine_that_exits_before_it_is_ready_is_an_error(self):+ class Dead:+ pid = 1++ def poll(self):+ return 3++ def wait(self, timeout=None):+ return 3+ target = mock.Mock()+ target.get.return_value = None+ with tempfile.TemporaryDirectory() as temp, mock.patch("subprocess.Popen", lambda *a, **k: Dead()):+ with self.assertRaisesRegex(ProfileError, "did not become ready"):+ bs.start_engine("run.json", target, Path(temp) / "engine.log", lambda m: None, 5)+++class RealServerTests(unittest.TestCase):+ """The same request bodies against the real API server with its mock engine: every protocol setting is accepted."""++ def test_the_real_server_accepts_the_protocol_requests(self):+ from serve.frontend import ChatTemplate+ from serve.server import ByteTokenizer, MockEngine, Service, serve+ with tempfile.TemporaryDirectory() as temp:+ tok = ByteTokenizer()+ template = ChatTemplate(Path(__file__).resolve().parents[2] / "serve/chat_template.jinja")+ service = Service(MockEngine(tok, "A hash table hashes the key.", max_context=400000), tok, template)+ service.api_key = "inference-test-secret"+ httpd = serve(service, port=0)+ self.addCleanup(httpd.server_close)+ self.addCleanup(httpd.shutdown)+ target = bs.Target(f"http://127.0.0.1:{httpd.server_address[1]}", "inference-test-secret")+ patches = [mock.patch.object(bench_facts, "git_commit", return_value=COMMIT)]+ for patch in patches:+ patch.start()+ self.addCleanup(patch.stop)+ doc, out = bs.run(target, Path(temp) / "real", hash_model=True)+ self.assertEqual(bs.validate(doc), [])+ failed = [(r["id"], r["error"]) for r in doc["requests"] if r["status"] == "failed"]+ self.assertEqual(failed, [])+ self.assertTrue(all(r["status"] == "completed" for r in doc["requests"]))+ self.assertEqual(doc["configuration"]["template"]["sha256"], bench_facts.sha256_text(template.source))+ self.assertIsNotNone(doc["configuration"]["context_tokens"])+ # the mock engine has no clock: the rates stay null and say why instead of being estimated+ record = next(r for r in doc["requests"] if r["id"] == "performance-short-r1")+ self.assertIsNone(record["metrics"]["decode_tps"])+ self.assertIsNotNone(record["tokens"]["input_tokens"])+ self.assertIsNotNone(record["timings"]["ttft_s"])+ self.assertEqual(doc["run"]["status"], "nonconforming") # no model file to hash, so the identity is incomplete+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/terminal/test_runtime.py b/tests/terminal/test_runtime.pyindex 56a726a50..feef2dec2 100644--- a/tests/terminal/test_runtime.py+++ b/tests/terminal/test_runtime.py@@ -571,7 +571,7 @@ class RuntimeTests(unittest.TestCase):self.assertTrue(result["profile_running_check"]["after"]["matched"])output_json = io.StringIO()with redirect_stdout(output_json):- code = main(["bench", "run", "--profile", "local", "--require-managed-match",+ code = main(["bench", "run", "--suite", "serving-smoke", "--profile", "local", "--require-managed-match","--concurrency", "1", "--out", str(self.root / "cli-managed"),"--output", "json"])self.assertEqual(code, 0)