Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
bench: measure persistent cross-engine datapoints
JustVugg176a493c042d
diff --git a/c/tests/test_datapoint.py b/c/tests/test_datapoint.pyindex ae1b6a81..09a8b465 100644--- a/c/tests/test_datapoint.py+++ b/c/tests/test_datapoint.py@@ -6,10 +6,16 @@ measured warm and published as cold. These tests run on any host bymocking sys.platform and the two win32 probes."""import ctypes+import osimport sys+import tempfile+import threadingimport unittest+from pathlib import Path+from types import SimpleNamespacefrom unittest import mock+from family_registry import FAMILIESfrom tools import datapointGB = 1073741824@@ -49,5 +55,174 @@ class MachineInfoWin32Test(unittest.TestCase):self.assertEqual(info["ram"], "?")+class PersistentDatapointTest(unittest.TestCase):+ def setUp(self):+ self.family = SimpleNamespace(+ id="deepseek_v4",+ display_name="DeepSeek V4",+ has_gateway_adapter=True,+ build_target="deepseek_v4",+ )+ self.instances = []++ def _runtime(self, engine_type=None):+ instances = self.instances++ class FakeEngine:+ def __init__(self, executable, model, cap, max_tokens, env, kv_slots,+ family):+ self.executable = executable+ self.model = model+ self.cap = cap+ self.max_tokens = max_tokens+ self.env = env+ self.kv_slots = kv_slots+ self.family = family+ self.calls = []+ self.closed = False+ self.tiers = {"vram": 2, "ram": 3, "disk": 4,+ "vram_gb": 1.5, "ram_gb": 2.5}+ self.hwinfo = {"cores": 16}+ self.profile = []+ self.profile_seq = 0+ instances.append(self)++ def generate(self, prompt, max_tokens, temperature, top_p, on_text,+ cache_slot=0):+ self.calls.append((prompt, max_tokens, temperature, top_p, cache_slot))+ on_text("token")+ self.profile.append({"expert_disk_s": 0.1,+ "expert_wait_s": 0.2,+ "expert_matmul_s": 0.3,+ "attention_s": 0.4,+ "lm_head_s": 0.5})+ self.profile_seq += 1+ return {"completion_tokens": max_tokens,+ "tokens_per_second": float(len(self.calls)),+ "cache_hit_percent": float(len(self.calls) * 10),+ "rss_gb": 12.5,+ "prompt_tokens": 17,+ "length_limited": True}++ def close(self):+ self.closed = True++ return SimpleNamespace(+ ARCH=None,+ Engine=engine_type or FakeEngine,+ resolve_model=lambda _snap: SimpleNamespace(descriptor=self.family),+ default_engine=lambda _family: "/unused/default-engine",+ render_chat_for_arch=lambda messages, enable_thinking: (+ f"rendered:{messages[0]['content']}:{enable_thinking}"),+ )++ def test_default_campaign_reuses_one_engine_for_every_request(self):+ runtime = self._runtime()+ with tempfile.TemporaryDirectory() as tmp:+ executable = Path(tmp) / "deepseek_v4"+ executable.touch()+ with mock.patch.dict(os.environ, {}, clear=True):+ campaign = datapoint.run_persistent_engine(+ str(executable), "/model", "hello", 8, 1, 3, 16,+ memory_gb=64, runtime=runtime, physical_cores=8)++ self.assertEqual(len(self.instances), 1)+ engine = self.instances[0]+ self.assertEqual(len(engine.calls), 5) # cold + warm-up + 3 measured+ self.assertTrue(all(call[0] == "rendered:hello:False" for call in engine.calls))+ self.assertTrue(all(call[4] == 0 for call in engine.calls))+ self.assertTrue(engine.closed)+ self.assertEqual(engine.env["RAM_GB"], "64")+ self.assertEqual(engine.env["COLI_TEMP"], "0")+ self.assertNotIn("OMP_NUM_THREADS", engine.env) # V4 owns its thread split+ self.assertEqual(runtime.ARCH, "deepseek_v4")+ self.assertEqual(len(campaign["cold"]), 1)+ self.assertEqual(len(campaign["warmup"]), 1)+ self.assertEqual(len(campaign["warm"]), 3)+ self.assertEqual([row["tok_s"] for row in campaign["warm"]], [3.0, 4.0, 5.0])+ self.assertEqual(campaign["warm"][0]["tokens"], 8)+ self.assertEqual(campaign["warm"][0]["prompt_tokens"], 17)++ def test_engine_is_closed_when_a_request_fails(self):+ instances = self.instances++ class FailingEngine:+ def __init__(self, *_args, **_kwargs):+ self.profile_seq = 0+ self.profile = []+ self.closed = False+ instances.append(self)++ def generate(self, *_args, **_kwargs):+ raise RuntimeError("generation failed")++ def close(self):+ self.closed = True++ runtime = self._runtime(FailingEngine)+ with tempfile.TemporaryDirectory() as tmp:+ executable = Path(tmp) / "deepseek_v4"+ executable.touch()+ with self.assertRaisesRegex(RuntimeError, "generation failed"):+ datapoint.run_persistent_engine(+ str(executable), "/model", "hello", 8, 1, 3, 16,+ runtime=runtime, physical_cores=8)+ self.assertTrue(self.instances[0].closed)++ def test_trailing_profile_after_done_is_attached_to_the_request(self):+ class DelayedProfileEngine:+ profile_seq = 0+ profile = []++ def generate(self, _prompt, max_tokens, _temperature, _top_p,+ on_text, cache_slot=0):+ self.asserted_slot = cache_slot+ on_text("token")++ def emit_profile():+ self.profile = [{"expert_disk_s": 1.25}]+ self.profile_seq += 1++ self.timer = threading.Timer(0.01, emit_profile)+ self.timer.start()+ return {"completion_tokens": max_tokens,+ "tokens_per_second": 2.0,+ "cache_hit_percent": 50.0,+ "rss_gb": 4.0,+ "prompt_tokens": 3,+ "length_limited": True}++ engine = DelayedProfileEngine()+ result = datapoint._measure_persistent_request(engine, "prompt", 8)+ engine.timer.join()+ self.assertEqual(engine.asserted_slot, 0)+ self.assertEqual(result["profile"]["expert_disk_s"], 1.25)++ def test_sister_engines_use_physical_cores_without_overriding_user_value(self):+ family = SimpleNamespace(id="qwen36")+ with mock.patch.dict(os.environ, {}, clear=True):+ env = datapoint._persistent_environment(family, 128, 32, 16)+ self.assertEqual(env["OMP_NUM_THREADS"], "16")+ with mock.patch.dict(os.environ, {"OMP_NUM_THREADS": "6"}, clear=True):+ env = datapoint._persistent_environment(family, 128, 32, 16)+ self.assertEqual(env["OMP_NUM_THREADS"], "6")++ def test_persistent_mode_rejects_the_cli_wrapper(self):+ with self.assertRaisesRegex(ValueError, "engine binary"):+ datapoint._persistent_executable(None, self.family, "./coli")++ def test_parser_defaults_to_persistent_campaign(self):+ args = datapoint.build_parser().parse_args(["--snap", "/model"])+ self.assertEqual(args.mode, "persistent")+ self.assertEqual(args.warmup_runs, 1)+ self.assertEqual(args.warm_runs, 3)+ self.assertIsNone(args.engine)++ def test_every_registered_family_has_the_shared_persistent_adapter(self):+ expected = {"glm", "inkling", "kimi", "olmoe", "qwen36", "deepseek_v4"}+ self.assertTrue(expected.issubset({family.id for family in FAMILIES}))+ self.assertTrue(all(family.has_gateway_adapter for family in FAMILIES))++if __name__ == "__main__":unittest.main()diff --git a/c/tools/datapoint.py b/c/tools/datapoint.pyindex 12077ca8..c8b8ba0d 100644--- a/c/tools/datapoint.py+++ b/c/tools/datapoint.py@@ -1,34 +1,41 @@#!/usr/bin/env python3-"""datapoint.py - run the standard colibri machine datapoint.+"""datapoint.py - run the standard cross-engine colibri datapoint.-Measures what docs/benchmarks.md asks for - engine load, capped greedy-decode tok/s, and iobench disk figures - and prints a ready-to-paste-markdown datapoint block.+The default ``persistent`` mode starts one engine through the shared mux+``SERVE=1`` protocol and keeps it alive for the complete campaign: one cold+request, discarded warm-up request(s), then measured warm requests. Expert,+KV, GPU-residency and allocator state therefore survive exactly as they do in+``coli serve``. The engine's DONE frame supplies the real completion count,+decode tok/s, expert hit rate and RSS; no output re-tokenization is needed.-The decode number is exact: generation is capped at --max-new tokens,-so the token count is known without re-tokenizing the output.+``--mode fresh-process`` preserves the historical runner for startup/cold-cache+work. In that mode every row is a new process, so a row labelled warm means+only that the operating-system page cache may be warm -- the in-process expert+cache is empty again. Keeping the two modes explicit prevents those different+states from being compared as if they were the same benchmark.-Cold vs warm: before measuring, the page cache is evicted (macOS: `purge`-if permitted, otherwise by writing a temp file larger than RAM, which-forces resident pages out). The first decode run is then a true cold-run; the second is warm. iobench cold runs right after eviction, before-the decode runs can re-warm the container. Use --no-evict to skip the-eviction (e.g. when cache warmth is irrelevant or eviction is unwanted).+Before either campaign, the page cache is evicted when possible (macOS:+``purge``; Linux: ``posix_fadvise(DONTNEED)`` over the model shards). iobench+runs immediately after eviction, before model loading can re-warm the files.Usage:- python tools/datapoint.py --snap models/olmoe_merged [--engine ./olmoe]- [--shard models/olmoe_merged/model-00000.safetensors]- [--max-new 128] [--warm-runs 1]+ python tools/datapoint.py --snap models/olmoe_merged+ python tools/datapoint.py --snap models/v4 --engine ./deepseek_v4 \+ --warm-runs 3+ python tools/datapoint.py --mode fresh-process --snap models/olmoe_merged \+ --engine ./olmoe"""import argparseimport osimport platformimport re+import statisticsimport subprocessimport sysimport tempfileimport time+from pathlib import PathLOAD_RE = re.compile(r"(?:resident weights loaded in|init done in|loaded \d+ layers in|\bloaded\b.*?\bin\b)\s+([\d.]+)s.*?\bRSS\b\s*(?:after load:\s*)?([\d.]+)\s*GB",@@ -40,6 +47,9 @@ V4_TOKENS_RE = re.compile(r"v4_tokens.*?generated=(\d+)", re.IGNORECASE)RAM_RE = re.compile(r"(?:projected=|dense=resident\(|available=)([\d.]+)G[iI]?B", re.IGNORECASE)IOBENCH_RE = re.compile(r"-> ([\d.]+) GB/s")+HERE = Path(__file__).resolve().parent+C_ROOT = HERE.parent+def machine_info():info = {}@@ -161,7 +171,162 @@ def evict_cache(ram_gb, snap_dir=None):return False-def run_engine(engine, snap, prompt, max_new, runs, cap, bits, memory_gb=None):+def _load_server_runtime():+ """Import the shared gateway from either a source tree or an installation."""+ root = str(C_ROOT)+ if root not in sys.path:+ sys.path.insert(0, root)+ import openai_server+ return openai_server+++def physical_core_count():+ """Use the launcher's cross-platform topology probe, without making it required."""+ try:+ root = str(C_ROOT)+ if root not in sys.path:+ sys.path.insert(0, root)+ from resource_plan import physical_cpu_count+ return physical_cpu_count()+ except (ImportError, OSError, ValueError):+ return None+++def _persistent_executable(runtime, family, engine):+ if engine:+ name = os.path.basename(engine).lower()+ if name in ("coli", "coli.exe", "coli-wrapper", "coli-wrapper.exe"):+ raise ValueError(+ "persistent mode needs an engine binary, not the coli wrapper; "+ "omit --engine to select the model family's engine automatically"+ )+ path = Path(engine).expanduser()+ else:+ path = Path(runtime.default_engine(family))+ path = path.resolve()+ if not path.is_file():+ raise FileNotFoundError(+ f"{family.display_name} engine is not built: {path} "+ f"(run: make -C c {family.build_target})"+ )+ return str(path)+++def _persistent_environment(family, max_new, memory_gb, physical_cores):+ env = dict(os.environ, COLI_TEMP="0", MAX_NEW=str(max_new), NGEN=str(max_new))+ if memory_gb is not None:+ env["RAM_GB"] = f"{memory_gb:g}"+ # Match the public coli launcher: sister engines use physical cores; V4's+ # runtime deliberately owns its compute/loader split (#897/#958).+ if (family.id != "deepseek_v4" and not env.get("COLI_NO_OMP_TUNE") and+ physical_cores):+ env.setdefault("OMP_NUM_THREADS", str(physical_cores))+ return env+++def _measure_persistent_request(engine, prompt, max_new):+ first_text_at = [None]+ start = time.monotonic()+ profile_seq = getattr(engine, "profile_seq", 0)++ def on_text(_text):+ if first_text_at[0] is None:+ first_text_at[0] = time.monotonic()++ stats = engine.generate(prompt, max_new, 0.0, 1.0, on_text, cache_slot=0)+ wall = time.monotonic() - start+ tok_count = int(stats["completion_tokens"])+ tok_s = float(stats["tokens_per_second"])+ # The engine's own decode rate is authoritative. This reconstructed+ # duration is useful for a compact table and avoids conflating prefill with+ # decode when the request wall includes both.+ gen_s = tok_count / tok_s if tok_count > 0 and tok_s > 0 else 0.0+ ttft_s = ((first_text_at[0] - start) if first_text_at[0] is not None else wall)++ # Some engines emit PROF immediately before DONE, while DeepSeek V4 emits+ # it immediately after. generate() returns on DONE, so give the dispatcher+ # a small bounded window to consume trailing telemetry. This wait is not+ # included in request wall/TTFT and does not affect the authoritative tok/s.+ profile_deadline = time.monotonic() + 0.1+ while (getattr(engine, "profile_seq", 0) <= profile_seq and+ time.monotonic() < profile_deadline):+ time.sleep(0.001)+ profile = None+ if (getattr(engine, "profile_seq", 0) > profile_seq and+ getattr(engine, "profile", None)):+ profile = dict(engine.profile[-1])+ return {+ "tokens": tok_count,+ "prompt_tokens": int(stats.get("prompt_tokens", 0)),+ "wall_s": wall,+ "request_s": wall,+ "ttft_s": ttft_s,+ "gen_s": gen_s,+ "tok_s": tok_s,+ "hit": float(stats.get("cache_hit_percent", 0.0)),+ "rss": float(stats.get("rss_gb", 0.0)),+ "length_limited": bool(stats.get("length_limited", False)),+ "profile": profile,+ }+++def run_persistent_engine(engine, snap, prompt, max_new, warmup_runs, warm_runs,+ cap, memory_gb=None, runtime=None, physical_cores=None):+ """Measure multiple requests against one shared SERVE=1 engine process."""+ runtime = runtime or _load_server_runtime()+ resolved = runtime.resolve_model(snap)+ family = resolved.descriptor+ if not family.has_gateway_adapter:+ raise ValueError(f"{family.display_name} has no persistent gateway adapter")+ executable = _persistent_executable(runtime, family, engine)+ if physical_cores is None:+ physical_cores = physical_core_count()+ env = _persistent_environment(family, max_new, memory_gb, physical_cores)+ runtime.ARCH = family.id+ rendered = runtime.render_chat_for_arch(+ [{"role": "user", "content": prompt}], enable_thinking=False)++ started = time.monotonic()+ persistent = runtime.Engine(executable, snap, cap=cap, max_tokens=max_new,+ env=env, kv_slots=1, family=family)+ load_s = time.monotonic() - started+ cold = []+ warmup = []+ warm = []+ try:+ print("[datapoint] persistent cold request 1/1", file=sys.stderr)+ cold.append(_measure_persistent_request(persistent, rendered, max_new))+ for index in range(warmup_runs):+ print(f"[datapoint] discarded warm-up {index + 1}/{warmup_runs}",+ file=sys.stderr)+ warmup.append(_measure_persistent_request(persistent, rendered, max_new))+ for index in range(warm_runs):+ print(f"[datapoint] measured warm request {index + 1}/{warm_runs}",+ file=sys.stderr)+ warm.append(_measure_persistent_request(persistent, rendered, max_new))+ tiers = dict(persistent.tiers) if getattr(persistent, "tiers", None) else None+ hwinfo = dict(persistent.hwinfo) if getattr(persistent, "hwinfo", None) else None+ finally:+ persistent.close()++ return {+ "mode": "persistent",+ "family": family.id,+ "family_name": family.display_name,+ "engine": executable,+ "load_s": load_s,+ "cold": cold,+ "warmup": warmup,+ "warm": warm,+ "tiers": tiers,+ "hwinfo": hwinfo,+ "omp_threads": env.get("OMP_NUM_THREADS", "engine runtime auto"),+ "loader_lanes": (env.get("V4_LOADER_LANES", "engine default")+ if family.id == "deepseek_v4" else "n/a"),+ }+++def run_fresh_engine(engine, snap, prompt, max_new, runs, cap, bits, memory_gb=None):results = []engine_name = os.path.basename(engine).lower()@@ -209,8 +374,11 @@ def run_engine(engine, snap, prompt, max_new, runs, cap, bits, memory_gb=None):rss = float(m_ram.group(1)) if m_ram else (float(memory_gb) if memory_gb is not None else 0.0)tok_s = tok_count / gen_s if gen_s > 0 else 0.0- results.append({"tokens": tok_count, "wall_s": wall, "gen_s": gen_s,- "tok_s": tok_s, "rss": rss, "load_s": load_s})+ results.append({"tokens": tok_count, "prompt_tokens": 0,+ "wall_s": wall, "request_s": max(0.0, wall - load_s),+ "ttft_s": None, "gen_s": gen_s, "tok_s": tok_s,+ "hit": None, "rss": rss, "load_s": load_s,+ "length_limited": tok_count >= max_new, "profile": None})return results@@ -221,63 +389,216 @@ def run_iobench(iobench, shard, mode):return float(m.group(1)) if m else None-def main():- ap = argparse.ArgumentParser()- ap.add_argument("--engine", default="./olmoe", help="engine binary (default ./olmoe)")+def build_parser():+ ap = argparse.ArgumentParser(description=__doc__)+ ap.add_argument("--mode", choices=("persistent", "fresh-process"),+ default="persistent",+ help="one long-lived serve engine (default), or one process per row")+ ap.add_argument("--engine", default=None,+ help="engine binary (default: select from model config.json)")ap.add_argument("--snap", required=True, help="model snapshot directory")ap.add_argument("--iobench", default="./iobench", help="iobench binary")ap.add_argument("--shard", default=None, help="shard file for iobench")ap.add_argument("--prompt", default="Continue this story: The lighthouse keeper climbed the stairs and saw something impossible in the fog. ",help="prompt used for the decode runs")ap.add_argument("--max-new", type=int, default=128, help="decode cap per run")- ap.add_argument("--warm-runs", type=int, default=1, help="warm decode runs after the cold run")+ ap.add_argument("--warmup-runs", type=int, default=1,+ help="discarded warm-up requests in persistent mode (default: 1)")+ ap.add_argument("--warm-runs", type=int, default=3,+ help="measured warm requests after warm-up (default: 3)")ap.add_argument("--cap", type=int, default=16, help="per-layer expert cache cap passed to the engine")- ap.add_argument("--bits", type=int, default=8, help="quant bits passed to the engine")- ap.add_argument("--memory-gb", type=float, default=None, help="RAM memory cap in GB passed to engine (default: auto-derived from host RAM)")+ ap.add_argument("--bits", type=int, default=8,+ help="quant bits passed by fresh-process legacy engines")+ ap.add_argument("--memory-gb", type=float, default=None,+ help="RAM cap in GB (default: auto-derived from host RAM)")ap.add_argument("--no-evict", action="store_true", help="skip page-cache eviction")- args = ap.parse_args()+ return ap+++def _median(rows, key):+ values = [row[key] for row in rows if row.get(key) is not None]+ return statistics.median(values) if values else None+++def _cell(value, suffix="", digits=2):+ if value is None:+ return "n/a"+ return f"{value:.{digits}f}{suffix}"+++def _append_result_row(out, label, result):+ out.append(+ f"| {label} | {result['prompt_tokens']} | {result['tokens']} | "+ f"{_cell(result['request_s'])} | {_cell(result['ttft_s'])} | "+ f"{_cell(result['tok_s'])} | {_cell(result['hit'], '%', 1)} | "+ f"{_cell(result['rss'], ' GB')} |"+ )+++def _append_profile_summary(out, warm):+ profiles = [row["profile"] for row in warm if row.get("profile")]+ if not profiles:+ return+ fields = (+ ("expert disk", "expert_disk_s"),+ ("expert wait", "expert_wait_s"),+ ("expert matmul", "expert_matmul_s"),+ ("attention", "attention_s"),+ ("lm head", "lm_head_s"),+ )+ out += ["", "**Warm profile medians (engine telemetry):**", "",+ "| phase | seconds/request |", "|---|---|"]+ for label, key in fields:+ values = [profile[key] for profile in profiles if key in profile]+ if values:+ out.append(f"| {label} | {statistics.median(values):.3f} |")+++def format_report(info, args, campaign, cold_label, disk):+ logical = info.get("cores") or "?"+ physical = info.get("physical_cores") or "?"+ engine_hw = campaign.get("hwinfo") or {}+ cpu = engine_hw.get("cpu") or info["cpu"]+ mode_note = (+ "one SERVE=1 process; expert/KV/GPU state retained across requests"+ if campaign["mode"] == "persistent" else+ "one process per row; warm rows retain OS page cache only"+ )+ family = campaign.get("family_name") or campaign.get("family") or "legacy"+ out = ["**Datapoint (automated runner)**", "", "| component | detail |",+ "|---|---|", f"| machine | {cpu} |",+ f"| RAM | {info['ram']} |",+ f"| CPU topology | {physical} physical / {logical} logical |",+ f"| OS | {info['os']} |",+ f"| mode | {campaign['mode']}: {mode_note} |",+ f"| engine | {os.path.basename(campaign['engine'])} ({family}) |",+ f"| snapshot | {args.snap} |",+ f"| cache settings | cap={args.cap}, RAM={args.memory_gb or info.get('ram_gb', '?')} GB |",+ f"| OpenMP team | {campaign.get('omp_threads', 'process default')} |",+ f"| V4 loader lanes | {campaign.get('loader_lanes', 'n/a')} |",+ f"| engine load | {campaign['load_s']:.2f} s |"]+ if engine_hw.get("gpu"):+ out.append(f"| GPU | {engine_hw['gpu']} ({engine_hw.get('gpus', '?')} device(s), "+ f"{engine_hw.get('vram_total_gb', 0):.2f} GB VRAM) |")+ tiers = campaign.get("tiers")+ if tiers:+ out.append("| expert tiers | "+ f"VRAM {tiers['vram']} / RAM {tiers['ram']} / disk {tiers['disk']} "+ f"({tiers['vram_gb']:.2f} + {tiers['ram_gb']:.2f} GB resident) |")++ count_note = ("actual counts from engine" if campaign["mode"] == "persistent"+ else "legacy count inference")+ out += ["", f"**Decode (COLI_TEMP=0, capped at {args.max_new} tokens; "+ f"{count_note}):**", "",+ "| phase | prompt tok | completion tok | request s | TTFT s | decode tok/s | hit | RSS |",+ "|---|---:|---:|---:|---:|---:|---:|---:|"]+ for result in campaign["cold"]:+ _append_result_row(out, cold_label, result)+ for index, result in enumerate(campaign.get("warmup", ())):+ _append_result_row(out, f"warm-up {index + 1} (discarded)", result)+ for index, result in enumerate(campaign["warm"]):+ _append_result_row(out, f"warm {index + 1}", result)++ warm = campaign["warm"]+ if warm:+ tok_values = [row["tok_s"] for row in warm]+ sigma = statistics.pstdev(tok_values) if len(tok_values) > 1 else 0.0+ median_row = {+ "prompt_tokens": round(_median(warm, "prompt_tokens") or 0),+ "tokens": round(_median(warm, "tokens") or 0),+ "request_s": _median(warm, "request_s"),+ "ttft_s": _median(warm, "ttft_s"),+ "tok_s": _median(warm, "tok_s"),+ "hit": _median(warm, "hit"),+ "rss": _median(warm, "rss"),+ }+ _append_result_row(out, f"**warm median (n={len(warm)}, sigma={sigma:.3f} tok/s)**",+ median_row)+ _append_profile_summary(out, warm)++ if campaign["mode"] == "persistent":+ out += ["", "Persistent rows intentionally reuse the same engine process and cache slot;",+ "the warm figures therefore represent serving state, including any valid KV-prefix reuse."]+ else:+ out += ["", "Fresh-process warm rows do not retain the engine's expert, KV, allocator, or GPU state."]++ if args.shard:+ out += ["", "**Disk (iobench, 19 MB x 64, 8 threads):**", "",+ "| mode | GB/s |", "|---|---|"]+ for name, val in disk.items():+ out.append(f"| {name} | {val:.2f} |" if val is not None else f"| {name} | n/a |")+ if sys.platform == "darwin":+ out += ["", "Note: macOS has no O_DIRECT; iobench uses F_NOCACHE, which stops new caching",+ "but cannot evict pages already resident. The cold figure was taken immediately",+ "after the attempted cache eviction, before the engine loaded the container."]+ return "\n".join(out)+++def _auto_engine_for_fresh(snap):+ runtime = _load_server_runtime()+ family = runtime.resolve_model(snap).descriptor+ return _persistent_executable(runtime, family, None), family+++def main(argv=None):+ ap = build_parser()+ args = ap.parse_args(argv)+ if args.max_new < 1:+ ap.error("--max-new must be positive")+ if args.warmup_runs < 0:+ ap.error("--warmup-runs cannot be negative")+ if args.warm_runs < 1:+ ap.error("--warm-runs must be positive")+ if args.memory_gb is not None and args.memory_gb <= 0:+ ap.error("--memory-gb must be positive")info = machine_info()+ info["physical_cores"] = physical_core_count()mem_gb = args.memory_gb if args.memory_gb is not None else info.get("ram_gb", 32.0)evicted = False if args.no_evict else evict_cache(info.get("ram_gb", 8.0), snap_dir=args.snap)- cold_label = "cold (cache evicted)" if evicted else "cold (cache not evicted)"+ cold_label = ("cold request (cache evicted before engine load)" if evicted else+ "cold request (page cache retained)")disk = {}if args.shard and evicted:# true cold disk figure first, before decode re-warms the containerdisk["cold"] = run_iobench(args.iobench, args.shard, 1)- cold = run_engine(args.engine, args.snap, args.prompt, args.max_new, 1, args.cap, args.bits, memory_gb=mem_gb)- warm = run_engine(args.engine, args.snap, args.prompt, args.max_new, max(1, args.warm_runs), args.cap, args.bits, memory_gb=mem_gb)+ if args.mode == "persistent":+ campaign = run_persistent_engine(+ args.engine, args.snap, args.prompt, args.max_new,+ args.warmup_runs, args.warm_runs, args.cap,+ memory_gb=mem_gb, physical_cores=info["physical_cores"])+ else:+ family = None+ engine = args.engine+ if engine is None:+ engine, family = _auto_engine_for_fresh(args.snap)+ cold = run_fresh_engine(engine, args.snap, args.prompt, args.max_new, 1,+ args.cap, args.bits, memory_gb=mem_gb)+ warm = run_fresh_engine(engine, args.snap, args.prompt, args.max_new,+ args.warm_runs, args.cap, args.bits, memory_gb=mem_gb)+ campaign = {+ "mode": "fresh-process",+ "family": family.id if family else None,+ "family_name": family.display_name if family else None,+ "engine": str(Path(engine).expanduser()),+ "load_s": cold[0]["load_s"],+ "cold": cold,+ "warmup": [],+ "warm": warm,+ "tiers": None,+ "hwinfo": None,+ "omp_threads": os.environ.get("OMP_NUM_THREADS", "process default"),+ "loader_lanes": os.environ.get("V4_LOADER_LANES", "engine default"),+ }if args.shard:if "cold" not in disk:disk["cold"] = run_iobench(args.iobench, args.shard, 1)disk["buffered"] = run_iobench(args.iobench, args.shard, 0)- out = ["**Datapoint (automated runner)**", "", "| component | detail |",- "|---|---|", f"| machine | {info['cpu']} |",- f"| RAM | {info['ram']} |", f"| cores | {info['cores']} |",- f"| OS | {info['os']} |", f"| engine | {os.path.basename(args.engine)}, snapshot {args.snap}, cap={args.cap}, bits={args.bits} |",- "", f"**Decode (COLI_TEMP=0, capped at {args.max_new} tokens, exact count):**", "",- "| phase | tokens | gen s | tok/s | RSS after load |", "|---|---|---|---|---|"]- for r in cold:- out.append(f"| {cold_label} | {r['tokens']} | {r['gen_s']:.2f} | {r['tok_s']:.2f} | {r['rss']:.2f} GB |")- for r in warm:- out.append(f"| warm | {r['tokens']} | {r['gen_s']:.2f} | {r['tok_s']:.2f} | {r['rss']:.2f} GB |")- out.append(f"| load | - | {cold[0]['load_s']:.2f} | - | - |")-- if args.shard:- out += ["", "**Disk (iobench, 19 MB x 64, 8 threads):**", "",- "| mode | GB/s |", "|---|---|"]- for name, val in disk.items():- out.append(f"| {name} | {val:.2f} |" if val is not None else f"| {name} | n/a |")- if sys.platform == "darwin":- out += ["", "Note: macOS has no O_DIRECT; iobench uses F_NOCACHE, which stops new caching",- "but cannot evict pages already resident (doc caveat #86). The cold figure above was",- "taken right after cache eviction, before decode re-warmed the container."]-- print("\n".join(out))+ print(format_report(info, args, campaign, cold_label, disk))if __name__ == "__main__":diff --git a/docs/benchmarks.md b/docs/benchmarks.mdindex 9c43ff00..d3251235 100644--- a/docs/benchmarks.md+++ b/docs/benchmarks.md@@ -54,8 +54,9 @@ COLI_MODEL=/path/to/glm52_i4 ./coli chat# 3) full automated datapoint — machine info + cold/warm decode + disk, one command:python tools/datapoint.py --snap /path/to/model --shard /path/to/container/model-00000.safetensors-# (stdlib-only; evicts the page cache before the cold run, caps decode at --max-new-# tokens so tok/s is exact, and prints a ready-to-paste datapoint block)+# (stdlib-only; auto-selects GLM, Inkling, Kimi K3, OLMoE, Qwen3.6, or DeepSeek V4+# from config.json; evicts the page cache before engine load; then keeps one+# SERVE=1 engine alive for a discarded warm-up and 3 measured warm requests)# 4) record expert usage, then pin the hottest experts in your spare RAM:STATS=stats.txt ./coli chat@@ -65,6 +66,16 @@ PIN=stats.txt PIN_GB=20 ./coli chat # scale PIN_GB to your free RAM./coli bench```+The default datapoint measures serving behavior, not repeated startup: the same+engine process and cache slot are used for one cold request, one discarded+warm-up, and three measured warm requests. The reported completion count,+decode tok/s, expert hit rate, and RSS come from each engine's `DONE` frame.+This preserves expert-cache, KV-prefix, GPU-residency, and allocator state and+makes the warm median comparable across every registered model family. Use+`--mode fresh-process` only when measuring startup or OS page-cache effects;+that compatibility mode launches a new engine for every row and therefore does+not retain in-process caches.+## Back-of-envelope predictionsDecode is disk-bound: a cold token costs ~11.4 GB of expert reads; MTP