Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
bench: make rotating prompts the primary datapoint
JustVugg6c22f9879649
diff --git a/c/tests/test_datapoint.py b/c/tests/test_datapoint.pyindex 09a8b465..f5f069b4 100644--- a/c/tests/test_datapoint.py+++ b/c/tests/test_datapoint.py@@ -116,20 +116,26 @@ class PersistentDatapointTest(unittest.TestCase):f"rendered:{messages[0]['content']}:{enable_thinking}"),)- def test_default_campaign_reuses_one_engine_for_every_request(self):+ def test_default_campaign_uses_rotating_prompts_as_primary_workload(self):runtime = self._runtime()with tempfile.TemporaryDirectory() as tmp:executable = Path(tmp) / "deepseek_v4"executable.touch()with mock.patch.dict(os.environ, {}, clear=True):campaign = datapoint.run_persistent_engine(- str(executable), "/model", "hello", 8, 1, 3, 16,+ str(executable), "/model", "hello", 8, 0, 1, 16,memory_gb=64, runtime=runtime, physical_cores=8)self.assertEqual(len(self.instances), 1)engine = self.instances[0]- self.assertEqual(len(engine.calls), 5) # cold + warm-up + 3 measured- self.assertTrue(all(call[0] == "rendered:hello:False" for call in engine.calls))+ self.assertEqual(len(engine.calls), 6) # cold + identical + 4 rotating+ self.assertTrue(all(call[0] == "rendered:hello:False"+ for call in engine.calls[:2]))+ self.assertEqual(+ [call[0] for call in engine.calls[2:]],+ [f"rendered:{prompt}:False"+ for prompt in datapoint.DEFAULT_ROTATING_PROMPTS],+ )self.assertTrue(all(call[4] == 0 for call in engine.calls))self.assertTrue(engine.closed)self.assertEqual(engine.env["RAM_GB"], "64")@@ -137,11 +143,16 @@ class PersistentDatapointTest(unittest.TestCase):self.assertNotIn("OMP_NUM_THREADS", engine.env) # V4 owns its thread splitself.assertEqual(runtime.ARCH, "deepseek_v4")self.assertEqual(len(campaign["cold"]), 1)- self.assertEqual(len(campaign["warmup"]), 1)- self.assertEqual(len(campaign["warm"]), 3)- self.assertEqual([row["tok_s"] for row in campaign["warm"]], [3.0, 4.0, 5.0])+ self.assertEqual(len(campaign["warmup"]), 0)+ self.assertEqual(len(campaign["warm"]), 1)+ self.assertEqual(len(campaign["rotating"]), 4)+ self.assertEqual([row["tok_s"] for row in campaign["warm"]], [2.0])+ self.assertEqual([row["tok_s"] for row in campaign["rotating"]],+ [3.0, 4.0, 5.0, 6.0])self.assertEqual(campaign["warm"][0]["tokens"], 8)self.assertEqual(campaign["warm"][0]["prompt_tokens"], 17)+ self.assertEqual(campaign["rotating_prompt_count"], 4)+ self.assertEqual(campaign["rotating_prompt_source"], "built-in")def test_engine_is_closed_when_a_request_fails(self):instances = self.instances@@ -214,10 +225,49 @@ class PersistentDatapointTest(unittest.TestCase):def test_parser_defaults_to_persistent_campaign(self):args = datapoint.build_parser().parse_args(["--snap", "/model"])self.assertEqual(args.mode, "persistent")- self.assertEqual(args.warmup_runs, 1)- self.assertEqual(args.warm_runs, 3)+ self.assertEqual(args.warmup_runs, 0)+ self.assertEqual(args.warm_runs, 1)+ self.assertEqual(args.rotating_runs, 4)+ self.assertIsNone(args.rotating_prompts)self.assertIsNone(args.engine)+ def test_custom_rotating_suite_requires_two_distinct_nonempty_prompts(self):+ self.assertEqual(datapoint._rotating_prompt_suite(["one", "two"]),+ ("one", "two"))+ with self.assertRaisesRegex(ValueError, "two distinct"):+ datapoint._rotating_prompt_suite(["same", "same"])+ with self.assertRaisesRegex(ValueError, "two distinct"):+ datapoint._rotating_prompt_suite([])+ with self.assertRaisesRegex(ValueError, "non-empty"):+ datapoint._rotating_prompt_suite(["one", " "])++ def test_report_makes_rotating_primary_and_identical_an_upper_bound(self):+ args = SimpleNamespace(snap="/model", cap=16, memory_gb=64,+ max_new=8, shard=None)+ row = {"prompt_tokens": 4, "tokens": 8, "request_s": 2.0,+ "ttft_s": 0.5, "tok_s": 4.0, "hit": 60.0,+ "rss": 12.0, "profile": {"expert_disk_s": 0.25}}+ campaign = {+ "mode": "persistent", "family_name": "DeepSeek V4",+ "engine": "/tmp/deepseek_v4", "load_s": 1.0,+ "cold": [row], "warmup": [], "warm": [dict(row, tok_s=9.0)],+ "rotating": [dict(row, tok_s=value, request_s=10.0 - value)+ for value in (3.0, 4.0, 5.0, 6.0)],+ "rotating_prompt_count": 4, "rotating_prompt_source": "built-in",+ "tiers": None, "hwinfo": None, "omp_threads": "8",+ "loader_lanes": "engine default",+ }+ info = {"cpu": "test CPU", "ram": "64 GB", "ram_gb": 64,+ "cores": 16, "physical_cores": 8, "os": "test OS"}+ report = datapoint.format_report(+ info, args, campaign, "cold request", {})+ rotating = report.index("| **rotating prompts (primary)** |")+ identical = report.index("| warm-identical (upper bound) |")+ self.assertLess(rotating, identical)+ self.assertIn("| **rotating prompts (primary)** | 4 | 4.50 |", report)+ self.assertIn("p95 request s", report)+ self.assertIn("4 built-in prompts, fixed rotation", report)+def test_every_registered_family_has_the_shared_persistent_adapter(self):expected = {"glm", "inkling", "kimi", "olmoe", "qwen36", "deepseek_v4"}self.assertTrue(expected.issubset({family.id for family in FAMILIES}))diff --git a/c/tools/datapoint.py b/c/tools/datapoint.pyindex c8b8ba0d..ec155ca9 100644--- a/c/tools/datapoint.py+++ b/c/tools/datapoint.py@@ -3,10 +3,12 @@The default ``persistent`` mode starts one engine through the shared mux``SERVE=1`` protocol and keeps it alive for the complete campaign: one cold-request, discarded warm-up request(s), then measured warm requests. Expert,-KV, GPU-residency and allocator state therefore survive exactly as they do in-``coli serve``. The engine's DONE frame supplies the real completion count,-decode tok/s, expert hit rate and RSS; no output re-tokenization is needed.+request, a repeated-prompt warm upper bound, then a deterministic rotation of+different prompts as the primary serving workload. Expert, KV, GPU-residency+and allocator state therefore survive exactly as they do in ``coli serve``,+without letting one repeated prompt define the headline result. The engine's+DONE frame supplies the real completion count, decode tok/s, expert hit rate+and RSS; no output re-tokenization is needed.``--mode fresh-process`` preserves the historical runner for startup/cold-cachework. In that mode every row is a new process, so a row labelled warm means@@ -21,7 +23,7 @@ runs immediately after eviction, before model loading can re-warm the files.Usage:python tools/datapoint.py --snap models/olmoe_mergedpython tools/datapoint.py --snap models/v4 --engine ./deepseek_v4 \- --warm-runs 3+ --rotating-runs 8python tools/datapoint.py --mode fresh-process --snap models/olmoe_merged \--engine ./olmoe"""@@ -50,6 +52,21 @@ IOBENCH_RE = re.compile(r"-> ([\d.]+) GB/s")HERE = Path(__file__).resolve().parentC_ROOT = HERE.parent+# Fixed order, no network corpus and no RNG: two machines run the same request+# mix. The prompts deliberately span reasoning, code, systems and multilingual+# prose so a MoE cache must serve changing routes instead of memorising one+# prompt. --rotate-prompt replaces this suite for model-specific campaigns.+DEFAULT_ROTATING_PROMPTS = (+ "Explain how a database transaction can deadlock, give a concrete example, "+ "and compare two practical prevention strategies in detail.",+ "Write a Python function that merges overlapping intervals. Explain its "+ "correctness, time complexity, edge cases, and include two examples.",+ "A service has high median throughput but poor tail latency under load. "+ "Develop a step-by-step investigation plan covering CPU, memory, and storage.",+ "Rispondi in italiano: confronta memoria virtuale e memoria fisica, spiegando "+ "page fault, cache del filesystem e un esempio pratico.",+)+def machine_info():info = {}@@ -270,9 +287,29 @@ def _measure_persistent_request(engine, prompt, max_new):}+def _rotating_prompt_suite(prompts=None):+ selected = tuple(DEFAULT_ROTATING_PROMPTS if prompts is None else prompts)+ if any(not isinstance(prompt, str) or not prompt.strip() for prompt in selected):+ raise ValueError("rotating prompts must be non-empty strings")+ if len(set(selected)) < 2:+ raise ValueError("rotating workload needs at least two distinct prompts")+ return selected+++def _render_prompt(runtime, prompt):+ return runtime.render_chat_for_arch(+ [{"role": "user", "content": prompt}], enable_thinking=False)++def run_persistent_engine(engine, snap, prompt, max_new, warmup_runs, warm_runs,- cap, memory_gb=None, runtime=None, physical_cores=None):- """Measure multiple requests against one shared SERVE=1 engine process."""+ cap, memory_gb=None, runtime=None, physical_cores=None,+ rotating_prompts=None, rotating_runs=4):+ """Measure repeated and rotating requests in one SERVE=1 process."""+ if rotating_runs < 2:+ raise ValueError("rotating_runs must be at least 2")+ rotating_prompt_source = ("custom" if rotating_prompts is not None+ else "built-in")+ rotating_prompts = _rotating_prompt_suite(rotating_prompts)runtime = runtime or _load_server_runtime()resolved = runtime.resolve_model(snap)family = resolved.descriptor@@ -283,8 +320,9 @@ def run_persistent_engine(engine, snap, prompt, max_new, warmup_runs, warm_runs,physical_cores = physical_core_count()env = _persistent_environment(family, max_new, memory_gb, physical_cores)runtime.ARCH = family.id- rendered = runtime.render_chat_for_arch(- [{"role": "user", "content": prompt}], enable_thinking=False)+ rendered = _render_prompt(runtime, prompt)+ rendered_rotating = [_render_prompt(runtime, item)+ for item in rotating_prompts]started = time.monotonic()persistent = runtime.Engine(executable, snap, cap=cap, max_tokens=max_new,@@ -293,6 +331,7 @@ def run_persistent_engine(engine, snap, prompt, max_new, warmup_runs, warm_runs,cold = []warmup = []warm = []+ rotating = []try:print("[datapoint] persistent cold request 1/1", file=sys.stderr)cold.append(_measure_persistent_request(persistent, rendered, max_new))@@ -301,9 +340,16 @@ def run_persistent_engine(engine, snap, prompt, max_new, warmup_runs, warm_runs,file=sys.stderr)warmup.append(_measure_persistent_request(persistent, rendered, max_new))for index in range(warm_runs):- print(f"[datapoint] measured warm request {index + 1}/{warm_runs}",+ print(f"[datapoint] warm-identical upper bound {index + 1}/{warm_runs}",file=sys.stderr)warm.append(_measure_persistent_request(persistent, rendered, max_new))+ for index in range(rotating_runs):+ prompt_index = index % len(rendered_rotating)+ print(f"[datapoint] rotating prompt {index + 1}/{rotating_runs} "+ f"(suite {prompt_index + 1}/{len(rendered_rotating)})",+ file=sys.stderr)+ rotating.append(_measure_persistent_request(+ persistent, rendered_rotating[prompt_index], max_new))tiers = dict(persistent.tiers) if getattr(persistent, "tiers", None) else Nonehwinfo = dict(persistent.hwinfo) if getattr(persistent, "hwinfo", None) else Nonefinally:@@ -318,6 +364,9 @@ def run_persistent_engine(engine, snap, prompt, max_new, warmup_runs, warm_runs,"cold": cold,"warmup": warmup,"warm": warm,+ "rotating": rotating,+ "rotating_prompt_count": len(rotating_prompts),+ "rotating_prompt_source": rotating_prompt_source,"tiers": tiers,"hwinfo": hwinfo,"omp_threads": env.get("OMP_NUM_THREADS", "engine runtime auto"),@@ -400,12 +449,16 @@ def build_parser():ap.add_argument("--iobench", default="./iobench", help="iobench binary")ap.add_argument("--shard", default=None, help="shard file for iobench")ap.add_argument("--prompt", default="Continue this story: The lighthouse keeper climbed the stairs and saw something impossible in the fog. ",- help="prompt used for the decode runs")+ help="prompt used for cold and warm-identical requests")+ ap.add_argument("--rotate-prompt", action="append", dest="rotating_prompts",+ help="custom rotating prompt; repeat at least twice to replace the built-in suite")ap.add_argument("--max-new", type=int, default=128, help="decode cap per run")- ap.add_argument("--warmup-runs", type=int, default=1,- help="discarded warm-up requests in persistent mode (default: 1)")- ap.add_argument("--warm-runs", type=int, default=3,- help="measured warm requests after warm-up (default: 3)")+ ap.add_argument("--warmup-runs", type=int, default=0,+ help="extra discarded identical-prompt warm-ups (default: 0; cold already warms it)")+ ap.add_argument("--warm-runs", type=int, default=1,+ help="measured identical-prompt upper-bound requests (default: 1)")+ ap.add_argument("--rotating-runs", type=int, default=4,+ help="measured changing-prompt requests (primary result; default: 4)")ap.add_argument("--cap", type=int, default=16, help="per-layer expert cache cap passed to the engine")ap.add_argument("--bits", type=int, default=8,help="quant bits passed by fresh-process legacy engines")@@ -420,6 +473,17 @@ def _median(rows, key):return statistics.median(values) if values else None+def _percentile(rows, key, quantile):+ values = sorted(row[key] for row in rows if row.get(key) is not None)+ if not values:+ return None+ position = (len(values) - 1) * quantile+ lower = int(position)+ upper = min(lower + 1, len(values) - 1)+ fraction = position - lower+ return values[lower] + (values[upper] - values[lower]) * fraction++def _cell(value, suffix="", digits=2):if value is None:return "n/a"@@ -435,8 +499,8 @@ def _append_result_row(out, label, result):)-def _append_profile_summary(out, warm):- profiles = [row["profile"] for row in warm if row.get("profile")]+def _append_profile_summary(out, label, rows):+ profiles = [row["profile"] for row in rows if row.get("profile")]if not profiles:returnfields = (@@ -446,12 +510,36 @@ def _append_profile_summary(out, warm):("attention", "attention_s"),("lm head", "lm_head_s"),)- out += ["", "**Warm profile medians (engine telemetry):**", "",+ out += ["", f"**{label} profile medians (engine telemetry):**", "","| phase | seconds/request |", "|---|---|"]- for label, key in fields:+ for phase, key in fields:values = [profile[key] for profile in profiles if key in profile]if values:- out.append(f"| {label} | {statistics.median(values):.3f} |")+ out.append(f"| {phase} | {statistics.median(values):.3f} |")+++def _append_workload_summary(out, campaign):+ persistent = campaign["mode"] == "persistent"+ workloads = []+ if persistent and campaign.get("rotating"):+ workloads.append(("**rotating prompts (primary)**", campaign["rotating"]))+ if campaign.get("warm"):+ label = ("warm-identical (upper bound)" if persistent else+ "fresh-process repeats")+ workloads.append((label, campaign["warm"]))+ if not workloads:+ return+ out += ["", "**Workload summary:**", "",+ "| workload | n | median tok/s | sigma tok/s | p95 request s | median hit |",+ "|---|---:|---:|---:|---:|---:|"]+ for label, rows in workloads:+ tok_values = [row["tok_s"] for row in rows]+ sigma = statistics.pstdev(tok_values) if len(tok_values) > 1 else 0.0+ out.append(+ f"| {label} | {len(rows)} | {_cell(_median(rows, 'tok_s'))} | "+ f"{sigma:.3f} | {_cell(_percentile(rows, 'request_s', 0.95))} | "+ f"{_cell(_median(rows, 'hit'), '%', 1)} |"+ )def format_report(info, args, campaign, cold_label, disk):@@ -485,6 +573,11 @@ def format_report(info, args, campaign, cold_label, disk):out.append("| expert tiers | "f"VRAM {tiers['vram']} / RAM {tiers['ram']} / disk {tiers['disk']} "f"({tiers['vram_gb']:.2f} + {tiers['ram_gb']:.2f} GB resident) |")+ if campaign["mode"] == "persistent":+ out.append("| prompt workload | "+ f"{campaign.get('rotating_prompt_count', 0)} "+ f"{campaign.get('rotating_prompt_source', 'unknown')} prompts, "+ "fixed rotation |")count_note = ("actual counts from engine" if campaign["mode"] == "persistent"else "legacy count inference")@@ -495,30 +588,28 @@ def format_report(info, args, campaign, cold_label, disk):for result in campaign["cold"]:_append_result_row(out, cold_label, result)for index, result in enumerate(campaign.get("warmup", ())):- _append_result_row(out, f"warm-up {index + 1} (discarded)", result)+ _append_result_row(out, f"identical warm-up {index + 1} (discarded)", result)for index, result in enumerate(campaign["warm"]):- _append_result_row(out, f"warm {index + 1}", result)-- warm = campaign["warm"]- if warm:- tok_values = [row["tok_s"] for row in warm]- sigma = statistics.pstdev(tok_values) if len(tok_values) > 1 else 0.0- median_row = {- "prompt_tokens": round(_median(warm, "prompt_tokens") or 0),- "tokens": round(_median(warm, "tokens") or 0),- "request_s": _median(warm, "request_s"),- "ttft_s": _median(warm, "ttft_s"),- "tok_s": _median(warm, "tok_s"),- "hit": _median(warm, "hit"),- "rss": _median(warm, "rss"),- }- _append_result_row(out, f"**warm median (n={len(warm)}, sigma={sigma:.3f} tok/s)**",- median_row)- _append_profile_summary(out, warm)+ label = (f"warm-identical {index + 1} (upper bound)"+ if campaign["mode"] == "persistent" else+ f"fresh-process repeat {index + 1}")+ _append_result_row(out, label, result)+ for index, result in enumerate(campaign.get("rotating", ())):+ _append_result_row(out, f"rotating {index + 1} (primary)", result)++ _append_workload_summary(out, campaign)+ if campaign.get("rotating"):+ _append_profile_summary(out, "Rotating workload", campaign["rotating"])+ if campaign.get("warm"):+ profile_label = ("Warm-identical upper bound"+ if campaign["mode"] == "persistent" else+ "Fresh-process repeats")+ _append_profile_summary(out, profile_label, campaign["warm"])if campaign["mode"] == "persistent":- out += ["", "Persistent rows intentionally reuse the same engine process and cache slot;",- "the warm figures therefore represent serving state, including any valid KV-prefix reuse."]+ out += ["", "Rotating prompts are the primary serving result: each request changes prompt",+ "while the engine and expert cache remain alive. Warm-identical intentionally reuses",+ "the same prompt and cache slot and is reported only as a cache/KV upper bound."]else:out += ["", "Fresh-process warm rows do not retain the engine's expert, KV, allocator, or GPU state."]@@ -551,6 +642,13 @@ def main(argv=None):ap.error("--warm-runs must be positive")if args.memory_gb is not None and args.memory_gb <= 0:ap.error("--memory-gb must be positive")+ if args.mode == "persistent":+ if args.rotating_runs < 2:+ ap.error("--rotating-runs must be at least 2")+ try:+ _rotating_prompt_suite(args.rotating_prompts)+ except ValueError as error:+ ap.error(str(error))info = machine_info()info["physical_cores"] = physical_core_count()@@ -568,7 +666,9 @@ def main(argv=None):campaign = run_persistent_engine(args.engine, args.snap, args.prompt, args.max_new,args.warmup_runs, args.warm_runs, args.cap,- memory_gb=mem_gb, physical_cores=info["physical_cores"])+ memory_gb=mem_gb, physical_cores=info["physical_cores"],+ rotating_prompts=args.rotating_prompts,+ rotating_runs=args.rotating_runs)else:family = Noneengine = args.engine@@ -587,6 +687,7 @@ def main(argv=None):"cold": cold,"warmup": [],"warm": warm,+ "rotating": [],"tiers": None,"hwinfo": None,"omp_threads": os.environ.get("OMP_NUM_THREADS", "process default"),diff --git a/docs/benchmarks.md b/docs/benchmarks.mdindex d3251235..2c8bedf0 100644--- a/docs/benchmarks.md+++ b/docs/benchmarks.md@@ -56,7 +56,7 @@ COLI_MODEL=/path/to/glm52_i4 ./coli chatpython tools/datapoint.py --snap /path/to/model --shard /path/to/container/model-00000.safetensors# (stdlib-only; auto-selects GLM, Inkling, Kimi K3, OLMoE, Qwen3.6, or DeepSeek V4# from config.json; evicts the page cache before engine load; then keeps one-# SERVE=1 engine alive for a discarded warm-up and 3 measured warm requests)+# SERVE=1 engine alive for cold, warm-identical, and rotating-prompt measurements)# 4) record expert usage, then pin the hottest experts in your spare RAM:STATS=stats.txt ./coli chat@@ -67,14 +67,19 @@ PIN=stats.txt PIN_GB=20 ./coli chat # scale PIN_GB to your free RAM```The default datapoint measures serving behavior, not repeated startup: the same-engine process and cache slot are used for one cold request, one discarded-warm-up, and three measured warm requests. The reported completion count,-decode tok/s, expert hit rate, and RSS come from each engine's `DONE` frame.-This preserves expert-cache, KV-prefix, GPU-residency, and allocator state and-makes the warm median comparable across every registered model family. Use-`--mode fresh-process` only when measuring startup or OS page-cache effects;-that compatibility mode launches a new engine for every row and therefore does-not retain in-process caches.+engine process and cache slot are used for one cold request, one repeated-prompt+warm request, and four requests drawn in fixed order from a diverse built-in+prompt suite. The **rotating-prompt median is the primary result** because the+expert cache must keep adapting as requests change. Warm-identical remains in+the report only as a useful cache/KV upper bound; it is not representative of a+mixed chat workload. The reported completion count, decode tok/s, expert hit+rate, and RSS come from each engine's `DONE` frame.++Use repeated `--rotate-prompt '...'` arguments to replace the built-in suite+(at least two distinct prompts), or `--rotating-runs N` to run more fixed-order+samples. Use `--mode fresh-process` only when measuring startup or OS page-cache+effects; that compatibility mode launches a new engine for every row and+therefore does not retain in-process caches.## Back-of-envelope predictions