Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

bench: make rotating prompts the primary datapoint

JustVugg6c22f9879649
diff --git a/c/tests/test_datapoint.py b/c/tests/test_datapoint.py
index 09a8b465..f5f069b4 100644
--- a/c/tests/test_datapoint.py
+++ b/c/tests/test_datapoint.py
@@ -116,20 +116,26 @@ class PersistentDatapointTest(unittest.TestCase):
f"rendered:{messages[0]['content']}:{enable_thinking}"),
)
- def test_default_campaign_reuses_one_engine_for_every_request(self):
+ def test_default_campaign_uses_rotating_prompts_as_primary_workload(self):
runtime = self._runtime()
with tempfile.TemporaryDirectory() as tmp:
executable = Path(tmp) / "deepseek_v4"
executable.touch()
with mock.patch.dict(os.environ, {}, clear=True):
campaign = datapoint.run_persistent_engine(
- str(executable), "/model", "hello", 8, 1, 3, 16,
+ str(executable), "/model", "hello", 8, 0, 1, 16,
memory_gb=64, runtime=runtime, physical_cores=8)
self.assertEqual(len(self.instances), 1)
engine = self.instances[0]
- self.assertEqual(len(engine.calls), 5) # cold + warm-up + 3 measured
- self.assertTrue(all(call[0] == "rendered:hello:False" for call in engine.calls))
+ self.assertEqual(len(engine.calls), 6) # cold + identical + 4 rotating
+ self.assertTrue(all(call[0] == "rendered:hello:False"
+ for call in engine.calls[:2]))
+ self.assertEqual(
+ [call[0] for call in engine.calls[2:]],
+ [f"rendered:{prompt}:False"
+ for prompt in datapoint.DEFAULT_ROTATING_PROMPTS],
+ )
self.assertTrue(all(call[4] == 0 for call in engine.calls))
self.assertTrue(engine.closed)
self.assertEqual(engine.env["RAM_GB"], "64")
@@ -137,11 +143,16 @@ class PersistentDatapointTest(unittest.TestCase):
self.assertNotIn("OMP_NUM_THREADS", engine.env) # V4 owns its thread split
self.assertEqual(runtime.ARCH, "deepseek_v4")
self.assertEqual(len(campaign["cold"]), 1)
- self.assertEqual(len(campaign["warmup"]), 1)
- self.assertEqual(len(campaign["warm"]), 3)
- self.assertEqual([row["tok_s"] for row in campaign["warm"]], [3.0, 4.0, 5.0])
+ self.assertEqual(len(campaign["warmup"]), 0)
+ self.assertEqual(len(campaign["warm"]), 1)
+ self.assertEqual(len(campaign["rotating"]), 4)
+ self.assertEqual([row["tok_s"] for row in campaign["warm"]], [2.0])
+ self.assertEqual([row["tok_s"] for row in campaign["rotating"]],
+ [3.0, 4.0, 5.0, 6.0])
self.assertEqual(campaign["warm"][0]["tokens"], 8)
self.assertEqual(campaign["warm"][0]["prompt_tokens"], 17)
+ self.assertEqual(campaign["rotating_prompt_count"], 4)
+ self.assertEqual(campaign["rotating_prompt_source"], "built-in")
def test_engine_is_closed_when_a_request_fails(self):
instances = self.instances
@@ -214,10 +225,49 @@ class PersistentDatapointTest(unittest.TestCase):
def test_parser_defaults_to_persistent_campaign(self):
args = datapoint.build_parser().parse_args(["--snap", "/model"])
self.assertEqual(args.mode, "persistent")
- self.assertEqual(args.warmup_runs, 1)
- self.assertEqual(args.warm_runs, 3)
+ self.assertEqual(args.warmup_runs, 0)
+ self.assertEqual(args.warm_runs, 1)
+ self.assertEqual(args.rotating_runs, 4)
+ self.assertIsNone(args.rotating_prompts)
self.assertIsNone(args.engine)
+ def test_custom_rotating_suite_requires_two_distinct_nonempty_prompts(self):
+ self.assertEqual(datapoint._rotating_prompt_suite(["one", "two"]),
+ ("one", "two"))
+ with self.assertRaisesRegex(ValueError, "two distinct"):
+ datapoint._rotating_prompt_suite(["same", "same"])
+ with self.assertRaisesRegex(ValueError, "two distinct"):
+ datapoint._rotating_prompt_suite([])
+ with self.assertRaisesRegex(ValueError, "non-empty"):
+ datapoint._rotating_prompt_suite(["one", " "])
+
+ def test_report_makes_rotating_primary_and_identical_an_upper_bound(self):
+ args = SimpleNamespace(snap="/model", cap=16, memory_gb=64,
+ max_new=8, shard=None)
+ row = {"prompt_tokens": 4, "tokens": 8, "request_s": 2.0,
+ "ttft_s": 0.5, "tok_s": 4.0, "hit": 60.0,
+ "rss": 12.0, "profile": {"expert_disk_s": 0.25}}
+ campaign = {
+ "mode": "persistent", "family_name": "DeepSeek V4",
+ "engine": "/tmp/deepseek_v4", "load_s": 1.0,
+ "cold": [row], "warmup": [], "warm": [dict(row, tok_s=9.0)],
+ "rotating": [dict(row, tok_s=value, request_s=10.0 - value)
+ for value in (3.0, 4.0, 5.0, 6.0)],
+ "rotating_prompt_count": 4, "rotating_prompt_source": "built-in",
+ "tiers": None, "hwinfo": None, "omp_threads": "8",
+ "loader_lanes": "engine default",
+ }
+ info = {"cpu": "test CPU", "ram": "64 GB", "ram_gb": 64,
+ "cores": 16, "physical_cores": 8, "os": "test OS"}
+ report = datapoint.format_report(
+ info, args, campaign, "cold request", {})
+ rotating = report.index("| **rotating prompts (primary)** |")
+ identical = report.index("| warm-identical (upper bound) |")
+ self.assertLess(rotating, identical)
+ self.assertIn("| **rotating prompts (primary)** | 4 | 4.50 |", report)
+ self.assertIn("p95 request s", report)
+ self.assertIn("4 built-in prompts, fixed rotation", report)
+
def test_every_registered_family_has_the_shared_persistent_adapter(self):
expected = {"glm", "inkling", "kimi", "olmoe", "qwen36", "deepseek_v4"}
self.assertTrue(expected.issubset({family.id for family in FAMILIES}))
diff --git a/c/tools/datapoint.py b/c/tools/datapoint.py
index c8b8ba0d..ec155ca9 100644
--- a/c/tools/datapoint.py
+++ b/c/tools/datapoint.py
@@ -3,10 +3,12 @@
The default ``persistent`` mode starts one engine through the shared mux
``SERVE=1`` protocol and keeps it alive for the complete campaign: one cold
-request, discarded warm-up request(s), then measured warm requests. Expert,
-KV, GPU-residency and allocator state therefore survive exactly as they do in
-``coli serve``. The engine's DONE frame supplies the real completion count,
-decode tok/s, expert hit rate and RSS; no output re-tokenization is needed.
+request, a repeated-prompt warm upper bound, then a deterministic rotation of
+different prompts as the primary serving workload. Expert, KV, GPU-residency
+and allocator state therefore survive exactly as they do in ``coli serve``,
+without letting one repeated prompt define the headline result. The engine's
+DONE frame supplies the real completion count, decode tok/s, expert hit rate
+and RSS; no output re-tokenization is needed.
``--mode fresh-process`` preserves the historical runner for startup/cold-cache
work. In that mode every row is a new process, so a row labelled warm means
@@ -21,7 +23,7 @@ runs immediately after eviction, before model loading can re-warm the files.
Usage:
python tools/datapoint.py --snap models/olmoe_merged
python tools/datapoint.py --snap models/v4 --engine ./deepseek_v4 \
- --warm-runs 3
+ --rotating-runs 8
python tools/datapoint.py --mode fresh-process --snap models/olmoe_merged \
--engine ./olmoe
"""
@@ -50,6 +52,21 @@ IOBENCH_RE = re.compile(r"-> ([\d.]+) GB/s")
HERE = Path(__file__).resolve().parent
C_ROOT = HERE.parent
+# Fixed order, no network corpus and no RNG: two machines run the same request
+# mix. The prompts deliberately span reasoning, code, systems and multilingual
+# prose so a MoE cache must serve changing routes instead of memorising one
+# prompt. --rotate-prompt replaces this suite for model-specific campaigns.
+DEFAULT_ROTATING_PROMPTS = (
+ "Explain how a database transaction can deadlock, give a concrete example, "
+ "and compare two practical prevention strategies in detail.",
+ "Write a Python function that merges overlapping intervals. Explain its "
+ "correctness, time complexity, edge cases, and include two examples.",
+ "A service has high median throughput but poor tail latency under load. "
+ "Develop a step-by-step investigation plan covering CPU, memory, and storage.",
+ "Rispondi in italiano: confronta memoria virtuale e memoria fisica, spiegando "
+ "page fault, cache del filesystem e un esempio pratico.",
+)
+
def machine_info():
info = {}
@@ -270,9 +287,29 @@ def _measure_persistent_request(engine, prompt, max_new):
}
+def _rotating_prompt_suite(prompts=None):
+ selected = tuple(DEFAULT_ROTATING_PROMPTS if prompts is None else prompts)
+ if any(not isinstance(prompt, str) or not prompt.strip() for prompt in selected):
+ raise ValueError("rotating prompts must be non-empty strings")
+ if len(set(selected)) < 2:
+ raise ValueError("rotating workload needs at least two distinct prompts")
+ return selected
+
+
+def _render_prompt(runtime, prompt):
+ return runtime.render_chat_for_arch(
+ [{"role": "user", "content": prompt}], enable_thinking=False)
+
+
def run_persistent_engine(engine, snap, prompt, max_new, warmup_runs, warm_runs,
- cap, memory_gb=None, runtime=None, physical_cores=None):
- """Measure multiple requests against one shared SERVE=1 engine process."""
+ cap, memory_gb=None, runtime=None, physical_cores=None,
+ rotating_prompts=None, rotating_runs=4):
+ """Measure repeated and rotating requests in one SERVE=1 process."""
+ if rotating_runs < 2:
+ raise ValueError("rotating_runs must be at least 2")
+ rotating_prompt_source = ("custom" if rotating_prompts is not None
+ else "built-in")
+ rotating_prompts = _rotating_prompt_suite(rotating_prompts)
runtime = runtime or _load_server_runtime()
resolved = runtime.resolve_model(snap)
family = resolved.descriptor
@@ -283,8 +320,9 @@ def run_persistent_engine(engine, snap, prompt, max_new, warmup_runs, warm_runs,
physical_cores = physical_core_count()
env = _persistent_environment(family, max_new, memory_gb, physical_cores)
runtime.ARCH = family.id
- rendered = runtime.render_chat_for_arch(
- [{"role": "user", "content": prompt}], enable_thinking=False)
+ rendered = _render_prompt(runtime, prompt)
+ rendered_rotating = [_render_prompt(runtime, item)
+ for item in rotating_prompts]
started = time.monotonic()
persistent = runtime.Engine(executable, snap, cap=cap, max_tokens=max_new,
@@ -293,6 +331,7 @@ def run_persistent_engine(engine, snap, prompt, max_new, warmup_runs, warm_runs,
cold = []
warmup = []
warm = []
+ rotating = []
try:
print("[datapoint] persistent cold request 1/1", file=sys.stderr)
cold.append(_measure_persistent_request(persistent, rendered, max_new))
@@ -301,9 +340,16 @@ def run_persistent_engine(engine, snap, prompt, max_new, warmup_runs, warm_runs,
file=sys.stderr)
warmup.append(_measure_persistent_request(persistent, rendered, max_new))
for index in range(warm_runs):
- print(f"[datapoint] measured warm request {index + 1}/{warm_runs}",
+ print(f"[datapoint] warm-identical upper bound {index + 1}/{warm_runs}",
file=sys.stderr)
warm.append(_measure_persistent_request(persistent, rendered, max_new))
+ for index in range(rotating_runs):
+ prompt_index = index % len(rendered_rotating)
+ print(f"[datapoint] rotating prompt {index + 1}/{rotating_runs} "
+ f"(suite {prompt_index + 1}/{len(rendered_rotating)})",
+ file=sys.stderr)
+ rotating.append(_measure_persistent_request(
+ persistent, rendered_rotating[prompt_index], max_new))
tiers = dict(persistent.tiers) if getattr(persistent, "tiers", None) else None
hwinfo = dict(persistent.hwinfo) if getattr(persistent, "hwinfo", None) else None
finally:
@@ -318,6 +364,9 @@ def run_persistent_engine(engine, snap, prompt, max_new, warmup_runs, warm_runs,
"cold": cold,
"warmup": warmup,
"warm": warm,
+ "rotating": rotating,
+ "rotating_prompt_count": len(rotating_prompts),
+ "rotating_prompt_source": rotating_prompt_source,
"tiers": tiers,
"hwinfo": hwinfo,
"omp_threads": env.get("OMP_NUM_THREADS", "engine runtime auto"),
@@ -400,12 +449,16 @@ def build_parser():
ap.add_argument("--iobench", default="./iobench", help="iobench binary")
ap.add_argument("--shard", default=None, help="shard file for iobench")
ap.add_argument("--prompt", default="Continue this story: The lighthouse keeper climbed the stairs and saw something impossible in the fog. ",
- help="prompt used for the decode runs")
+ help="prompt used for cold and warm-identical requests")
+ ap.add_argument("--rotate-prompt", action="append", dest="rotating_prompts",
+ help="custom rotating prompt; repeat at least twice to replace the built-in suite")
ap.add_argument("--max-new", type=int, default=128, help="decode cap per run")
- ap.add_argument("--warmup-runs", type=int, default=1,
- help="discarded warm-up requests in persistent mode (default: 1)")
- ap.add_argument("--warm-runs", type=int, default=3,
- help="measured warm requests after warm-up (default: 3)")
+ ap.add_argument("--warmup-runs", type=int, default=0,
+ help="extra discarded identical-prompt warm-ups (default: 0; cold already warms it)")
+ ap.add_argument("--warm-runs", type=int, default=1,
+ help="measured identical-prompt upper-bound requests (default: 1)")
+ ap.add_argument("--rotating-runs", type=int, default=4,
+ help="measured changing-prompt requests (primary result; default: 4)")
ap.add_argument("--cap", type=int, default=16, help="per-layer expert cache cap passed to the engine")
ap.add_argument("--bits", type=int, default=8,
help="quant bits passed by fresh-process legacy engines")
@@ -420,6 +473,17 @@ def _median(rows, key):
return statistics.median(values) if values else None
+def _percentile(rows, key, quantile):
+ values = sorted(row[key] for row in rows if row.get(key) is not None)
+ if not values:
+ return None
+ position = (len(values) - 1) * quantile
+ lower = int(position)
+ upper = min(lower + 1, len(values) - 1)
+ fraction = position - lower
+ return values[lower] + (values[upper] - values[lower]) * fraction
+
+
def _cell(value, suffix="", digits=2):
if value is None:
return "n/a"
@@ -435,8 +499,8 @@ def _append_result_row(out, label, result):
)
-def _append_profile_summary(out, warm):
- profiles = [row["profile"] for row in warm if row.get("profile")]
+def _append_profile_summary(out, label, rows):
+ profiles = [row["profile"] for row in rows if row.get("profile")]
if not profiles:
return
fields = (
@@ -446,12 +510,36 @@ def _append_profile_summary(out, warm):
("attention", "attention_s"),
("lm head", "lm_head_s"),
)
- out += ["", "**Warm profile medians (engine telemetry):**", "",
+ out += ["", f"**{label} profile medians (engine telemetry):**", "",
"| phase | seconds/request |", "|---|---|"]
- for label, key in fields:
+ for phase, key in fields:
values = [profile[key] for profile in profiles if key in profile]
if values:
- out.append(f"| {label} | {statistics.median(values):.3f} |")
+ out.append(f"| {phase} | {statistics.median(values):.3f} |")
+
+
+def _append_workload_summary(out, campaign):
+ persistent = campaign["mode"] == "persistent"
+ workloads = []
+ if persistent and campaign.get("rotating"):
+ workloads.append(("**rotating prompts (primary)**", campaign["rotating"]))
+ if campaign.get("warm"):
+ label = ("warm-identical (upper bound)" if persistent else
+ "fresh-process repeats")
+ workloads.append((label, campaign["warm"]))
+ if not workloads:
+ return
+ out += ["", "**Workload summary:**", "",
+ "| workload | n | median tok/s | sigma tok/s | p95 request s | median hit |",
+ "|---|---:|---:|---:|---:|---:|"]
+ for label, rows in workloads:
+ tok_values = [row["tok_s"] for row in rows]
+ sigma = statistics.pstdev(tok_values) if len(tok_values) > 1 else 0.0
+ out.append(
+ f"| {label} | {len(rows)} | {_cell(_median(rows, 'tok_s'))} | "
+ f"{sigma:.3f} | {_cell(_percentile(rows, 'request_s', 0.95))} | "
+ f"{_cell(_median(rows, 'hit'), '%', 1)} |"
+ )
def format_report(info, args, campaign, cold_label, disk):
@@ -485,6 +573,11 @@ def format_report(info, args, campaign, cold_label, disk):
out.append("| expert tiers | "
f"VRAM {tiers['vram']} / RAM {tiers['ram']} / disk {tiers['disk']} "
f"({tiers['vram_gb']:.2f} + {tiers['ram_gb']:.2f} GB resident) |")
+ if campaign["mode"] == "persistent":
+ out.append("| prompt workload | "
+ f"{campaign.get('rotating_prompt_count', 0)} "
+ f"{campaign.get('rotating_prompt_source', 'unknown')} prompts, "
+ "fixed rotation |")
count_note = ("actual counts from engine" if campaign["mode"] == "persistent"
else "legacy count inference")
@@ -495,30 +588,28 @@ def format_report(info, args, campaign, cold_label, disk):
for result in campaign["cold"]:
_append_result_row(out, cold_label, result)
for index, result in enumerate(campaign.get("warmup", ())):
- _append_result_row(out, f"warm-up {index + 1} (discarded)", result)
+ _append_result_row(out, f"identical warm-up {index + 1} (discarded)", result)
for index, result in enumerate(campaign["warm"]):
- _append_result_row(out, f"warm {index + 1}", result)
-
- warm = campaign["warm"]
- if warm:
- tok_values = [row["tok_s"] for row in warm]
- sigma = statistics.pstdev(tok_values) if len(tok_values) > 1 else 0.0
- median_row = {
- "prompt_tokens": round(_median(warm, "prompt_tokens") or 0),
- "tokens": round(_median(warm, "tokens") or 0),
- "request_s": _median(warm, "request_s"),
- "ttft_s": _median(warm, "ttft_s"),
- "tok_s": _median(warm, "tok_s"),
- "hit": _median(warm, "hit"),
- "rss": _median(warm, "rss"),
- }
- _append_result_row(out, f"**warm median (n={len(warm)}, sigma={sigma:.3f} tok/s)**",
- median_row)
- _append_profile_summary(out, warm)
+ label = (f"warm-identical {index + 1} (upper bound)"
+ if campaign["mode"] == "persistent" else
+ f"fresh-process repeat {index + 1}")
+ _append_result_row(out, label, result)
+ for index, result in enumerate(campaign.get("rotating", ())):
+ _append_result_row(out, f"rotating {index + 1} (primary)", result)
+
+ _append_workload_summary(out, campaign)
+ if campaign.get("rotating"):
+ _append_profile_summary(out, "Rotating workload", campaign["rotating"])
+ if campaign.get("warm"):
+ profile_label = ("Warm-identical upper bound"
+ if campaign["mode"] == "persistent" else
+ "Fresh-process repeats")
+ _append_profile_summary(out, profile_label, campaign["warm"])
if campaign["mode"] == "persistent":
- out += ["", "Persistent rows intentionally reuse the same engine process and cache slot;",
- "the warm figures therefore represent serving state, including any valid KV-prefix reuse."]
+ out += ["", "Rotating prompts are the primary serving result: each request changes prompt",
+ "while the engine and expert cache remain alive. Warm-identical intentionally reuses",
+ "the same prompt and cache slot and is reported only as a cache/KV upper bound."]
else:
out += ["", "Fresh-process warm rows do not retain the engine's expert, KV, allocator, or GPU state."]
@@ -551,6 +642,13 @@ def main(argv=None):
ap.error("--warm-runs must be positive")
if args.memory_gb is not None and args.memory_gb <= 0:
ap.error("--memory-gb must be positive")
+ if args.mode == "persistent":
+ if args.rotating_runs < 2:
+ ap.error("--rotating-runs must be at least 2")
+ try:
+ _rotating_prompt_suite(args.rotating_prompts)
+ except ValueError as error:
+ ap.error(str(error))
info = machine_info()
info["physical_cores"] = physical_core_count()
@@ -568,7 +666,9 @@ def main(argv=None):
campaign = run_persistent_engine(
args.engine, args.snap, args.prompt, args.max_new,
args.warmup_runs, args.warm_runs, args.cap,
- memory_gb=mem_gb, physical_cores=info["physical_cores"])
+ memory_gb=mem_gb, physical_cores=info["physical_cores"],
+ rotating_prompts=args.rotating_prompts,
+ rotating_runs=args.rotating_runs)
else:
family = None
engine = args.engine
@@ -587,6 +687,7 @@ def main(argv=None):
"cold": cold,
"warmup": [],
"warm": warm,
+ "rotating": [],
"tiers": None,
"hwinfo": None,
"omp_threads": os.environ.get("OMP_NUM_THREADS", "process default"),
diff --git a/docs/benchmarks.md b/docs/benchmarks.md
index d3251235..2c8bedf0 100644
--- a/docs/benchmarks.md
+++ b/docs/benchmarks.md
@@ -56,7 +56,7 @@ COLI_MODEL=/path/to/glm52_i4 ./coli chat
python tools/datapoint.py --snap /path/to/model --shard /path/to/container/model-00000.safetensors
# (stdlib-only; auto-selects GLM, Inkling, Kimi K3, OLMoE, Qwen3.6, or DeepSeek V4
# from config.json; evicts the page cache before engine load; then keeps one
-# SERVE=1 engine alive for a discarded warm-up and 3 measured warm requests)
+# SERVE=1 engine alive for cold, warm-identical, and rotating-prompt measurements)
# 4) record expert usage, then pin the hottest experts in your spare RAM:
STATS=stats.txt ./coli chat
@@ -67,14 +67,19 @@ PIN=stats.txt PIN_GB=20 ./coli chat # scale PIN_GB to your free RAM
```
The default datapoint measures serving behavior, not repeated startup: the same
-engine process and cache slot are used for one cold request, one discarded
-warm-up, and three measured warm requests. The reported completion count,
-decode tok/s, expert hit rate, and RSS come from each engine's `DONE` frame.
-This preserves expert-cache, KV-prefix, GPU-residency, and allocator state and
-makes the warm median comparable across every registered model family. Use
-`--mode fresh-process` only when measuring startup or OS page-cache effects;
-that compatibility mode launches a new engine for every row and therefore does
-not retain in-process caches.
+engine process and cache slot are used for one cold request, one repeated-prompt
+warm request, and four requests drawn in fixed order from a diverse built-in
+prompt suite. The **rotating-prompt median is the primary result** because the
+expert cache must keep adapting as requests change. Warm-identical remains in
+the report only as a useful cache/KV upper bound; it is not representative of a
+mixed chat workload. The reported completion count, decode tok/s, expert hit
+rate, and RSS come from each engine's `DONE` frame.
+
+Use repeated `--rotate-prompt '...'` arguments to replace the built-in suite
+(at least two distinct prompts), or `--rotating-runs N` to run more fixed-order
+samples. Use `--mode fresh-process` only when measuring startup or OS page-cache
+effects; that compatibility mode launches a new engine for every row and
+therefore does not retain in-process caches.
## Back-of-envelope predictions