Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

bench: measure persistent cross-engine datapoints

JustVugg176a493c042d
diff --git a/c/tests/test_datapoint.py b/c/tests/test_datapoint.py
index ae1b6a81..09a8b465 100644
--- a/c/tests/test_datapoint.py
+++ b/c/tests/test_datapoint.py
@@ -6,10 +6,16 @@ measured warm and published as cold. These tests run on any host by
mocking sys.platform and the two win32 probes.
"""
import ctypes
+import os
import sys
+import tempfile
+import threading
import unittest
+from pathlib import Path
+from types import SimpleNamespace
from unittest import mock
+from family_registry import FAMILIES
from tools import datapoint
GB = 1073741824
@@ -49,5 +55,174 @@ class MachineInfoWin32Test(unittest.TestCase):
self.assertEqual(info["ram"], "?")
+class PersistentDatapointTest(unittest.TestCase):
+ def setUp(self):
+ self.family = SimpleNamespace(
+ id="deepseek_v4",
+ display_name="DeepSeek V4",
+ has_gateway_adapter=True,
+ build_target="deepseek_v4",
+ )
+ self.instances = []
+
+ def _runtime(self, engine_type=None):
+ instances = self.instances
+
+ class FakeEngine:
+ def __init__(self, executable, model, cap, max_tokens, env, kv_slots,
+ family):
+ self.executable = executable
+ self.model = model
+ self.cap = cap
+ self.max_tokens = max_tokens
+ self.env = env
+ self.kv_slots = kv_slots
+ self.family = family
+ self.calls = []
+ self.closed = False
+ self.tiers = {"vram": 2, "ram": 3, "disk": 4,
+ "vram_gb": 1.5, "ram_gb": 2.5}
+ self.hwinfo = {"cores": 16}
+ self.profile = []
+ self.profile_seq = 0
+ instances.append(self)
+
+ def generate(self, prompt, max_tokens, temperature, top_p, on_text,
+ cache_slot=0):
+ self.calls.append((prompt, max_tokens, temperature, top_p, cache_slot))
+ on_text("token")
+ self.profile.append({"expert_disk_s": 0.1,
+ "expert_wait_s": 0.2,
+ "expert_matmul_s": 0.3,
+ "attention_s": 0.4,
+ "lm_head_s": 0.5})
+ self.profile_seq += 1
+ return {"completion_tokens": max_tokens,
+ "tokens_per_second": float(len(self.calls)),
+ "cache_hit_percent": float(len(self.calls) * 10),
+ "rss_gb": 12.5,
+ "prompt_tokens": 17,
+ "length_limited": True}
+
+ def close(self):
+ self.closed = True
+
+ return SimpleNamespace(
+ ARCH=None,
+ Engine=engine_type or FakeEngine,
+ resolve_model=lambda _snap: SimpleNamespace(descriptor=self.family),
+ default_engine=lambda _family: "/unused/default-engine",
+ render_chat_for_arch=lambda messages, enable_thinking: (
+ f"rendered:{messages[0]['content']}:{enable_thinking}"),
+ )
+
+ def test_default_campaign_reuses_one_engine_for_every_request(self):
+ runtime = self._runtime()
+ with tempfile.TemporaryDirectory() as tmp:
+ executable = Path(tmp) / "deepseek_v4"
+ executable.touch()
+ with mock.patch.dict(os.environ, {}, clear=True):
+ campaign = datapoint.run_persistent_engine(
+ str(executable), "/model", "hello", 8, 1, 3, 16,
+ memory_gb=64, runtime=runtime, physical_cores=8)
+
+ self.assertEqual(len(self.instances), 1)
+ engine = self.instances[0]
+ self.assertEqual(len(engine.calls), 5) # cold + warm-up + 3 measured
+ self.assertTrue(all(call[0] == "rendered:hello:False" for call in engine.calls))
+ self.assertTrue(all(call[4] == 0 for call in engine.calls))
+ self.assertTrue(engine.closed)
+ self.assertEqual(engine.env["RAM_GB"], "64")
+ self.assertEqual(engine.env["COLI_TEMP"], "0")
+ self.assertNotIn("OMP_NUM_THREADS", engine.env) # V4 owns its thread split
+ self.assertEqual(runtime.ARCH, "deepseek_v4")
+ self.assertEqual(len(campaign["cold"]), 1)
+ self.assertEqual(len(campaign["warmup"]), 1)
+ self.assertEqual(len(campaign["warm"]), 3)
+ self.assertEqual([row["tok_s"] for row in campaign["warm"]], [3.0, 4.0, 5.0])
+ self.assertEqual(campaign["warm"][0]["tokens"], 8)
+ self.assertEqual(campaign["warm"][0]["prompt_tokens"], 17)
+
+ def test_engine_is_closed_when_a_request_fails(self):
+ instances = self.instances
+
+ class FailingEngine:
+ def __init__(self, *_args, **_kwargs):
+ self.profile_seq = 0
+ self.profile = []
+ self.closed = False
+ instances.append(self)
+
+ def generate(self, *_args, **_kwargs):
+ raise RuntimeError("generation failed")
+
+ def close(self):
+ self.closed = True
+
+ runtime = self._runtime(FailingEngine)
+ with tempfile.TemporaryDirectory() as tmp:
+ executable = Path(tmp) / "deepseek_v4"
+ executable.touch()
+ with self.assertRaisesRegex(RuntimeError, "generation failed"):
+ datapoint.run_persistent_engine(
+ str(executable), "/model", "hello", 8, 1, 3, 16,
+ runtime=runtime, physical_cores=8)
+ self.assertTrue(self.instances[0].closed)
+
+ def test_trailing_profile_after_done_is_attached_to_the_request(self):
+ class DelayedProfileEngine:
+ profile_seq = 0
+ profile = []
+
+ def generate(self, _prompt, max_tokens, _temperature, _top_p,
+ on_text, cache_slot=0):
+ self.asserted_slot = cache_slot
+ on_text("token")
+
+ def emit_profile():
+ self.profile = [{"expert_disk_s": 1.25}]
+ self.profile_seq += 1
+
+ self.timer = threading.Timer(0.01, emit_profile)
+ self.timer.start()
+ return {"completion_tokens": max_tokens,
+ "tokens_per_second": 2.0,
+ "cache_hit_percent": 50.0,
+ "rss_gb": 4.0,
+ "prompt_tokens": 3,
+ "length_limited": True}
+
+ engine = DelayedProfileEngine()
+ result = datapoint._measure_persistent_request(engine, "prompt", 8)
+ engine.timer.join()
+ self.assertEqual(engine.asserted_slot, 0)
+ self.assertEqual(result["profile"]["expert_disk_s"], 1.25)
+
+ def test_sister_engines_use_physical_cores_without_overriding_user_value(self):
+ family = SimpleNamespace(id="qwen36")
+ with mock.patch.dict(os.environ, {}, clear=True):
+ env = datapoint._persistent_environment(family, 128, 32, 16)
+ self.assertEqual(env["OMP_NUM_THREADS"], "16")
+ with mock.patch.dict(os.environ, {"OMP_NUM_THREADS": "6"}, clear=True):
+ env = datapoint._persistent_environment(family, 128, 32, 16)
+ self.assertEqual(env["OMP_NUM_THREADS"], "6")
+
+ def test_persistent_mode_rejects_the_cli_wrapper(self):
+ with self.assertRaisesRegex(ValueError, "engine binary"):
+ datapoint._persistent_executable(None, self.family, "./coli")
+
+ def test_parser_defaults_to_persistent_campaign(self):
+ args = datapoint.build_parser().parse_args(["--snap", "/model"])
+ self.assertEqual(args.mode, "persistent")
+ self.assertEqual(args.warmup_runs, 1)
+ self.assertEqual(args.warm_runs, 3)
+ self.assertIsNone(args.engine)
+
+ def test_every_registered_family_has_the_shared_persistent_adapter(self):
+ expected = {"glm", "inkling", "kimi", "olmoe", "qwen36", "deepseek_v4"}
+ self.assertTrue(expected.issubset({family.id for family in FAMILIES}))
+ self.assertTrue(all(family.has_gateway_adapter for family in FAMILIES))
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/c/tools/datapoint.py b/c/tools/datapoint.py
index 12077ca8..c8b8ba0d 100644
--- a/c/tools/datapoint.py
+++ b/c/tools/datapoint.py
@@ -1,34 +1,41 @@
#!/usr/bin/env python3
-"""datapoint.py - run the standard colibri machine datapoint.
+"""datapoint.py - run the standard cross-engine colibri datapoint.
-Measures what docs/benchmarks.md asks for - engine load, capped greedy
-decode tok/s, and iobench disk figures - and prints a ready-to-paste
-markdown datapoint block.
+The default ``persistent`` mode starts one engine through the shared mux
+``SERVE=1`` protocol and keeps it alive for the complete campaign: one cold
+request, discarded warm-up request(s), then measured warm requests. Expert,
+KV, GPU-residency and allocator state therefore survive exactly as they do in
+``coli serve``. The engine's DONE frame supplies the real completion count,
+decode tok/s, expert hit rate and RSS; no output re-tokenization is needed.
-The decode number is exact: generation is capped at --max-new tokens,
-so the token count is known without re-tokenizing the output.
+``--mode fresh-process`` preserves the historical runner for startup/cold-cache
+work. In that mode every row is a new process, so a row labelled warm means
+only that the operating-system page cache may be warm -- the in-process expert
+cache is empty again. Keeping the two modes explicit prevents those different
+states from being compared as if they were the same benchmark.
-Cold vs warm: before measuring, the page cache is evicted (macOS: `purge`
-if permitted, otherwise by writing a temp file larger than RAM, which
-forces resident pages out). The first decode run is then a true cold
-run; the second is warm. iobench cold runs right after eviction, before
-the decode runs can re-warm the container. Use --no-evict to skip the
-eviction (e.g. when cache warmth is irrelevant or eviction is unwanted).
+Before either campaign, the page cache is evicted when possible (macOS:
+``purge``; Linux: ``posix_fadvise(DONTNEED)`` over the model shards). iobench
+runs immediately after eviction, before model loading can re-warm the files.
Usage:
- python tools/datapoint.py --snap models/olmoe_merged [--engine ./olmoe]
- [--shard models/olmoe_merged/model-00000.safetensors]
- [--max-new 128] [--warm-runs 1]
+ python tools/datapoint.py --snap models/olmoe_merged
+ python tools/datapoint.py --snap models/v4 --engine ./deepseek_v4 \
+ --warm-runs 3
+ python tools/datapoint.py --mode fresh-process --snap models/olmoe_merged \
+ --engine ./olmoe
"""
import argparse
import os
import platform
import re
+import statistics
import subprocess
import sys
import tempfile
import time
+from pathlib import Path
LOAD_RE = re.compile(
r"(?:resident weights loaded in|init done in|loaded \d+ layers in|\bloaded\b.*?\bin\b)\s+([\d.]+)s.*?\bRSS\b\s*(?:after load:\s*)?([\d.]+)\s*GB",
@@ -40,6 +47,9 @@ V4_TOKENS_RE = re.compile(r"v4_tokens.*?generated=(\d+)", re.IGNORECASE)
RAM_RE = re.compile(r"(?:projected=|dense=resident\(|available=)([\d.]+)G[iI]?B", re.IGNORECASE)
IOBENCH_RE = re.compile(r"-> ([\d.]+) GB/s")
+HERE = Path(__file__).resolve().parent
+C_ROOT = HERE.parent
+
def machine_info():
info = {}
@@ -161,7 +171,162 @@ def evict_cache(ram_gb, snap_dir=None):
return False
-def run_engine(engine, snap, prompt, max_new, runs, cap, bits, memory_gb=None):
+def _load_server_runtime():
+ """Import the shared gateway from either a source tree or an installation."""
+ root = str(C_ROOT)
+ if root not in sys.path:
+ sys.path.insert(0, root)
+ import openai_server
+ return openai_server
+
+
+def physical_core_count():
+ """Use the launcher's cross-platform topology probe, without making it required."""
+ try:
+ root = str(C_ROOT)
+ if root not in sys.path:
+ sys.path.insert(0, root)
+ from resource_plan import physical_cpu_count
+ return physical_cpu_count()
+ except (ImportError, OSError, ValueError):
+ return None
+
+
+def _persistent_executable(runtime, family, engine):
+ if engine:
+ name = os.path.basename(engine).lower()
+ if name in ("coli", "coli.exe", "coli-wrapper", "coli-wrapper.exe"):
+ raise ValueError(
+ "persistent mode needs an engine binary, not the coli wrapper; "
+ "omit --engine to select the model family's engine automatically"
+ )
+ path = Path(engine).expanduser()
+ else:
+ path = Path(runtime.default_engine(family))
+ path = path.resolve()
+ if not path.is_file():
+ raise FileNotFoundError(
+ f"{family.display_name} engine is not built: {path} "
+ f"(run: make -C c {family.build_target})"
+ )
+ return str(path)
+
+
+def _persistent_environment(family, max_new, memory_gb, physical_cores):
+ env = dict(os.environ, COLI_TEMP="0", MAX_NEW=str(max_new), NGEN=str(max_new))
+ if memory_gb is not None:
+ env["RAM_GB"] = f"{memory_gb:g}"
+ # Match the public coli launcher: sister engines use physical cores; V4's
+ # runtime deliberately owns its compute/loader split (#897/#958).
+ if (family.id != "deepseek_v4" and not env.get("COLI_NO_OMP_TUNE") and
+ physical_cores):
+ env.setdefault("OMP_NUM_THREADS", str(physical_cores))
+ return env
+
+
+def _measure_persistent_request(engine, prompt, max_new):
+ first_text_at = [None]
+ start = time.monotonic()
+ profile_seq = getattr(engine, "profile_seq", 0)
+
+ def on_text(_text):
+ if first_text_at[0] is None:
+ first_text_at[0] = time.monotonic()
+
+ stats = engine.generate(prompt, max_new, 0.0, 1.0, on_text, cache_slot=0)
+ wall = time.monotonic() - start
+ tok_count = int(stats["completion_tokens"])
+ tok_s = float(stats["tokens_per_second"])
+ # The engine's own decode rate is authoritative. This reconstructed
+ # duration is useful for a compact table and avoids conflating prefill with
+ # decode when the request wall includes both.
+ gen_s = tok_count / tok_s if tok_count > 0 and tok_s > 0 else 0.0
+ ttft_s = ((first_text_at[0] - start) if first_text_at[0] is not None else wall)
+
+ # Some engines emit PROF immediately before DONE, while DeepSeek V4 emits
+ # it immediately after. generate() returns on DONE, so give the dispatcher
+ # a small bounded window to consume trailing telemetry. This wait is not
+ # included in request wall/TTFT and does not affect the authoritative tok/s.
+ profile_deadline = time.monotonic() + 0.1
+ while (getattr(engine, "profile_seq", 0) <= profile_seq and
+ time.monotonic() < profile_deadline):
+ time.sleep(0.001)
+ profile = None
+ if (getattr(engine, "profile_seq", 0) > profile_seq and
+ getattr(engine, "profile", None)):
+ profile = dict(engine.profile[-1])
+ return {
+ "tokens": tok_count,
+ "prompt_tokens": int(stats.get("prompt_tokens", 0)),
+ "wall_s": wall,
+ "request_s": wall,
+ "ttft_s": ttft_s,
+ "gen_s": gen_s,
+ "tok_s": tok_s,
+ "hit": float(stats.get("cache_hit_percent", 0.0)),
+ "rss": float(stats.get("rss_gb", 0.0)),
+ "length_limited": bool(stats.get("length_limited", False)),
+ "profile": profile,
+ }
+
+
+def run_persistent_engine(engine, snap, prompt, max_new, warmup_runs, warm_runs,
+ cap, memory_gb=None, runtime=None, physical_cores=None):
+ """Measure multiple requests against one shared SERVE=1 engine process."""
+ runtime = runtime or _load_server_runtime()
+ resolved = runtime.resolve_model(snap)
+ family = resolved.descriptor
+ if not family.has_gateway_adapter:
+ raise ValueError(f"{family.display_name} has no persistent gateway adapter")
+ executable = _persistent_executable(runtime, family, engine)
+ if physical_cores is None:
+ physical_cores = physical_core_count()
+ env = _persistent_environment(family, max_new, memory_gb, physical_cores)
+ runtime.ARCH = family.id
+ rendered = runtime.render_chat_for_arch(
+ [{"role": "user", "content": prompt}], enable_thinking=False)
+
+ started = time.monotonic()
+ persistent = runtime.Engine(executable, snap, cap=cap, max_tokens=max_new,
+ env=env, kv_slots=1, family=family)
+ load_s = time.monotonic() - started
+ cold = []
+ warmup = []
+ warm = []
+ try:
+ print("[datapoint] persistent cold request 1/1", file=sys.stderr)
+ cold.append(_measure_persistent_request(persistent, rendered, max_new))
+ for index in range(warmup_runs):
+ print(f"[datapoint] discarded warm-up {index + 1}/{warmup_runs}",
+ file=sys.stderr)
+ warmup.append(_measure_persistent_request(persistent, rendered, max_new))
+ for index in range(warm_runs):
+ print(f"[datapoint] measured warm request {index + 1}/{warm_runs}",
+ file=sys.stderr)
+ warm.append(_measure_persistent_request(persistent, rendered, max_new))
+ tiers = dict(persistent.tiers) if getattr(persistent, "tiers", None) else None
+ hwinfo = dict(persistent.hwinfo) if getattr(persistent, "hwinfo", None) else None
+ finally:
+ persistent.close()
+
+ return {
+ "mode": "persistent",
+ "family": family.id,
+ "family_name": family.display_name,
+ "engine": executable,
+ "load_s": load_s,
+ "cold": cold,
+ "warmup": warmup,
+ "warm": warm,
+ "tiers": tiers,
+ "hwinfo": hwinfo,
+ "omp_threads": env.get("OMP_NUM_THREADS", "engine runtime auto"),
+ "loader_lanes": (env.get("V4_LOADER_LANES", "engine default")
+ if family.id == "deepseek_v4" else "n/a"),
+ }
+
+
+def run_fresh_engine(engine, snap, prompt, max_new, runs, cap, bits, memory_gb=None):
results = []
engine_name = os.path.basename(engine).lower()
@@ -209,8 +374,11 @@ def run_engine(engine, snap, prompt, max_new, runs, cap, bits, memory_gb=None):
rss = float(m_ram.group(1)) if m_ram else (float(memory_gb) if memory_gb is not None else 0.0)
tok_s = tok_count / gen_s if gen_s > 0 else 0.0
- results.append({"tokens": tok_count, "wall_s": wall, "gen_s": gen_s,
- "tok_s": tok_s, "rss": rss, "load_s": load_s})
+ results.append({"tokens": tok_count, "prompt_tokens": 0,
+ "wall_s": wall, "request_s": max(0.0, wall - load_s),
+ "ttft_s": None, "gen_s": gen_s, "tok_s": tok_s,
+ "hit": None, "rss": rss, "load_s": load_s,
+ "length_limited": tok_count >= max_new, "profile": None})
return results
@@ -221,63 +389,216 @@ def run_iobench(iobench, shard, mode):
return float(m.group(1)) if m else None
-def main():
- ap = argparse.ArgumentParser()
- ap.add_argument("--engine", default="./olmoe", help="engine binary (default ./olmoe)")
+def build_parser():
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--mode", choices=("persistent", "fresh-process"),
+ default="persistent",
+ help="one long-lived serve engine (default), or one process per row")
+ ap.add_argument("--engine", default=None,
+ help="engine binary (default: select from model config.json)")
ap.add_argument("--snap", required=True, help="model snapshot directory")
ap.add_argument("--iobench", default="./iobench", help="iobench binary")
ap.add_argument("--shard", default=None, help="shard file for iobench")
ap.add_argument("--prompt", default="Continue this story: The lighthouse keeper climbed the stairs and saw something impossible in the fog. ",
help="prompt used for the decode runs")
ap.add_argument("--max-new", type=int, default=128, help="decode cap per run")
- ap.add_argument("--warm-runs", type=int, default=1, help="warm decode runs after the cold run")
+ ap.add_argument("--warmup-runs", type=int, default=1,
+ help="discarded warm-up requests in persistent mode (default: 1)")
+ ap.add_argument("--warm-runs", type=int, default=3,
+ help="measured warm requests after warm-up (default: 3)")
ap.add_argument("--cap", type=int, default=16, help="per-layer expert cache cap passed to the engine")
- ap.add_argument("--bits", type=int, default=8, help="quant bits passed to the engine")
- ap.add_argument("--memory-gb", type=float, default=None, help="RAM memory cap in GB passed to engine (default: auto-derived from host RAM)")
+ ap.add_argument("--bits", type=int, default=8,
+ help="quant bits passed by fresh-process legacy engines")
+ ap.add_argument("--memory-gb", type=float, default=None,
+ help="RAM cap in GB (default: auto-derived from host RAM)")
ap.add_argument("--no-evict", action="store_true", help="skip page-cache eviction")
- args = ap.parse_args()
+ return ap
+
+
+def _median(rows, key):
+ values = [row[key] for row in rows if row.get(key) is not None]
+ return statistics.median(values) if values else None
+
+
+def _cell(value, suffix="", digits=2):
+ if value is None:
+ return "n/a"
+ return f"{value:.{digits}f}{suffix}"
+
+
+def _append_result_row(out, label, result):
+ out.append(
+ f"| {label} | {result['prompt_tokens']} | {result['tokens']} | "
+ f"{_cell(result['request_s'])} | {_cell(result['ttft_s'])} | "
+ f"{_cell(result['tok_s'])} | {_cell(result['hit'], '%', 1)} | "
+ f"{_cell(result['rss'], ' GB')} |"
+ )
+
+
+def _append_profile_summary(out, warm):
+ profiles = [row["profile"] for row in warm if row.get("profile")]
+ if not profiles:
+ return
+ fields = (
+ ("expert disk", "expert_disk_s"),
+ ("expert wait", "expert_wait_s"),
+ ("expert matmul", "expert_matmul_s"),
+ ("attention", "attention_s"),
+ ("lm head", "lm_head_s"),
+ )
+ out += ["", "**Warm profile medians (engine telemetry):**", "",
+ "| phase | seconds/request |", "|---|---|"]
+ for label, key in fields:
+ values = [profile[key] for profile in profiles if key in profile]
+ if values:
+ out.append(f"| {label} | {statistics.median(values):.3f} |")
+
+
+def format_report(info, args, campaign, cold_label, disk):
+ logical = info.get("cores") or "?"
+ physical = info.get("physical_cores") or "?"
+ engine_hw = campaign.get("hwinfo") or {}
+ cpu = engine_hw.get("cpu") or info["cpu"]
+ mode_note = (
+ "one SERVE=1 process; expert/KV/GPU state retained across requests"
+ if campaign["mode"] == "persistent" else
+ "one process per row; warm rows retain OS page cache only"
+ )
+ family = campaign.get("family_name") or campaign.get("family") or "legacy"
+ out = ["**Datapoint (automated runner)**", "", "| component | detail |",
+ "|---|---|", f"| machine | {cpu} |",
+ f"| RAM | {info['ram']} |",
+ f"| CPU topology | {physical} physical / {logical} logical |",
+ f"| OS | {info['os']} |",
+ f"| mode | {campaign['mode']}: {mode_note} |",
+ f"| engine | {os.path.basename(campaign['engine'])} ({family}) |",
+ f"| snapshot | {args.snap} |",
+ f"| cache settings | cap={args.cap}, RAM={args.memory_gb or info.get('ram_gb', '?')} GB |",
+ f"| OpenMP team | {campaign.get('omp_threads', 'process default')} |",
+ f"| V4 loader lanes | {campaign.get('loader_lanes', 'n/a')} |",
+ f"| engine load | {campaign['load_s']:.2f} s |"]
+ if engine_hw.get("gpu"):
+ out.append(f"| GPU | {engine_hw['gpu']} ({engine_hw.get('gpus', '?')} device(s), "
+ f"{engine_hw.get('vram_total_gb', 0):.2f} GB VRAM) |")
+ tiers = campaign.get("tiers")
+ if tiers:
+ out.append("| expert tiers | "
+ f"VRAM {tiers['vram']} / RAM {tiers['ram']} / disk {tiers['disk']} "
+ f"({tiers['vram_gb']:.2f} + {tiers['ram_gb']:.2f} GB resident) |")
+
+ count_note = ("actual counts from engine" if campaign["mode"] == "persistent"
+ else "legacy count inference")
+ out += ["", f"**Decode (COLI_TEMP=0, capped at {args.max_new} tokens; "
+ f"{count_note}):**", "",
+ "| phase | prompt tok | completion tok | request s | TTFT s | decode tok/s | hit | RSS |",
+ "|---|---:|---:|---:|---:|---:|---:|---:|"]
+ for result in campaign["cold"]:
+ _append_result_row(out, cold_label, result)
+ for index, result in enumerate(campaign.get("warmup", ())):
+ _append_result_row(out, f"warm-up {index + 1} (discarded)", result)
+ for index, result in enumerate(campaign["warm"]):
+ _append_result_row(out, f"warm {index + 1}", result)
+
+ warm = campaign["warm"]
+ if warm:
+ tok_values = [row["tok_s"] for row in warm]
+ sigma = statistics.pstdev(tok_values) if len(tok_values) > 1 else 0.0
+ median_row = {
+ "prompt_tokens": round(_median(warm, "prompt_tokens") or 0),
+ "tokens": round(_median(warm, "tokens") or 0),
+ "request_s": _median(warm, "request_s"),
+ "ttft_s": _median(warm, "ttft_s"),
+ "tok_s": _median(warm, "tok_s"),
+ "hit": _median(warm, "hit"),
+ "rss": _median(warm, "rss"),
+ }
+ _append_result_row(out, f"**warm median (n={len(warm)}, sigma={sigma:.3f} tok/s)**",
+ median_row)
+ _append_profile_summary(out, warm)
+
+ if campaign["mode"] == "persistent":
+ out += ["", "Persistent rows intentionally reuse the same engine process and cache slot;",
+ "the warm figures therefore represent serving state, including any valid KV-prefix reuse."]
+ else:
+ out += ["", "Fresh-process warm rows do not retain the engine's expert, KV, allocator, or GPU state."]
+
+ if args.shard:
+ out += ["", "**Disk (iobench, 19 MB x 64, 8 threads):**", "",
+ "| mode | GB/s |", "|---|---|"]
+ for name, val in disk.items():
+ out.append(f"| {name} | {val:.2f} |" if val is not None else f"| {name} | n/a |")
+ if sys.platform == "darwin":
+ out += ["", "Note: macOS has no O_DIRECT; iobench uses F_NOCACHE, which stops new caching",
+ "but cannot evict pages already resident. The cold figure was taken immediately",
+ "after the attempted cache eviction, before the engine loaded the container."]
+ return "\n".join(out)
+
+
+def _auto_engine_for_fresh(snap):
+ runtime = _load_server_runtime()
+ family = runtime.resolve_model(snap).descriptor
+ return _persistent_executable(runtime, family, None), family
+
+
+def main(argv=None):
+ ap = build_parser()
+ args = ap.parse_args(argv)
+ if args.max_new < 1:
+ ap.error("--max-new must be positive")
+ if args.warmup_runs < 0:
+ ap.error("--warmup-runs cannot be negative")
+ if args.warm_runs < 1:
+ ap.error("--warm-runs must be positive")
+ if args.memory_gb is not None and args.memory_gb <= 0:
+ ap.error("--memory-gb must be positive")
info = machine_info()
+ info["physical_cores"] = physical_core_count()
mem_gb = args.memory_gb if args.memory_gb is not None else info.get("ram_gb", 32.0)
evicted = False if args.no_evict else evict_cache(info.get("ram_gb", 8.0), snap_dir=args.snap)
- cold_label = "cold (cache evicted)" if evicted else "cold (cache not evicted)"
+ cold_label = ("cold request (cache evicted before engine load)" if evicted else
+ "cold request (page cache retained)")
disk = {}
if args.shard and evicted:
# true cold disk figure first, before decode re-warms the container
disk["cold"] = run_iobench(args.iobench, args.shard, 1)
- cold = run_engine(args.engine, args.snap, args.prompt, args.max_new, 1, args.cap, args.bits, memory_gb=mem_gb)
- warm = run_engine(args.engine, args.snap, args.prompt, args.max_new, max(1, args.warm_runs), args.cap, args.bits, memory_gb=mem_gb)
+ if args.mode == "persistent":
+ campaign = run_persistent_engine(
+ args.engine, args.snap, args.prompt, args.max_new,
+ args.warmup_runs, args.warm_runs, args.cap,
+ memory_gb=mem_gb, physical_cores=info["physical_cores"])
+ else:
+ family = None
+ engine = args.engine
+ if engine is None:
+ engine, family = _auto_engine_for_fresh(args.snap)
+ cold = run_fresh_engine(engine, args.snap, args.prompt, args.max_new, 1,
+ args.cap, args.bits, memory_gb=mem_gb)
+ warm = run_fresh_engine(engine, args.snap, args.prompt, args.max_new,
+ args.warm_runs, args.cap, args.bits, memory_gb=mem_gb)
+ campaign = {
+ "mode": "fresh-process",
+ "family": family.id if family else None,
+ "family_name": family.display_name if family else None,
+ "engine": str(Path(engine).expanduser()),
+ "load_s": cold[0]["load_s"],
+ "cold": cold,
+ "warmup": [],
+ "warm": warm,
+ "tiers": None,
+ "hwinfo": None,
+ "omp_threads": os.environ.get("OMP_NUM_THREADS", "process default"),
+ "loader_lanes": os.environ.get("V4_LOADER_LANES", "engine default"),
+ }
if args.shard:
if "cold" not in disk:
disk["cold"] = run_iobench(args.iobench, args.shard, 1)
disk["buffered"] = run_iobench(args.iobench, args.shard, 0)
- out = ["**Datapoint (automated runner)**", "", "| component | detail |",
- "|---|---|", f"| machine | {info['cpu']} |",
- f"| RAM | {info['ram']} |", f"| cores | {info['cores']} |",
- f"| OS | {info['os']} |", f"| engine | {os.path.basename(args.engine)}, snapshot {args.snap}, cap={args.cap}, bits={args.bits} |",
- "", f"**Decode (COLI_TEMP=0, capped at {args.max_new} tokens, exact count):**", "",
- "| phase | tokens | gen s | tok/s | RSS after load |", "|---|---|---|---|---|"]
- for r in cold:
- out.append(f"| {cold_label} | {r['tokens']} | {r['gen_s']:.2f} | {r['tok_s']:.2f} | {r['rss']:.2f} GB |")
- for r in warm:
- out.append(f"| warm | {r['tokens']} | {r['gen_s']:.2f} | {r['tok_s']:.2f} | {r['rss']:.2f} GB |")
- out.append(f"| load | - | {cold[0]['load_s']:.2f} | - | - |")
-
- if args.shard:
- out += ["", "**Disk (iobench, 19 MB x 64, 8 threads):**", "",
- "| mode | GB/s |", "|---|---|"]
- for name, val in disk.items():
- out.append(f"| {name} | {val:.2f} |" if val is not None else f"| {name} | n/a |")
- if sys.platform == "darwin":
- out += ["", "Note: macOS has no O_DIRECT; iobench uses F_NOCACHE, which stops new caching",
- "but cannot evict pages already resident (doc caveat #86). The cold figure above was",
- "taken right after cache eviction, before decode re-warmed the container."]
-
- print("\n".join(out))
+ print(format_report(info, args, campaign, cold_label, disk))
if __name__ == "__main__":
diff --git a/docs/benchmarks.md b/docs/benchmarks.md
index 9c43ff00..d3251235 100644
--- a/docs/benchmarks.md
+++ b/docs/benchmarks.md
@@ -54,8 +54,9 @@ COLI_MODEL=/path/to/glm52_i4 ./coli chat
# 3) full automated datapoint — machine info + cold/warm decode + disk, one command:
python tools/datapoint.py --snap /path/to/model --shard /path/to/container/model-00000.safetensors
-# (stdlib-only; evicts the page cache before the cold run, caps decode at --max-new
-# tokens so tok/s is exact, and prints a ready-to-paste datapoint block)
+# (stdlib-only; auto-selects GLM, Inkling, Kimi K3, OLMoE, Qwen3.6, or DeepSeek V4
+# from config.json; evicts the page cache before engine load; then keeps one
+# SERVE=1 engine alive for a discarded warm-up and 3 measured warm requests)
# 4) record expert usage, then pin the hottest experts in your spare RAM:
STATS=stats.txt ./coli chat
@@ -65,6 +66,16 @@ PIN=stats.txt PIN_GB=20 ./coli chat # scale PIN_GB to your free RAM
./coli bench
```
+The default datapoint measures serving behavior, not repeated startup: the same
+engine process and cache slot are used for one cold request, one discarded
+warm-up, and three measured warm requests. The reported completion count,
+decode tok/s, expert hit rate, and RSS come from each engine's `DONE` frame.
+This preserves expert-cache, KV-prefix, GPU-residency, and allocator state and
+makes the warm median comparable across every registered model family. Use
+`--mode fresh-process` only when measuring startup or OS page-cache effects;
+that compatibility mode launches a new engine for every row and therefore does
+not retain in-process caches.
+
## Back-of-envelope predictions
Decode is disk-bound: a cold token costs ~11.4 GB of expert reads; MTP