Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
perf(tune): measure persistent rotating workloads
JustVugg5e4b247d8af7
diff --git a/c/autotune.py b/c/autotune.pyindex 3c5f267a..0fb2514e 100644--- a/c/autotune.py+++ b/c/autotune.py@@ -1,9 +1,10 @@"""Measured, quality-preserving execution tuning for colibri.The tuner changes execution scheduling only. It never sweeps quantization,-expert selection, sampling, or model weights. A calibration continuation is-generated once and then teacher-forced for every candidate so every run sees-the same token and routing inputs.+expert selection, sampling, or model weights. GLM uses its fixed-token replay+protocol. The sibling engines keep one serve process alive per candidate and+rotate deterministic prompts, so their expert caches behave like a real chat+instead of being reset between every sample."""from __future__ import annotations@@ -36,7 +37,7 @@ LATENCY_RE = re.compile(r"latency p50\s+([0-9.]+)\s*ms.*?p99\s+([0-9.]+)\s*ms")# as TOPK, TOPP, DRAFT, quantization and CACHE_ROUTE are intentionally excluded.TUNABLE_KEYS = frozenset({"OMP_NUM_THREADS", "COLI_NUMA", "PIPE", "DIRECT",- "COLI_CUDA_PIPE", "COLI_CUDA_ASYNC",+ "COLI_CUDA_PIPE", "COLI_CUDA_ASYNC", "V4_LOADER_LANES",})@@ -153,9 +154,10 @@ def candidate_steps(plan: dict, base_env: dict, arch: str = "glm") -> list[tuplesteps = []cores = max(1, int(plan.get("cpu", {}).get("physical_cores", os.cpu_count() or 1)))current_threads = int(base_env.get("OMP_NUM_THREADS", cores))- for threads in dict.fromkeys((cores, max(1, cores // 2))):- if threads != current_threads:- steps.append((f"omp-{threads}", {"OMP_NUM_THREADS": str(threads)}))+ if not base_env.get("COLI_NO_OMP_TUNE"):+ for threads in dict.fromkeys((cores, max(1, cores // 2))):+ if threads != current_threads:+ steps.append((f"omp-{threads}", {"OMP_NUM_THREADS": str(threads)}))sockets = int(plan.get("cpu", {}).get("sockets", 1))if sockets > 1 and base_env.get("COLI_NUMA") != "1":steps.append(("numa-on", {"COLI_NUMA": "1"}))@@ -173,6 +175,19 @@ def candidate_steps(plan: dict, base_env: dict, arch: str = "glm") -> list[tuplepipe = int(base_env.get("PIPE", "0"))if pipe != 1:steps.append(("io-pipe-on", {"PIPE": "1"}))+ if (arch == "deepseek_v4"+ and plan.get("tiers", {}).get("disk", {}).get("cold_expert_bytes", 0) > 0):+ # The pool default (9) was measured on one CPU/NVMe pair. GPU-tier+ # systems often prefer 3 because a deeper reader pool contends with+ # bank refill, while other SSDs keep scaling past 9. Sweep a bounded+ # set around both measured optima; lanes block in pread and therefore+ # do not need one CPU apiece.+ current_lanes = int(base_env.get("V4_LOADER_LANES", "9"))+ lane_values = (3, 6, 9, 12)+ for lanes in lane_values:+ if lanes != current_lanes:+ steps.append((f"v4-loader-{lanes}",+ {"V4_LOADER_LANES": str(lanes)}))return steps@@ -199,6 +214,24 @@ def parse_replay(output: str) -> dict:}+def _summarize_measurement(name: str, overlay: dict, samples: list[dict]) -> dict:+ """Reduce one candidate's fixed request budget to comparable medians."""+ return {+ "name": name,+ "env": dict(overlay),+ "tok_s": statistics.median(sample["tok_s"] for sample in samples),+ "hit_pct": statistics.median(+ sample["hit_pct"] for sample in samples+ if sample["hit_pct"] is not None+ ) if any(sample["hit_pct"] is not None for sample in samples) else None,+ "p99_ms": statistics.median(+ sample["p99_ms"] for sample in samples+ if sample["p99_ms"] is not None+ ) if any(sample["p99_ms"] is not None for sample in samples) else None,+ "samples": samples,+ }++class OutputDrift(RuntimeError):"""A run produced different tokens than the session's first run.@@ -234,7 +267,8 @@ def run_tune(engine: str, cap: int, base_env: dict, plan: dict, model: str,prompt: str, arch: str = "glm",tokens: int = 16, repeats: int = 2, timeout: int = 900,min_gain: float = 0.03, profile_dir: str | None = None,- progress=None, family=None, engine_cls=None) -> tuple[dict, Path]:+ progress=None, family=None, engine_cls=None,+ prompts=None) -> tuple[dict, Path]:"""Coordinate-descent tuning with a reverse-order confirmation gate."""progress = progress or (lambda _message: None)if cap is None:@@ -266,6 +300,10 @@ def run_tune(engine: str, cap: int, base_env: dict, plan: dict, model: str,else:replay = Nonecontext = contextlib.nullcontext()+ serving_prompts = tuple(prompts or (prompt,))+ if not serving_prompts or any(not isinstance(item, str) or not item+ for item in serving_prompts):+ raise ValueError("tuning prompts must be non-empty strings")with context as directory:if arch == "glm":ref = Path(directory) / "replay.json"@@ -284,51 +322,59 @@ def run_tune(engine: str, cap: int, base_env: dict, plan: dict, model: str,else:if engine_cls is None:from openai_server import Engine as engine_cls- expected = [] # bytes of the session's first run: the contract+ # One expected byte stream per prompt. Different prompts naturally+ # produce different text; the same prompt changing across+ # candidates is the quality violation.+ expected = {}- def run_once(name, overlay):+ def measure(name, overlay):env = dict(base_env)env.update(overlay)env["COLI_TEMP"] = "0"served = engine_cls(engine, model, cap, tokens, env, 1, family)- pieces = []+ samples = []try:- result = served.generate(prompt, tokens, 0.0, 1.0,- pieces.append)+ # Keep this engine alive for the whole candidate. `repeats`+ # is still the request budget, but requests now rotate+ # instead of repeatedly teaching the cache one prompt.+ for repeat in range(repeats):+ active_prompt = serving_prompts[repeat % len(serving_prompts)]+ progress(f"{name} ({repeat + 1}/{repeats}, prompt "+ f"{repeat % len(serving_prompts) + 1}/"+ f"{len(serving_prompts)})")+ pieces = []+ result = served.generate(active_prompt, tokens, 0.0, 1.0,+ pieces.append)+ text = "".join(pieces)+ contract = expected.get(active_prompt)+ if contract is None:+ expected[active_prompt] = text+ elif text != contract:+ raise OutputDrift(+ f"{name} changed the generated tokens for prompt "+ f"{repeat % len(serving_prompts) + 1}; a scheduling "+ f"knob must never do that")+ tok_s = float(result.get("tokens_per_second") or 0.0)+ if tok_s <= 0:+ raise RuntimeError(+ f"{name}: engine reported no decode speed")+ samples.append({+ "tok_s": tok_s,+ "hit_pct": result.get("cache_hit_percent"),+ "p50_ms": None,+ "p99_ms": None,+ })finally:served.close()- text = "".join(pieces)- if not expected:- expected.append(text)- elif text != expected[0]:- raise OutputDrift(- f"{name} changed the generated tokens; a scheduling "- f"knob must never do that")- tok_s = float(result.get("tokens_per_second") or 0.0)- if tok_s <= 0:- raise RuntimeError(- f"{name}: engine reported no decode speed")- return {"tok_s": tok_s,- "hit_pct": result.get("cache_hit_percent"),- "p50_ms": None, "p99_ms": None}-- def measure(name, overlay):- samples = []- for repeat in range(repeats):- progress(f"{name} ({repeat + 1}/{repeats})")- samples.append(run_once(name, overlay))- return {- "name": name,- "env": dict(overlay),- "tok_s": statistics.median(sample["tok_s"] for sample in samples),- "hit_pct": statistics.median(- sample["hit_pct"] for sample in samples if sample["hit_pct"] is not None- ) if any(sample["hit_pct"] is not None for sample in samples) else None,- "p99_ms": statistics.median(- sample["p99_ms"] for sample in samples if sample["p99_ms"] is not None- ) if any(sample["p99_ms"] is not None for sample in samples) else None,- "samples": samples,- }+ return _summarize_measurement(name, overlay, samples)++ if arch == "glm":+ def measure(name, overlay):+ samples = []+ for repeat in range(repeats):+ progress(f"{name} ({repeat + 1}/{repeats})")+ samples.append(run_once(name, overlay))+ return _summarize_measurement(name, overlay, samples)baseline = measure("baseline", {})winner = baselinediff --git a/c/coli b/c/coliindex 26d286e3..c87aca20 100755--- a/c/coli+++ b/c/coli@@ -71,6 +71,11 @@ except ModuleNotFoundError:from family_registry import (FamilyConfigError, UnknownFamilyError, all_families,family_by_id, resolve_model, tuning_replay_prompt)+_TUNE_ROTATION_PROMPT = (+ "A service has high median throughput but poor tail latency. Explain how "+ "you would distinguish a CPU, memory, and storage bottleneck."+)+# Run-in-place (source checkout, "cd c && ./coli ..."): the engine, the# support modules (resource_plan.py, doctor.py, autotune.py, openai_server.py) and# tools/ all live next to this script — unchanged from before.@@ -363,9 +368,10 @@ def ngen_for(a, interactive=False, family=None):return limits.interactive_max_output if interactive else limits.default_max_outputreturn 16384 if interactive else 1024-def env_for_engine(a, arch):+def env_for_engine(a, arch, plan=None):if arch == "glm":return env_for(a)+ explicit_env = set(os.environ)env = os.environ.copy()# OMP_NUM_THREADS for the sister engines that do not size their own team.#@@ -461,6 +467,40 @@ def env_for_engine(a, arch):sys.exit(f"{C.yel}--vram needs the CUDA build:{C.r} the engine binary is CPU-only")env["COLI_CUDA"] = "1"env["CUDA_EXPERT_GB"] = str(vram)+ if getattr(a, "auto_tier", False):+ # GLM has historically applied both the resource plan and its measured+ # profile inside env_for(). The sibling launcher returned above that+ # code path, so #1196 could measure and save a perfectly valid Kimi,+ # Inkling, OLMoE, Qwen or V4 profile that chat/serve never loaded.+ from resource_plan import build_plan, environment_for_plan+ if plan is None:+ ram,ctx,devices,vram=resource_request(a,env)+ plan=build_plan(a.model,ram,ctx,devices,vram,policy=a.policy,+ kv_slots=requested_kv_slots(a,env))+ # Preserve the existing non-GLM GPU contract: explicit --gpu/--vram+ # selects a capable build above. Merely asking for auto-tier must not+ # turn a CPU-only sibling binary into an attempted CUDA launch.+ plan_must_not_choose_omp=("OMP_NUM_THREADS" not in env+ and (arch == "deepseek_v4"+ or env.get("COLI_NO_OMP_TUNE")))+ env=environment_for_plan(plan,env,+ cuda_enabled=env.get("COLI_CUDA") == "1")+ if plan_must_not_choose_omp:+ # environment_for_plan supplies the physical-core default used by+ # the other engines. V4 deliberately computes logical CPUs minus+ # its loader reservation at startup; COLI_NO_OMP_TUNE likewise+ # promises no automatic team. Keep those baselines unless a+ # measured profile (V4 only, below) or the user explicitly chooses+ # a team.+ env.pop("OMP_NUM_THREADS",None)+ if not getattr(a, "no_tune_profile", False):+ from autotune import apply_profile, load_profile+ profile=load_profile(plan,a.model,engine_for(a.model))+ if profile:+ env=apply_profile(env,profile,explicit_env)+ gain=100.0*profile["gain"]+ print(f" {C.dim}[TUNE] applied measured profile · +{gain:.1f}% "+ f"calibration throughput{C.r}",file=sys.stderr)return envdef dsv4_cuda_available(model=None):@@ -1015,13 +1055,24 @@ def cmd_tune(a):sys.exit(f"{family.display_name}: gateway adapter is not wired")engine=engine_for(a.model)need_model(a.model,engine)- base_env=env_for_engine(a,arch) if arch!="glm" else env_for(a)- # The replay prompt has to be the engine's own template: GLM's [gMASK]<sop>- # markers are ordinary text to every other tokenizer, so the sweep would have- # measured a different workload than the one users run.- prompt=tuning_replay_prompt(family,a.prompt)+ base_env=(env_for_engine(a,arch,plan=plan) if arch!="glm" else env_for(a))+ # Prompts have to use the engine's own template: GLM's [gMASK]<sop>+ # markers are ordinary text to every other tokenizer. Sibling engines keep+ # one serve process alive per candidate and alternate these prompts; this+ # preserves their real expert-cache state without training it on one exact+ # request. GLM retains its stronger fixed-token replay protocol.+ raw_prompts=[a.prompt,*(a.rotate_prompt or [_TUNE_ROTATION_PROMPT])]+ if len(set(raw_prompts)) < 2:+ sys.exit(f"{C.yel}tuning rotation needs at least two distinct prompts{C.r}")+ prompts=[tuning_replay_prompt(family,item) for item in raw_prompts]+ prompt=prompts[0]banner("tune · measured execution profile", model=a.model)- print(f" fixed replay: {a.tokens} tokens · {a.repeats} run(s) per candidate")+ if arch=="glm":+ print(f" fixed replay: {a.tokens} tokens · {a.repeats} run(s) per candidate")+ else:+ active=min(a.repeats,len(prompts))+ print(f" persistent rotation: {a.tokens} tokens · {a.repeats} request(s) "+ f"per candidate · {active} distinct prompt(s)")print(" only quality-preserving scheduling knobs are eligible\n")try:profile,path=run_tune(@@ -1029,6 +1080,7 @@ def cmd_tune(a):tokens=a.tokens,repeats=a.repeats,timeout=a.timeout,min_gain=a.min_gain,profile_dir=a.profile_dir,+ prompts=prompts,progress=lambda message: print(f" {C.dim}· {message}{C.r}",flush=True),)except (OSError,ValueError,RuntimeError,subprocess.SubprocessError) as error:@@ -1797,8 +1849,11 @@ def main():description="measure and save the fastest quality-preserving execution profile",help="measure and save the fastest quality-preserving execution profile")pt.add_argument("--prompt",default="Explain why fixed-token replay makes a benchmark reproducible.")+ pt.add_argument("--rotate-prompt",action="append",default=None,+ help="additional persistent-workload prompt; repeat for a custom rotation")pt.add_argument("--tokens",type=int,default=16,help="calibration continuation length")- pt.add_argument("--repeats",type=int,choices=range(1,6),default=2)+ pt.add_argument("--repeats",type=int,choices=range(1,6),default=2,+ help="requests per candidate (one persistent engine on non-GLM)")pt.add_argument("--timeout",type=int,default=900,help="seconds allowed for each engine run")pt.add_argument("--min-gain",type=float,default=0.03,help="minimum fractional throughput gain required to accept a candidate")diff --git a/c/tests/test_autotune.py b/c/tests/test_autotune.pyindex 3b4a1065..d5452447 100644--- a/c/tests/test_autotune.py+++ b/c/tests/test_autotune.py@@ -111,6 +111,23 @@ class AutotuneUnitTest(unittest.TestCase):# sweep has nothing left to measure on those engines.self.assertTrue(any(name.startswith("omp-") for name in steps), steps)+ def test_v4_loader_lanes_are_bounded_and_only_swept_when_disk_is_cold(self):+ cold_plan = plan(cores=16, gpu=False, cold=1 << 30)+ steps = dict(candidate_steps(cold_plan, {}, "deepseek_v4"))+ self.assertEqual(+ [name for name in steps if name.startswith("v4-loader-")],+ ["v4-loader-3", "v4-loader-6", "v4-loader-12"],+ )+ self.assertEqual(steps["v4-loader-3"], {"V4_LOADER_LANES": "3"})+ resident = dict(candidate_steps(plan(cores=16, gpu=False, cold=0), {},+ "deepseek_v4"))+ self.assertFalse(any(name.startswith("v4-loader-") for name in resident))++ def test_omp_kill_switch_removes_thread_candidates(self):+ steps = candidate_steps(plan(cores=16, gpu=False),+ {"COLI_NO_OMP_TUNE": "1"}, "kimi")+ self.assertFalse(any(name.startswith("omp-") for name, _ in steps))+def test_profile_round_trip_and_explicit_environment_wins(self):with tempfile.TemporaryDirectory() as directory:engine = Path(directory) / "engine"@@ -155,17 +172,21 @@ class FakeServeEngine:drift_on = None # knob name that changes the OUTPUT (illegal)drift_baseline = False # second run of ANY env differs (nondeterminism)calls = 0+ sessions = []def __init__(self, executable, model, cap, max_tokens, env, kv_slots,family):FakeServeEngine.launches.append({"cap": cap, "env": dict(env), "family": family})self.env = env+ self.prompts = []+ FakeServeEngine.sessions.append(self.prompts)def generate(self, prompt, max_tokens, temperature, top_p, on_text,cache_slot=0):FakeServeEngine.calls += 1- text = "hello world"+ self.prompts.append(prompt)+ text = f"answer:{prompt}"if FakeServeEngine.drift_on and \self.env.get(FakeServeEngine.drift_on) is not None:text = "HELLO DRIFT"@@ -174,6 +195,9 @@ class FakeServeEngine:on_text(text)threads = self.env.get("OMP_NUM_THREADS")speed = {None: 10.0, "8": 13.0, "4": 11.0}.get(threads, 10.0)+ speed *= {None: 1.0, "3": 1.30, "6": 1.10,+ "9": 1.0, "12": 0.90}.get(+ self.env.get("V4_LOADER_LANES"), 1.0)return {"completion_tokens": max_tokens, "tokens_per_second": speed,"cache_hit_percent": 95.0, "prompt_tokens": 3}@@ -182,7 +206,7 @@ class FakeServeEngine:@classmethoddef reset(cls, drift_on=None, drift_baseline=False):- cls.launches, cls.calls = [], 0+ cls.launches, cls.sessions, cls.calls = [], [], 0cls.drift_on, cls.drift_baseline = drift_on, drift_baseline@@ -205,7 +229,7 @@ class ServeTuneTest(unittest.TestCase):def test_sibling_arch_tunes_through_the_serve_protocol(self):FakeServeEngine.reset()- profile, _ = self.run_serve_tune()+ profile, _ = self.run_serve_tune(prompts=("prompt-a", "prompt-b"))self.assertTrue(profile["accepted"])# cores=8 with OMP unset means only omp-4 is offered (8 is already# the implicit default), so the sweep's one candidate must win@@ -216,6 +240,12 @@ class ServeTuneTest(unittest.TestCase):self.assertEqual(launch["cap"], 16)self.assertEqual(launch["family"], "fake-family")self.assertEqual(launch["env"].get("COLI_TEMP"), "0")+ # baseline, candidate, confirm-winner, confirm-baseline: one process+ # each, not one process per repeat. Every process sees the same+ # rotating prompt order and therefore retains its expert-cache state.+ self.assertEqual(len(FakeServeEngine.launches), 4)+ self.assertEqual(FakeServeEngine.sessions,+ [["prompt-a", "prompt-b"]] * 4)def test_candidate_that_changes_output_is_disqualified(self):FakeServeEngine.reset(drift_on="OMP_NUM_THREADS")@@ -226,6 +256,18 @@ class ServeTuneTest(unittest.TestCase):names = [candidate["name"] for candidate in profile["candidates"]]self.assertNotIn("omp-4", names)+ def test_v4_loader_winner_is_measured_and_persisted(self):+ FakeServeEngine.reset()+ p = plan(cores=8, gpu=False, cold=1 << 30)+ p["tiers"]["ram"] = {"cache_slots_per_layer": 16}+ profile, _ = self.run_serve_tune(+ plan=p, base_env={"OMP_NUM_THREADS": "8"},+ repeats=2, prompts=("prompt-a", "prompt-b"),+ )+ self.assertTrue(profile["accepted"])+ self.assertEqual(profile["winner"]["env"], {"V4_LOADER_LANES": "3"})+ self.assertAlmostEqual(profile["gain"], 0.30)+def test_nondeterministic_baseline_aborts_the_tune(self):FakeServeEngine.reset(drift_baseline=True)with self.assertRaises(RuntimeError):diff --git a/c/tests/test_v4_cli.py b/c/tests/test_v4_cli.pyindex ac0133fd..6fc67324 100644--- a/c/tests/test_v4_cli.py+++ b/c/tests/test_v4_cli.py@@ -133,6 +133,47 @@ class V4CliTest(unittest.TestCase):env = self.cli.env_for_engine(args, "kimi")self.assertNotIn("RAM_GB", env)+ def test_auto_tier_applies_saved_profile_to_sibling_engine(self):+ """#1196 saved sibling profiles, but only GLM's env_for() loaded them.+ A measured V4 lane winner must reach the real chat/serve child, while+ an explicit environment override remains authoritative."""+ directory, root = self.make_model()+ args = argparse.Namespace(+ model=str(root), ngen=8, temp=0.0, ram=0, ctx=0,+ gpu=None, vram=0, auto_tier=True, no_tune_profile=False,+ policy="quality", kv_slots=1,+ )+ measured = {+ "gain": 0.20,+ "winner": {"env": {"V4_LOADER_LANES": "3"}},+ }+ planned = {"sentinel": "same plan used by tune and launch"}+ def passthrough(_plan, env, cuda_enabled):+ result = dict(env)+ result.setdefault("OMP_NUM_THREADS", "16")+ return result+ try:+ with mock.patch.dict(os.environ, {}, clear=True), \+ mock.patch("resource_plan.environment_for_plan",+ side_effect=passthrough), \+ mock.patch("autotune.load_profile", return_value=measured), \+ mock.patch.object(self.cli, "engine_for",+ return_value="/engines/deepseek_v4"):+ env = self.cli.env_for_engine(args, "deepseek_v4", plan=planned)+ self.assertEqual(env["V4_LOADER_LANES"], "3")+ self.assertNotIn("OMP_NUM_THREADS", env)++ with mock.patch.dict(os.environ, {"V4_LOADER_LANES": "12"}, clear=True), \+ mock.patch("resource_plan.environment_for_plan",+ side_effect=passthrough), \+ mock.patch("autotune.load_profile", return_value=measured), \+ mock.patch.object(self.cli, "engine_for",+ return_value="/engines/deepseek_v4"):+ env = self.cli.env_for_engine(args, "deepseek_v4", plan=planned)+ self.assertEqual(env["V4_LOADER_LANES"], "12")+ finally:+ directory.cleanup()+def test_ngen_default_differs_for_interactive_commands(self):"""#889: `coli web` passed --max-tokens 1024, and openai_server clamps arequest to that ceiling (#260), so the browser's own "max output tokens"diff --git a/docs/tuning.md b/docs/tuning.mdindex 10312a0e..149bd931 100644--- a/docs/tuning.md+++ b/docs/tuning.md@@ -71,12 +71,19 @@ coli tune --model /models/glm52_i4coli run --model /models/glm52_i4 --auto-tier "Explain MoE offloading"```-The calibration prompt is generated once. Every candidate then teacher-forces-the same continuation, so answer length and sampling do not contaminate the-comparison. The bounded sweep only includes execution knobs such as OpenMP-thread count, NUMA placement, I/O overlap, direct I/O, and CUDA pipelining. It-never changes weights, quantization, router decisions, `TOPK`, `TOPP`, or-sampling.+GLM generates one calibration continuation and teacher-forces it for every+candidate. The sibling engines use their common serving protocol instead: one+engine stays alive for all requests of a candidate, and deterministic prompts+rotate in a fixed order. That preserves the expert cache and measures a real+mixed chat instead of repeatedly loading one process or teaching the cache one+exact prompt. Greedy output is compared per prompt across every candidate; any+byte drift disqualifies the scheduling change.++The bounded sweep only includes execution knobs such as OpenMP thread count,+NUMA placement, I/O overlap, direct I/O, CUDA pipelining, and DeepSeek V4 expert+loader lanes when cold experts remain on disk. It never changes weights,+quantization, router decisions, `TOPK`, `TOPP`, or sampling. Add repeatable+`--rotate-prompt` options to replace the secondary built-in workload prompt.A candidate is saved only when median throughput improves by at least 3% whileexpert hit rate remains within 0.5 percentage points and p99 latency stays