Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

perf(tune): measure persistent rotating workloads

JustVugg5e4b247d8af7
diff --git a/c/autotune.py b/c/autotune.py
index 3c5f267a..0fb2514e 100644
--- a/c/autotune.py
+++ b/c/autotune.py
@@ -1,9 +1,10 @@
"""Measured, quality-preserving execution tuning for colibri.
The tuner changes execution scheduling only. It never sweeps quantization,
-expert selection, sampling, or model weights. A calibration continuation is
-generated once and then teacher-forced for every candidate so every run sees
-the same token and routing inputs.
+expert selection, sampling, or model weights. GLM uses its fixed-token replay
+protocol. The sibling engines keep one serve process alive per candidate and
+rotate deterministic prompts, so their expert caches behave like a real chat
+instead of being reset between every sample.
"""
from __future__ import annotations
@@ -36,7 +37,7 @@ LATENCY_RE = re.compile(r"latency p50\s+([0-9.]+)\s*ms.*?p99\s+([0-9.]+)\s*ms")
# as TOPK, TOPP, DRAFT, quantization and CACHE_ROUTE are intentionally excluded.
TUNABLE_KEYS = frozenset({
"OMP_NUM_THREADS", "COLI_NUMA", "PIPE", "DIRECT",
- "COLI_CUDA_PIPE", "COLI_CUDA_ASYNC",
+ "COLI_CUDA_PIPE", "COLI_CUDA_ASYNC", "V4_LOADER_LANES",
})
@@ -153,9 +154,10 @@ def candidate_steps(plan: dict, base_env: dict, arch: str = "glm") -> list[tuple
steps = []
cores = max(1, int(plan.get("cpu", {}).get("physical_cores", os.cpu_count() or 1)))
current_threads = int(base_env.get("OMP_NUM_THREADS", cores))
- for threads in dict.fromkeys((cores, max(1, cores // 2))):
- if threads != current_threads:
- steps.append((f"omp-{threads}", {"OMP_NUM_THREADS": str(threads)}))
+ if not base_env.get("COLI_NO_OMP_TUNE"):
+ for threads in dict.fromkeys((cores, max(1, cores // 2))):
+ if threads != current_threads:
+ steps.append((f"omp-{threads}", {"OMP_NUM_THREADS": str(threads)}))
sockets = int(plan.get("cpu", {}).get("sockets", 1))
if sockets > 1 and base_env.get("COLI_NUMA") != "1":
steps.append(("numa-on", {"COLI_NUMA": "1"}))
@@ -173,6 +175,19 @@ def candidate_steps(plan: dict, base_env: dict, arch: str = "glm") -> list[tuple
pipe = int(base_env.get("PIPE", "0"))
if pipe != 1:
steps.append(("io-pipe-on", {"PIPE": "1"}))
+ if (arch == "deepseek_v4"
+ and plan.get("tiers", {}).get("disk", {}).get("cold_expert_bytes", 0) > 0):
+ # The pool default (9) was measured on one CPU/NVMe pair. GPU-tier
+ # systems often prefer 3 because a deeper reader pool contends with
+ # bank refill, while other SSDs keep scaling past 9. Sweep a bounded
+ # set around both measured optima; lanes block in pread and therefore
+ # do not need one CPU apiece.
+ current_lanes = int(base_env.get("V4_LOADER_LANES", "9"))
+ lane_values = (3, 6, 9, 12)
+ for lanes in lane_values:
+ if lanes != current_lanes:
+ steps.append((f"v4-loader-{lanes}",
+ {"V4_LOADER_LANES": str(lanes)}))
return steps
@@ -199,6 +214,24 @@ def parse_replay(output: str) -> dict:
}
+def _summarize_measurement(name: str, overlay: dict, samples: list[dict]) -> dict:
+ """Reduce one candidate's fixed request budget to comparable medians."""
+ return {
+ "name": name,
+ "env": dict(overlay),
+ "tok_s": statistics.median(sample["tok_s"] for sample in samples),
+ "hit_pct": statistics.median(
+ sample["hit_pct"] for sample in samples
+ if sample["hit_pct"] is not None
+ ) if any(sample["hit_pct"] is not None for sample in samples) else None,
+ "p99_ms": statistics.median(
+ sample["p99_ms"] for sample in samples
+ if sample["p99_ms"] is not None
+ ) if any(sample["p99_ms"] is not None for sample in samples) else None,
+ "samples": samples,
+ }
+
+
class OutputDrift(RuntimeError):
"""A run produced different tokens than the session's first run.
@@ -234,7 +267,8 @@ def run_tune(engine: str, cap: int, base_env: dict, plan: dict, model: str,
prompt: str, arch: str = "glm",
tokens: int = 16, repeats: int = 2, timeout: int = 900,
min_gain: float = 0.03, profile_dir: str | None = None,
- progress=None, family=None, engine_cls=None) -> tuple[dict, Path]:
+ progress=None, family=None, engine_cls=None,
+ prompts=None) -> tuple[dict, Path]:
"""Coordinate-descent tuning with a reverse-order confirmation gate."""
progress = progress or (lambda _message: None)
if cap is None:
@@ -266,6 +300,10 @@ def run_tune(engine: str, cap: int, base_env: dict, plan: dict, model: str,
else:
replay = None
context = contextlib.nullcontext()
+ serving_prompts = tuple(prompts or (prompt,))
+ if not serving_prompts or any(not isinstance(item, str) or not item
+ for item in serving_prompts):
+ raise ValueError("tuning prompts must be non-empty strings")
with context as directory:
if arch == "glm":
ref = Path(directory) / "replay.json"
@@ -284,51 +322,59 @@ def run_tune(engine: str, cap: int, base_env: dict, plan: dict, model: str,
else:
if engine_cls is None:
from openai_server import Engine as engine_cls
- expected = [] # bytes of the session's first run: the contract
+ # One expected byte stream per prompt. Different prompts naturally
+ # produce different text; the same prompt changing across
+ # candidates is the quality violation.
+ expected = {}
- def run_once(name, overlay):
+ def measure(name, overlay):
env = dict(base_env)
env.update(overlay)
env["COLI_TEMP"] = "0"
served = engine_cls(engine, model, cap, tokens, env, 1, family)
- pieces = []
+ samples = []
try:
- result = served.generate(prompt, tokens, 0.0, 1.0,
- pieces.append)
+ # Keep this engine alive for the whole candidate. `repeats`
+ # is still the request budget, but requests now rotate
+ # instead of repeatedly teaching the cache one prompt.
+ for repeat in range(repeats):
+ active_prompt = serving_prompts[repeat % len(serving_prompts)]
+ progress(f"{name} ({repeat + 1}/{repeats}, prompt "
+ f"{repeat % len(serving_prompts) + 1}/"
+ f"{len(serving_prompts)})")
+ pieces = []
+ result = served.generate(active_prompt, tokens, 0.0, 1.0,
+ pieces.append)
+ text = "".join(pieces)
+ contract = expected.get(active_prompt)
+ if contract is None:
+ expected[active_prompt] = text
+ elif text != contract:
+ raise OutputDrift(
+ f"{name} changed the generated tokens for prompt "
+ f"{repeat % len(serving_prompts) + 1}; a scheduling "
+ f"knob must never do that")
+ tok_s = float(result.get("tokens_per_second") or 0.0)
+ if tok_s <= 0:
+ raise RuntimeError(
+ f"{name}: engine reported no decode speed")
+ samples.append({
+ "tok_s": tok_s,
+ "hit_pct": result.get("cache_hit_percent"),
+ "p50_ms": None,
+ "p99_ms": None,
+ })
finally:
served.close()
- text = "".join(pieces)
- if not expected:
- expected.append(text)
- elif text != expected[0]:
- raise OutputDrift(
- f"{name} changed the generated tokens; a scheduling "
- f"knob must never do that")
- tok_s = float(result.get("tokens_per_second") or 0.0)
- if tok_s <= 0:
- raise RuntimeError(
- f"{name}: engine reported no decode speed")
- return {"tok_s": tok_s,
- "hit_pct": result.get("cache_hit_percent"),
- "p50_ms": None, "p99_ms": None}
-
- def measure(name, overlay):
- samples = []
- for repeat in range(repeats):
- progress(f"{name} ({repeat + 1}/{repeats})")
- samples.append(run_once(name, overlay))
- return {
- "name": name,
- "env": dict(overlay),
- "tok_s": statistics.median(sample["tok_s"] for sample in samples),
- "hit_pct": statistics.median(
- sample["hit_pct"] for sample in samples if sample["hit_pct"] is not None
- ) if any(sample["hit_pct"] is not None for sample in samples) else None,
- "p99_ms": statistics.median(
- sample["p99_ms"] for sample in samples if sample["p99_ms"] is not None
- ) if any(sample["p99_ms"] is not None for sample in samples) else None,
- "samples": samples,
- }
+ return _summarize_measurement(name, overlay, samples)
+
+ if arch == "glm":
+ def measure(name, overlay):
+ samples = []
+ for repeat in range(repeats):
+ progress(f"{name} ({repeat + 1}/{repeats})")
+ samples.append(run_once(name, overlay))
+ return _summarize_measurement(name, overlay, samples)
baseline = measure("baseline", {})
winner = baseline
diff --git a/c/coli b/c/coli
index 26d286e3..c87aca20 100755
--- a/c/coli
+++ b/c/coli
@@ -71,6 +71,11 @@ except ModuleNotFoundError:
from family_registry import (FamilyConfigError, UnknownFamilyError, all_families,
family_by_id, resolve_model, tuning_replay_prompt)
+_TUNE_ROTATION_PROMPT = (
+ "A service has high median throughput but poor tail latency. Explain how "
+ "you would distinguish a CPU, memory, and storage bottleneck."
+)
+
# Run-in-place (source checkout, "cd c && ./coli ..."): the engine, the
# support modules (resource_plan.py, doctor.py, autotune.py, openai_server.py) and
# tools/ all live next to this script — unchanged from before.
@@ -363,9 +368,10 @@ def ngen_for(a, interactive=False, family=None):
return limits.interactive_max_output if interactive else limits.default_max_output
return 16384 if interactive else 1024
-def env_for_engine(a, arch):
+def env_for_engine(a, arch, plan=None):
if arch == "glm":
return env_for(a)
+ explicit_env = set(os.environ)
env = os.environ.copy()
# OMP_NUM_THREADS for the sister engines that do not size their own team.
#
@@ -461,6 +467,40 @@ def env_for_engine(a, arch):
sys.exit(f"{C.yel}--vram needs the CUDA build:{C.r} the engine binary is CPU-only")
env["COLI_CUDA"] = "1"
env["CUDA_EXPERT_GB"] = str(vram)
+ if getattr(a, "auto_tier", False):
+ # GLM has historically applied both the resource plan and its measured
+ # profile inside env_for(). The sibling launcher returned above that
+ # code path, so #1196 could measure and save a perfectly valid Kimi,
+ # Inkling, OLMoE, Qwen or V4 profile that chat/serve never loaded.
+ from resource_plan import build_plan, environment_for_plan
+ if plan is None:
+ ram,ctx,devices,vram=resource_request(a,env)
+ plan=build_plan(a.model,ram,ctx,devices,vram,policy=a.policy,
+ kv_slots=requested_kv_slots(a,env))
+ # Preserve the existing non-GLM GPU contract: explicit --gpu/--vram
+ # selects a capable build above. Merely asking for auto-tier must not
+ # turn a CPU-only sibling binary into an attempted CUDA launch.
+ plan_must_not_choose_omp=("OMP_NUM_THREADS" not in env
+ and (arch == "deepseek_v4"
+ or env.get("COLI_NO_OMP_TUNE")))
+ env=environment_for_plan(plan,env,
+ cuda_enabled=env.get("COLI_CUDA") == "1")
+ if plan_must_not_choose_omp:
+ # environment_for_plan supplies the physical-core default used by
+ # the other engines. V4 deliberately computes logical CPUs minus
+ # its loader reservation at startup; COLI_NO_OMP_TUNE likewise
+ # promises no automatic team. Keep those baselines unless a
+ # measured profile (V4 only, below) or the user explicitly chooses
+ # a team.
+ env.pop("OMP_NUM_THREADS",None)
+ if not getattr(a, "no_tune_profile", False):
+ from autotune import apply_profile, load_profile
+ profile=load_profile(plan,a.model,engine_for(a.model))
+ if profile:
+ env=apply_profile(env,profile,explicit_env)
+ gain=100.0*profile["gain"]
+ print(f" {C.dim}[TUNE] applied measured profile · +{gain:.1f}% "
+ f"calibration throughput{C.r}",file=sys.stderr)
return env
def dsv4_cuda_available(model=None):
@@ -1015,13 +1055,24 @@ def cmd_tune(a):
sys.exit(f"{family.display_name}: gateway adapter is not wired")
engine=engine_for(a.model)
need_model(a.model,engine)
- base_env=env_for_engine(a,arch) if arch!="glm" else env_for(a)
- # The replay prompt has to be the engine's own template: GLM's [gMASK]<sop>
- # markers are ordinary text to every other tokenizer, so the sweep would have
- # measured a different workload than the one users run.
- prompt=tuning_replay_prompt(family,a.prompt)
+ base_env=(env_for_engine(a,arch,plan=plan) if arch!="glm" else env_for(a))
+ # Prompts have to use the engine's own template: GLM's [gMASK]<sop>
+ # markers are ordinary text to every other tokenizer. Sibling engines keep
+ # one serve process alive per candidate and alternate these prompts; this
+ # preserves their real expert-cache state without training it on one exact
+ # request. GLM retains its stronger fixed-token replay protocol.
+ raw_prompts=[a.prompt,*(a.rotate_prompt or [_TUNE_ROTATION_PROMPT])]
+ if len(set(raw_prompts)) < 2:
+ sys.exit(f"{C.yel}tuning rotation needs at least two distinct prompts{C.r}")
+ prompts=[tuning_replay_prompt(family,item) for item in raw_prompts]
+ prompt=prompts[0]
banner("tune · measured execution profile", model=a.model)
- print(f" fixed replay: {a.tokens} tokens · {a.repeats} run(s) per candidate")
+ if arch=="glm":
+ print(f" fixed replay: {a.tokens} tokens · {a.repeats} run(s) per candidate")
+ else:
+ active=min(a.repeats,len(prompts))
+ print(f" persistent rotation: {a.tokens} tokens · {a.repeats} request(s) "
+ f"per candidate · {active} distinct prompt(s)")
print(" only quality-preserving scheduling knobs are eligible\n")
try:
profile,path=run_tune(
@@ -1029,6 +1080,7 @@ def cmd_tune(a):
tokens=a.tokens,
repeats=a.repeats,timeout=a.timeout,min_gain=a.min_gain,
profile_dir=a.profile_dir,
+ prompts=prompts,
progress=lambda message: print(f" {C.dim}· {message}{C.r}",flush=True),
)
except (OSError,ValueError,RuntimeError,subprocess.SubprocessError) as error:
@@ -1797,8 +1849,11 @@ def main():
description="measure and save the fastest quality-preserving execution profile",
help="measure and save the fastest quality-preserving execution profile")
pt.add_argument("--prompt",default="Explain why fixed-token replay makes a benchmark reproducible.")
+ pt.add_argument("--rotate-prompt",action="append",default=None,
+ help="additional persistent-workload prompt; repeat for a custom rotation")
pt.add_argument("--tokens",type=int,default=16,help="calibration continuation length")
- pt.add_argument("--repeats",type=int,choices=range(1,6),default=2)
+ pt.add_argument("--repeats",type=int,choices=range(1,6),default=2,
+ help="requests per candidate (one persistent engine on non-GLM)")
pt.add_argument("--timeout",type=int,default=900,help="seconds allowed for each engine run")
pt.add_argument("--min-gain",type=float,default=0.03,
help="minimum fractional throughput gain required to accept a candidate")
diff --git a/c/tests/test_autotune.py b/c/tests/test_autotune.py
index 3b4a1065..d5452447 100644
--- a/c/tests/test_autotune.py
+++ b/c/tests/test_autotune.py
@@ -111,6 +111,23 @@ class AutotuneUnitTest(unittest.TestCase):
# sweep has nothing left to measure on those engines.
self.assertTrue(any(name.startswith("omp-") for name in steps), steps)
+ def test_v4_loader_lanes_are_bounded_and_only_swept_when_disk_is_cold(self):
+ cold_plan = plan(cores=16, gpu=False, cold=1 << 30)
+ steps = dict(candidate_steps(cold_plan, {}, "deepseek_v4"))
+ self.assertEqual(
+ [name for name in steps if name.startswith("v4-loader-")],
+ ["v4-loader-3", "v4-loader-6", "v4-loader-12"],
+ )
+ self.assertEqual(steps["v4-loader-3"], {"V4_LOADER_LANES": "3"})
+ resident = dict(candidate_steps(plan(cores=16, gpu=False, cold=0), {},
+ "deepseek_v4"))
+ self.assertFalse(any(name.startswith("v4-loader-") for name in resident))
+
+ def test_omp_kill_switch_removes_thread_candidates(self):
+ steps = candidate_steps(plan(cores=16, gpu=False),
+ {"COLI_NO_OMP_TUNE": "1"}, "kimi")
+ self.assertFalse(any(name.startswith("omp-") for name, _ in steps))
+
def test_profile_round_trip_and_explicit_environment_wins(self):
with tempfile.TemporaryDirectory() as directory:
engine = Path(directory) / "engine"
@@ -155,17 +172,21 @@ class FakeServeEngine:
drift_on = None # knob name that changes the OUTPUT (illegal)
drift_baseline = False # second run of ANY env differs (nondeterminism)
calls = 0
+ sessions = []
def __init__(self, executable, model, cap, max_tokens, env, kv_slots,
family):
FakeServeEngine.launches.append(
{"cap": cap, "env": dict(env), "family": family})
self.env = env
+ self.prompts = []
+ FakeServeEngine.sessions.append(self.prompts)
def generate(self, prompt, max_tokens, temperature, top_p, on_text,
cache_slot=0):
FakeServeEngine.calls += 1
- text = "hello world"
+ self.prompts.append(prompt)
+ text = f"answer:{prompt}"
if FakeServeEngine.drift_on and \
self.env.get(FakeServeEngine.drift_on) is not None:
text = "HELLO DRIFT"
@@ -174,6 +195,9 @@ class FakeServeEngine:
on_text(text)
threads = self.env.get("OMP_NUM_THREADS")
speed = {None: 10.0, "8": 13.0, "4": 11.0}.get(threads, 10.0)
+ speed *= {None: 1.0, "3": 1.30, "6": 1.10,
+ "9": 1.0, "12": 0.90}.get(
+ self.env.get("V4_LOADER_LANES"), 1.0)
return {"completion_tokens": max_tokens, "tokens_per_second": speed,
"cache_hit_percent": 95.0, "prompt_tokens": 3}
@@ -182,7 +206,7 @@ class FakeServeEngine:
@classmethod
def reset(cls, drift_on=None, drift_baseline=False):
- cls.launches, cls.calls = [], 0
+ cls.launches, cls.sessions, cls.calls = [], [], 0
cls.drift_on, cls.drift_baseline = drift_on, drift_baseline
@@ -205,7 +229,7 @@ class ServeTuneTest(unittest.TestCase):
def test_sibling_arch_tunes_through_the_serve_protocol(self):
FakeServeEngine.reset()
- profile, _ = self.run_serve_tune()
+ profile, _ = self.run_serve_tune(prompts=("prompt-a", "prompt-b"))
self.assertTrue(profile["accepted"])
# cores=8 with OMP unset means only omp-4 is offered (8 is already
# the implicit default), so the sweep's one candidate must win
@@ -216,6 +240,12 @@ class ServeTuneTest(unittest.TestCase):
self.assertEqual(launch["cap"], 16)
self.assertEqual(launch["family"], "fake-family")
self.assertEqual(launch["env"].get("COLI_TEMP"), "0")
+ # baseline, candidate, confirm-winner, confirm-baseline: one process
+ # each, not one process per repeat. Every process sees the same
+ # rotating prompt order and therefore retains its expert-cache state.
+ self.assertEqual(len(FakeServeEngine.launches), 4)
+ self.assertEqual(FakeServeEngine.sessions,
+ [["prompt-a", "prompt-b"]] * 4)
def test_candidate_that_changes_output_is_disqualified(self):
FakeServeEngine.reset(drift_on="OMP_NUM_THREADS")
@@ -226,6 +256,18 @@ class ServeTuneTest(unittest.TestCase):
names = [candidate["name"] for candidate in profile["candidates"]]
self.assertNotIn("omp-4", names)
+ def test_v4_loader_winner_is_measured_and_persisted(self):
+ FakeServeEngine.reset()
+ p = plan(cores=8, gpu=False, cold=1 << 30)
+ p["tiers"]["ram"] = {"cache_slots_per_layer": 16}
+ profile, _ = self.run_serve_tune(
+ plan=p, base_env={"OMP_NUM_THREADS": "8"},
+ repeats=2, prompts=("prompt-a", "prompt-b"),
+ )
+ self.assertTrue(profile["accepted"])
+ self.assertEqual(profile["winner"]["env"], {"V4_LOADER_LANES": "3"})
+ self.assertAlmostEqual(profile["gain"], 0.30)
+
def test_nondeterministic_baseline_aborts_the_tune(self):
FakeServeEngine.reset(drift_baseline=True)
with self.assertRaises(RuntimeError):
diff --git a/c/tests/test_v4_cli.py b/c/tests/test_v4_cli.py
index ac0133fd..6fc67324 100644
--- a/c/tests/test_v4_cli.py
+++ b/c/tests/test_v4_cli.py
@@ -133,6 +133,47 @@ class V4CliTest(unittest.TestCase):
env = self.cli.env_for_engine(args, "kimi")
self.assertNotIn("RAM_GB", env)
+ def test_auto_tier_applies_saved_profile_to_sibling_engine(self):
+ """#1196 saved sibling profiles, but only GLM's env_for() loaded them.
+ A measured V4 lane winner must reach the real chat/serve child, while
+ an explicit environment override remains authoritative."""
+ directory, root = self.make_model()
+ args = argparse.Namespace(
+ model=str(root), ngen=8, temp=0.0, ram=0, ctx=0,
+ gpu=None, vram=0, auto_tier=True, no_tune_profile=False,
+ policy="quality", kv_slots=1,
+ )
+ measured = {
+ "gain": 0.20,
+ "winner": {"env": {"V4_LOADER_LANES": "3"}},
+ }
+ planned = {"sentinel": "same plan used by tune and launch"}
+ def passthrough(_plan, env, cuda_enabled):
+ result = dict(env)
+ result.setdefault("OMP_NUM_THREADS", "16")
+ return result
+ try:
+ with mock.patch.dict(os.environ, {}, clear=True), \
+ mock.patch("resource_plan.environment_for_plan",
+ side_effect=passthrough), \
+ mock.patch("autotune.load_profile", return_value=measured), \
+ mock.patch.object(self.cli, "engine_for",
+ return_value="/engines/deepseek_v4"):
+ env = self.cli.env_for_engine(args, "deepseek_v4", plan=planned)
+ self.assertEqual(env["V4_LOADER_LANES"], "3")
+ self.assertNotIn("OMP_NUM_THREADS", env)
+
+ with mock.patch.dict(os.environ, {"V4_LOADER_LANES": "12"}, clear=True), \
+ mock.patch("resource_plan.environment_for_plan",
+ side_effect=passthrough), \
+ mock.patch("autotune.load_profile", return_value=measured), \
+ mock.patch.object(self.cli, "engine_for",
+ return_value="/engines/deepseek_v4"):
+ env = self.cli.env_for_engine(args, "deepseek_v4", plan=planned)
+ self.assertEqual(env["V4_LOADER_LANES"], "12")
+ finally:
+ directory.cleanup()
+
def test_ngen_default_differs_for_interactive_commands(self):
"""#889: `coli web` passed --max-tokens 1024, and openai_server clamps a
request to that ceiling (#260), so the browser's own "max output tokens"
diff --git a/docs/tuning.md b/docs/tuning.md
index 10312a0e..149bd931 100644
--- a/docs/tuning.md
+++ b/docs/tuning.md
@@ -71,12 +71,19 @@ coli tune --model /models/glm52_i4
coli run --model /models/glm52_i4 --auto-tier "Explain MoE offloading"
```
-The calibration prompt is generated once. Every candidate then teacher-forces
-the same continuation, so answer length and sampling do not contaminate the
-comparison. The bounded sweep only includes execution knobs such as OpenMP
-thread count, NUMA placement, I/O overlap, direct I/O, and CUDA pipelining. It
-never changes weights, quantization, router decisions, `TOPK`, `TOPP`, or
-sampling.
+GLM generates one calibration continuation and teacher-forces it for every
+candidate. The sibling engines use their common serving protocol instead: one
+engine stays alive for all requests of a candidate, and deterministic prompts
+rotate in a fixed order. That preserves the expert cache and measures a real
+mixed chat instead of repeatedly loading one process or teaching the cache one
+exact prompt. Greedy output is compared per prompt across every candidate; any
+byte drift disqualifies the scheduling change.
+
+The bounded sweep only includes execution knobs such as OpenMP thread count,
+NUMA placement, I/O overlap, direct I/O, CUDA pipelining, and DeepSeek V4 expert
+loader lanes when cold experts remain on disk. It never changes weights,
+quantization, router decisions, `TOPK`, `TOPP`, or sampling. Add repeatable
+`--rotate-prompt` options to replace the secondary built-in workload prompt.
A candidate is saved only when median throughput improves by at least 3% while
expert hit rate remains within 0.5 percentage points and p99 latency stays