Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(autotune): coli tune swept GLM-only CUDA knobs on every engine

kevin9327d361ab39d2bc
diff --git a/c/autotune.py b/c/autotune.py
index c61ccc26..6a22e0c3 100644
--- a/c/autotune.py
+++ b/c/autotune.py
@@ -195,7 +195,9 @@ def candidate_steps(plan: dict, base_env: dict, arch: str = "glm") -> list[tuple
if sockets > 1 and base_env.get("COLI_NUMA") != "1":
steps.append(("numa-on", {"COLI_NUMA": "1"}))
has_gpu = bool(plan.get("tiers", {}).get("vram", {}).get("devices"))
- if has_gpu:
+ # Same rule for the CUDA knobs: COLI_CUDA_PIPE is read in colibri.c only,
+ # and COLI_CUDA_ASYNC only on the grouped-expert call colibri.c makes.
+ if has_gpu and arch == "glm":
pipe = int(base_env.get("COLI_CUDA_PIPE", "0"))
for value in (1, 2):
if value != pipe:
diff --git a/c/tests/test_autotune.py b/c/tests/test_autotune.py
index a7fdc420..53e36f84 100644
--- a/c/tests/test_autotune.py
+++ b/c/tests/test_autotune.py
@@ -112,6 +112,24 @@ class AutotuneUnitTest(unittest.TestCase):
# sweep has nothing left to measure on those engines.
self.assertTrue(any(name.startswith("omp-") for name in steps), steps)
+ def test_cuda_knobs_are_not_swept_on_other_engines(self):
+ """COLI_CUDA_PIPE is read by colibri.c alone, and COLI_CUDA_ASYNC only
+ on the grouped-expert CUDA call colibri.c makes. A GPU in the plan
+ offered both to every engine, so a sibling tune measured identical
+ runs and could accept timing noise as a gain."""
+ gpu_plan = plan(gpu=True)
+ glm = dict(candidate_steps(gpu_plan, {}, "glm"))
+ for name in ("cuda-pipe-1", "cuda-pipe-2", "cuda-sync"):
+ self.assertIn(name, glm)
+ for arch in ("glm53", "deepseek_v4", "deepseek_v41", "kimi", "inkling",
+ "olmoe", "qwen36", "qwen38"):
+ with self.subTest(arch=arch):
+ steps = dict(candidate_steps(gpu_plan, {}, arch))
+ self.assertFalse(any(name.startswith("cuda-") for name in steps), steps)
+ self.assertFalse({"COLI_CUDA_PIPE", "COLI_CUDA_ASYNC"}
+ & {key for change in steps.values() for key in change})
+ self.assertTrue(any(name.startswith("omp-") for name in steps), steps)
+
def test_v4_loader_lanes_are_bounded_and_only_swept_when_disk_is_cold(self):
cold_plan = plan(cores=16, gpu=False, cold=1 << 30)
steps = dict(candidate_steps(cold_plan, {}, "deepseek_v4"))
@@ -332,6 +350,19 @@ class ServeTuneTest(unittest.TestCase):
self.assertEqual(FakeServeEngine.sessions,
[["prompt-a", "prompt-b"]] * 4)
+ def test_sibling_tune_with_a_gpu_launches_no_cuda_candidates(self):
+ FakeServeEngine.reset()
+ p = plan(cores=8, gpu=True)
+ p["tiers"]["ram"] = {"cache_slots_per_layer": 16}
+ profile, _ = self.run_serve_tune(arch="qwen36", plan=p,
+ prompts=("prompt-a", "prompt-b"))
+ names = [candidate["name"] for candidate in profile["candidates"]]
+ self.assertEqual(names, ["baseline", "omp-4"])
+ for launch in FakeServeEngine.launches:
+ self.assertNotIn("COLI_CUDA_PIPE", launch["env"])
+ self.assertNotIn("COLI_CUDA_ASYNC", launch["env"])
+ self.assertEqual(len(FakeServeEngine.launches), 4)
+
def test_candidate_that_changes_output_is_disqualified(self):
FakeServeEngine.reset(drift_on="OMP_NUM_THREADS")
profile, _ = self.run_serve_tune()