Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
fix(783): doctor inspects the model's own engine; CUDA=1 stops decorating kimi_k3
JustVugga1f9e8ebf46f
diff --git a/c/Makefile b/c/Makefileindex 06f4e8df..9a807f3e 100644--- a/c/Makefile+++ b/c/Makefile@@ -498,8 +498,24 @@ olmoe$(EXE): olmoe.c st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicoinkling$(EXE): inkling.c st.h json.h compat.h $(INK_CUDA_OBJ) $(METAL_OBJ)$(CC) $(CFLAGS) inkling.c $(INK_CUDA_OBJ) $(METAL_OBJ) -o inkling$(EXE) $(LDFLAGS)+# ENGINES WITHOUT A CUDA BACKEND (#783).+# kimi_k3.c contains no COLI_CUDA code at all β its GPU path is Vulkan. But+# CUDA=1 appends -DCOLI_CUDA to the global CFLAGS and the cudart libs to+# LDFLAGS, so `make kimi_k3 CUDA=1` used to compile a define that matches+# nothing, link a runtime it never calls, and emit no warning: every+# observable signal said "CUDA build" and the GPUs stayed idle. Strip the+# flags for these targets and say so once, loudly.+comma := ,+NOCUDA_CFLAGS = $(filter-out -DCOLI_CUDA -DCOLI_ANS,$(CFLAGS))+NOCUDA_LDFLAGS = $(filter-out -lcudart -lstdc++ -lcuda -L$(CUDA_HOME)/lib64 \+ -Wl$(comma)-rpath$(comma)$(CUDA_HOME)/lib64,$(LDFLAGS))+kimi_k3$(EXE): kimi_k3.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h omp_tune.h route_trace.h $(VK_OBJ) $(VK_SPV)- $(CC) $(CFLAGS) kimi_k3.c $(VK_OBJ) -o kimi_k3$(EXE) $(LDFLAGS)+ifeq ($(CUDA),1)+ @echo "*** kimi_k3 has no CUDA backend (#783): building without it."+ @echo "*** On NVIDIA, Kimi K3 runs on the Vulkan path β build VULKAN=1 and set COLI_VULKAN=1."+endif+ $(CC) $(NOCUDA_CFLAGS) kimi_k3.c $(VK_OBJ) -o kimi_k3$(EXE) $(NOCUDA_LDFLAGS)# Use a baseline that matches the compiler target. macOS already targets a# portable baseline when ARCH is empty; forcing the x86 value there breaksdiff --git a/c/coli b/c/coliindex 44a09179..4f5953a8 100755--- a/c/coli+++ b/c/coli@@ -614,7 +614,11 @@ def cmd_doctor(a):print(json.dumps(report,indent=2) if a.json else format_doctor(report))return 2report=run_doctor(- a.model, ram, ctx, devices, vram, engine_path=GLM, deep=a.deep,+ # engine_path was hardcoded to GLM, so `coli doctor` inspected colibri+ # no matter which model it was pointed at: a built kimi_k3/inkling was+ # reported "engine is not built", and a stale colibri would have been+ # reported ready for a model that never runs on it (#783).+ a.model, ram, ctx, devices, vram, engine_path=engine_for(a.model), deep=a.deep,mirror_dir=os.environ.get("COLI_MODEL_MIRROR"),)print(json.dumps(report,indent=2) if a.json else format_doctor(report))diff --git a/c/tests/test_doctor_engine_arch.py b/c/tests/test_doctor_engine_arch.pynew file mode 100644index 00000000..26644bfa--- /dev/null+++ b/c/tests/test_doctor_engine_arch.py@@ -0,0 +1,78 @@+"""`coli doctor` must inspect the engine the model would actually run on (#783).++engine_path was hardcoded to the GLM binary, so doctor reported against colibri+whatever the model was. That misled in both directions: a user who had built+kimi_k3 was told "engine is not built", and anyone with a stale colibri lying+around would have been told the engine was ready for a model that never runs on+it. A diagnostic that reports on the wrong file is worse than no diagnostic.+"""+import json+import struct+import subprocess+import sys+import tempfile+import unittest+from pathlib import Path++HERE = Path(__file__).resolve().parent.parent+CLI = HERE / "coli"++ARCHES = [+ ("glm-5.2", "colibri"),+ ("inkling", "inkling"),+ ("kimi_k3", "kimi_k3"),+]+++def write_model(root, model_type):+ root.mkdir(parents=True, exist_ok=True)+ (root / "config.json").write_text(json.dumps({"model_type": model_type}))+ (root / "tokenizer.json").write_text("{}")+ header = json.dumps({"w": {"dtype": "U8", "shape": [4],+ "data_offsets": [0, 4]}}).encode()+ (root / "model-00001-of-00001.safetensors").write_bytes(+ struct.pack("<Q", len(header)) + header + b"\0" * 4)+++class DoctorEngineArchTest(unittest.TestCase):+ def doctor(self, model):+ result = subprocess.run(+ [sys.executable, str(CLI), "doctor", "--model", str(model), "--json"],+ cwd=HERE, text=True, capture_output=True, check=False, timeout=60)+ self.assertTrue(result.stdout.strip(),+ f"no report on stdout; stderr={result.stderr}")+ return json.loads(result.stdout)++ def engine_check(self, report):+ for check in report["checks"]:+ if check["id"] == "engine.binary":+ return check+ self.fail("report has no engine.binary check")++ def test_engine_check_names_the_engine_for_this_model_arch(self):+ with tempfile.TemporaryDirectory() as tmp:+ for model_type, expected in ARCHES:+ model = Path(tmp) / model_type+ write_model(model, model_type)+ check = self.engine_check(self.doctor(model))+ path = check.get("details", {}).get("path", "")+ self.assertTrue(+ Path(path).name.startswith(expected),+ f"{model_type}: doctor inspected {path!r}, expected the "+ f"{expected} engine")++ def test_two_arches_do_not_resolve_to_the_same_engine(self):+ """The regression this guards: every arch mapping to the GLM binary."""+ with tempfile.TemporaryDirectory() as tmp:+ seen = []+ for model_type, _ in ARCHES:+ model = Path(tmp) / model_type+ write_model(model, model_type)+ check = self.engine_check(self.doctor(model))+ seen.append(check.get("details", {}).get("path", ""))+ self.assertEqual(len(set(seen)), len(ARCHES),+ f"engines collapsed to the same path: {seen}")+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tests/test_makefile_cuda_scope.py b/c/tests/test_makefile_cuda_scope.pynew file mode 100644index 00000000..c8325f03--- /dev/null+++ b/c/tests/test_makefile_cuda_scope.py@@ -0,0 +1,67 @@+"""CUDA=1 must not decorate engines that have no CUDA backend (#783).++kimi_k3.c contains no COLI_CUDA code β its GPU path is Vulkan. CUDA=1 appends+-DCOLI_CUDA to the global CFLAGS and the cudart libraries to LDFLAGS, so the+kimi_k3 target used to compile a define that matches nothing and link a runtime+it never calls. The build printed no warning, so the compile line, the linked+libraries and the exit status all said "CUDA build" while the GPUs sat idle.++This asserts the shape of the recipe rather than running a compiler, so it+works on hosts without CUDA installed.+"""+import shutil+import subprocess+import unittest+from pathlib import Path++HERE = Path(__file__).resolve().parent.parent+CUDA_MARKERS = ("-DCOLI_CUDA", "-lcudart")+++def recipe(target, *variables):+ """The commands make WOULD run, without running them.++ -B (always-make) as well as -n: without it an already-built target prints+ "is up to date" and no recipe at all, so the assertions below would pass+ vacuously on any tree where someone had run make first.+ """+ result = subprocess.run(["make", "-Bn", target, *variables],+ cwd=HERE, text=True, capture_output=True,+ check=False, timeout=120)+ return result.stdout + result.stderr++[email protected](shutil.which("make"), "make is not installed")+class MakefileCudaScopeTest(unittest.TestCase):+ def test_kimi_k3_is_not_built_with_cuda_flags(self):+ out = recipe("kimi_k3", "CUDA=1")+ compile_lines = [l for l in out.splitlines() if "kimi_k3.c" in l]+ self.assertTrue(compile_lines, f"no kimi_k3 compile line in:\n{out}")+ for line in compile_lines:+ for marker in CUDA_MARKERS:+ self.assertNotIn(marker, line,+ f"kimi_k3 has no CUDA backend but the recipe "+ f"carries {marker}:\n{line}")++ def test_kimi_k3_says_why_cuda_was_dropped(self):+ """Silently ignoring CUDA=1 would be its own trap: say it out loud."""+ out = recipe("kimi_k3", "CUDA=1")+ self.assertIn("no CUDA backend", out)+ self.assertIn("Vulkan", out, "the message must name the path that does work")++ def test_kimi_k3_without_cuda_is_unchanged(self):+ out = recipe("kimi_k3")+ for marker in CUDA_MARKERS:+ self.assertNotIn(marker, out)+ self.assertNotIn("no CUDA backend", out,+ "the warning must only appear when CUDA=1 was asked for")++ def test_colibri_still_gets_cuda(self):+ """The guard must be scoped to the engines that lack the backend."""+ out = recipe("colibri", "CUDA=1")+ self.assertIn("-DCOLI_CUDA", out,+ "colibri.c does have a CUDA backend and must keep it")+++if __name__ == "__main__":+ unittest.main()