Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(783): doctor inspects the model's own engine; CUDA=1 stops decorating kimi_k3

JustVugga1f9e8ebf46f
diff --git a/c/Makefile b/c/Makefile
index 06f4e8df..9a807f3e 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -498,8 +498,24 @@ olmoe$(EXE): olmoe.c st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unico
inkling$(EXE): inkling.c st.h json.h compat.h $(INK_CUDA_OBJ) $(METAL_OBJ)
$(CC) $(CFLAGS) inkling.c $(INK_CUDA_OBJ) $(METAL_OBJ) -o inkling$(EXE) $(LDFLAGS)
+# ENGINES WITHOUT A CUDA BACKEND (#783).
+# kimi_k3.c contains no COLI_CUDA code at all β€” its GPU path is Vulkan. But
+# CUDA=1 appends -DCOLI_CUDA to the global CFLAGS and the cudart libs to
+# LDFLAGS, so `make kimi_k3 CUDA=1` used to compile a define that matches
+# nothing, link a runtime it never calls, and emit no warning: every
+# observable signal said "CUDA build" and the GPUs stayed idle. Strip the
+# flags for these targets and say so once, loudly.
+comma := ,
+NOCUDA_CFLAGS = $(filter-out -DCOLI_CUDA -DCOLI_ANS,$(CFLAGS))
+NOCUDA_LDFLAGS = $(filter-out -lcudart -lstdc++ -lcuda -L$(CUDA_HOME)/lib64 \
+ -Wl$(comma)-rpath$(comma)$(CUDA_HOME)/lib64,$(LDFLAGS))
+
kimi_k3$(EXE): kimi_k3.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h omp_tune.h route_trace.h $(VK_OBJ) $(VK_SPV)
- $(CC) $(CFLAGS) kimi_k3.c $(VK_OBJ) -o kimi_k3$(EXE) $(LDFLAGS)
+ifeq ($(CUDA),1)
+ @echo "*** kimi_k3 has no CUDA backend (#783): building without it."
+ @echo "*** On NVIDIA, Kimi K3 runs on the Vulkan path β€” build VULKAN=1 and set COLI_VULKAN=1."
+endif
+ $(CC) $(NOCUDA_CFLAGS) kimi_k3.c $(VK_OBJ) -o kimi_k3$(EXE) $(NOCUDA_LDFLAGS)
# Use a baseline that matches the compiler target. macOS already targets a
# portable baseline when ARCH is empty; forcing the x86 value there breaks
diff --git a/c/coli b/c/coli
index 44a09179..4f5953a8 100755
--- a/c/coli
+++ b/c/coli
@@ -614,7 +614,11 @@ def cmd_doctor(a):
print(json.dumps(report,indent=2) if a.json else format_doctor(report))
return 2
report=run_doctor(
- a.model, ram, ctx, devices, vram, engine_path=GLM, deep=a.deep,
+ # engine_path was hardcoded to GLM, so `coli doctor` inspected colibri
+ # no matter which model it was pointed at: a built kimi_k3/inkling was
+ # reported "engine is not built", and a stale colibri would have been
+ # reported ready for a model that never runs on it (#783).
+ a.model, ram, ctx, devices, vram, engine_path=engine_for(a.model), deep=a.deep,
mirror_dir=os.environ.get("COLI_MODEL_MIRROR"),
)
print(json.dumps(report,indent=2) if a.json else format_doctor(report))
diff --git a/c/tests/test_doctor_engine_arch.py b/c/tests/test_doctor_engine_arch.py
new file mode 100644
index 00000000..26644bfa
--- /dev/null
+++ b/c/tests/test_doctor_engine_arch.py
@@ -0,0 +1,78 @@
+"""`coli doctor` must inspect the engine the model would actually run on (#783).
+
+engine_path was hardcoded to the GLM binary, so doctor reported against colibri
+whatever the model was. That misled in both directions: a user who had built
+kimi_k3 was told "engine is not built", and anyone with a stale colibri lying
+around would have been told the engine was ready for a model that never runs on
+it. A diagnostic that reports on the wrong file is worse than no diagnostic.
+"""
+import json
+import struct
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent.parent
+CLI = HERE / "coli"
+
+ARCHES = [
+ ("glm-5.2", "colibri"),
+ ("inkling", "inkling"),
+ ("kimi_k3", "kimi_k3"),
+]
+
+
+def write_model(root, model_type):
+ root.mkdir(parents=True, exist_ok=True)
+ (root / "config.json").write_text(json.dumps({"model_type": model_type}))
+ (root / "tokenizer.json").write_text("{}")
+ header = json.dumps({"w": {"dtype": "U8", "shape": [4],
+ "data_offsets": [0, 4]}}).encode()
+ (root / "model-00001-of-00001.safetensors").write_bytes(
+ struct.pack("<Q", len(header)) + header + b"\0" * 4)
+
+
+class DoctorEngineArchTest(unittest.TestCase):
+ def doctor(self, model):
+ result = subprocess.run(
+ [sys.executable, str(CLI), "doctor", "--model", str(model), "--json"],
+ cwd=HERE, text=True, capture_output=True, check=False, timeout=60)
+ self.assertTrue(result.stdout.strip(),
+ f"no report on stdout; stderr={result.stderr}")
+ return json.loads(result.stdout)
+
+ def engine_check(self, report):
+ for check in report["checks"]:
+ if check["id"] == "engine.binary":
+ return check
+ self.fail("report has no engine.binary check")
+
+ def test_engine_check_names_the_engine_for_this_model_arch(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ for model_type, expected in ARCHES:
+ model = Path(tmp) / model_type
+ write_model(model, model_type)
+ check = self.engine_check(self.doctor(model))
+ path = check.get("details", {}).get("path", "")
+ self.assertTrue(
+ Path(path).name.startswith(expected),
+ f"{model_type}: doctor inspected {path!r}, expected the "
+ f"{expected} engine")
+
+ def test_two_arches_do_not_resolve_to_the_same_engine(self):
+ """The regression this guards: every arch mapping to the GLM binary."""
+ with tempfile.TemporaryDirectory() as tmp:
+ seen = []
+ for model_type, _ in ARCHES:
+ model = Path(tmp) / model_type
+ write_model(model, model_type)
+ check = self.engine_check(self.doctor(model))
+ seen.append(check.get("details", {}).get("path", ""))
+ self.assertEqual(len(set(seen)), len(ARCHES),
+ f"engines collapsed to the same path: {seen}")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tests/test_makefile_cuda_scope.py b/c/tests/test_makefile_cuda_scope.py
new file mode 100644
index 00000000..c8325f03
--- /dev/null
+++ b/c/tests/test_makefile_cuda_scope.py
@@ -0,0 +1,67 @@
+"""CUDA=1 must not decorate engines that have no CUDA backend (#783).
+
+kimi_k3.c contains no COLI_CUDA code β€” its GPU path is Vulkan. CUDA=1 appends
+-DCOLI_CUDA to the global CFLAGS and the cudart libraries to LDFLAGS, so the
+kimi_k3 target used to compile a define that matches nothing and link a runtime
+it never calls. The build printed no warning, so the compile line, the linked
+libraries and the exit status all said "CUDA build" while the GPUs sat idle.
+
+This asserts the shape of the recipe rather than running a compiler, so it
+works on hosts without CUDA installed.
+"""
+import shutil
+import subprocess
+import unittest
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent.parent
+CUDA_MARKERS = ("-DCOLI_CUDA", "-lcudart")
+
+
+def recipe(target, *variables):
+ """The commands make WOULD run, without running them.
+
+ -B (always-make) as well as -n: without it an already-built target prints
+ "is up to date" and no recipe at all, so the assertions below would pass
+ vacuously on any tree where someone had run make first.
+ """
+ result = subprocess.run(["make", "-Bn", target, *variables],
+ cwd=HERE, text=True, capture_output=True,
+ check=False, timeout=120)
+ return result.stdout + result.stderr
+
+
[email protected](shutil.which("make"), "make is not installed")
+class MakefileCudaScopeTest(unittest.TestCase):
+ def test_kimi_k3_is_not_built_with_cuda_flags(self):
+ out = recipe("kimi_k3", "CUDA=1")
+ compile_lines = [l for l in out.splitlines() if "kimi_k3.c" in l]
+ self.assertTrue(compile_lines, f"no kimi_k3 compile line in:\n{out}")
+ for line in compile_lines:
+ for marker in CUDA_MARKERS:
+ self.assertNotIn(marker, line,
+ f"kimi_k3 has no CUDA backend but the recipe "
+ f"carries {marker}:\n{line}")
+
+ def test_kimi_k3_says_why_cuda_was_dropped(self):
+ """Silently ignoring CUDA=1 would be its own trap: say it out loud."""
+ out = recipe("kimi_k3", "CUDA=1")
+ self.assertIn("no CUDA backend", out)
+ self.assertIn("Vulkan", out, "the message must name the path that does work")
+
+ def test_kimi_k3_without_cuda_is_unchanged(self):
+ out = recipe("kimi_k3")
+ for marker in CUDA_MARKERS:
+ self.assertNotIn(marker, out)
+ self.assertNotIn("no CUDA backend", out,
+ "the warning must only appear when CUDA=1 was asked for")
+
+ def test_colibri_still_gets_cuda(self):
+ """The guard must be scoped to the engines that lack the backend."""
+ out = recipe("colibri", "CUDA=1")
+ self.assertIn("-DCOLI_CUDA", out,
+ "colibri.c does have a CUDA backend and must keep it")
+
+
+if __name__ == "__main__":
+ unittest.main()