Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(qwen36): the pre-init device-count probe must load the backend DLL
ZhiyangK249ddc61d19d
diff --git a/c/backend_loader.c b/c/backend_loader.cindex 91bc5cc3..d877553a 100644--- a/c/backend_loader.c+++ b/c/backend_loader.c@@ -1261,11 +1261,15 @@ static int coli_cuda_load(void){#ifdef COLI_HIP_DLL/* No synchronisation here, and that is a traced conclusion rather than an- * assumption: coli_cuda_load has exactly two callers, coli_cuda_init and- * coli_cuda_attention_project_ragged. colibri.c calls init on the main- * thread before model initialisation and therefore before any worker- * thread exists, and the ragged path is reachable only after that init- * succeeded. If a third caller ever appears, or the guard in colibri.c+ * assumption: coli_cuda_load has exactly three callers -- coli_cuda_init,+ * coli_cuda_attention_project_ragged, and (via the wrapper of #1577)+ * coli_cuda_available_device_count, which qwen36_tier.c calls while it is+ * still choosing the devices to hand to init. colibri.c calls init on the+ * main thread before model initialisation and therefore before any worker+ * thread exists; the ragged path is reachable only after that init+ * succeeded; and the tier's count is asked on that same main thread during+ * start-up, before the tier has created any thread of its own. If a caller+ * outside that start-up window ever appears, or the guard in colibri.c* moves, this needs an explicit lock instead. */if(!coli_hip_configure(&cfg)) return 0;runtime = coli_hip_acquire_runtime(&cfg);@@ -1518,9 +1522,15 @@ int coli_cuda_device_count(void){/* qwen36_tier.c's device selection asks for the usable count; the loader had* no wrapper for it, so the first CUDA_DLL build of qwen36 that compiled the- * tier in failed to link (#1533). */+ * tier in failed to link (#1533). It asks BEFORE coli_cuda_init -- the count+ * decides which devices init is given -- so this wrapper has to load the DLL+ * itself, exactly as coli_cuda_attention_project_ragged does: gating on+ * g_cuda.available alone answered 0 on every Windows host and the tier fell+ * back to the CPU path unless COLI_GPUS was set (#1577). Optional export:+ * a DLL predating it leaves the pointer NULL and the count falls back to+ * device_count(). */int coli_cuda_available_device_count(void){- if(!g_cuda.available) return 0;+ if(!coli_cuda_load()) return 0;if(!g_cuda.available_device_count) return g_cuda.device_count(); /* a DLL from before the export */return g_cuda.available_device_count();}diff --git a/c/tests/test_backend_loader.py b/c/tests/test_backend_loader.pyindex 40234247..af222d62 100644--- a/c/tests/test_backend_loader.py+++ b/c/tests/test_backend_loader.py@@ -40,6 +40,16 @@ from pathlib import PathHERE = Path(__file__).resolve().parent.parent+# The two device-count exports answer with DISTINCT non-zero values, so a+# pre-init caller can tell "the DLL answered" from "nothing answered": 0 is+# both a legitimate count and the value the g_cuda.available gate returned+# without ever loading anything, which is exactly what #1577 hid behind.+_AVAILABLE_DEVICE_SYMBOL = "coli_cuda_available_device_count"+_DEVICE_COUNT_SYMBOL = "coli_cuda_device_count"+_AVAILABLE_PROBE = 2+_DEVICE_COUNT_PROBE = 5++def _write_shard(path, tensors):"""Write a minimal safetensors file to *path*."""offset = 0@@ -199,6 +209,11 @@ class _StubFixture:self.runtime_a_src = self.src_dir / "runtime_a.c"self.runtime_b_src = self.src_dir / "runtime_b.c"self.backend_src = self.src_dir / "backend.c"+ # Same fake backend minus the optional available_device_count export:+ # the shape of a DLL built before #1533 added it, which the wrapper+ # must still answer from (#1577).+ self.cuda_backend_old = self.backend_dir / "coli_cuda_oldvariant.dll"+ self.backend_old_src = self.src_dir / "backend_old.c"# Diagnostic-only artifacts (W1-B2c2).self.dep_full = self.dep_full_dir / _DEP_BASENAMEself.dep_lean = self.dep_lean_dir / _DEP_BASENAME@@ -332,6 +347,19 @@ int main(int argc, char **argv)runtime_inventory("runtime_after_preload");devices[0] = 0;++ /* Pre-init availability probe (#1577). qwen36_tier.c asks HOW MANY devices+ * are usable to decide which indices it hands to coli_cuda_init, so the+ * answer cannot depend on that init having run. The two backend_loaded+ * readings bracket the call: 0 -> count -> 1 is the loader loading the DLL+ * on demand, while 0 -> 0 -> 0 is the miss path, and a 0 count with a+ * loaded backend is the bug this probe exists for. */+ printf("backend_loaded_before_probe=%d\n",+ GetModuleHandleW(HARNESS_BACKEND_DLL) != NULL);+ printf("available_before_init=%d\n", coli_cuda_available_device_count());+ printf("backend_loaded_after_probe=%d\n",+ GetModuleHandleW(HARNESS_BACKEND_DLL) != NULL);+rc = coli_cuda_init(devices, 1); /* the real loader runs here */printf("loader_init=%d\n", rc);@@ -585,8 +613,10 @@ int main(int argc, char **argv)"building test dependency " + name)return implib- def _backend_source(self, with_dep=False):+ def _backend_source(self, with_dep=False, omit=()):real = {"coli_cuda_init", "coli_cuda_e8_set_grid"}+ probed = {_AVAILABLE_DEVICE_SYMBOL: _AVAILABLE_PROBE,+ _DEVICE_COUNT_SYMBOL: _DEVICE_COUNT_PROBE}lines = ["/* generated fake backend: no HIP/ROCm/CUDA header, no GPU work,"," * no DllMain. Imports the marker from amdhip64_7.dll by basename",@@ -621,8 +651,14 @@ int main(int argc, char **argv)"{ return %s(); }" % _DEP_EXTRA,]for name in self.exports:- if name not in real:+ if name in real or name in omit:+ continue+ value = probed.get(name)+ if value is None:lines.append("__declspec(dllexport) int %s(void) { return 0; }" % name)+ else:+ lines.append("__declspec(dllexport) int %s(void) { return %d; }"+ % (name, value))return "\n".join(lines) + "\n"def _build(self):@@ -665,6 +701,13 @@ int main(int argc, char **argv)"-L" + str(implib_a.parent), "-lamdhip64_7","-L" + str(dep_implib.parent), "-lcoli_test_dep"],"building the dependency-carrying backend variant")+ self.backend_old_src.write_text(+ self._backend_source(omit=(_AVAILABLE_DEVICE_SYMBOL,)),+ encoding="ascii")+ self._gcc(["-O0", "-shared", str(self.backend_old_src),+ "-o", str(self.cuda_backend_old),+ "-L" + str(implib_a.parent), "-lamdhip64_7"],+ "building the pre-export backend variant")def _build_harness(self):"""Compile the harness together with the repository's backend_loader.c.@@ -737,6 +780,7 @@ int main(int argc, char **argv)def generated_paths(self):return [self.runtime_a, self.runtime_b, self.backend, self.cuda_backend,+ self.cuda_backend_old, self.backend_old_src,self.runtime_a_src, self.runtime_b_src, self.backend_src,self.harness_src, self.harness_exe, self.cuda_harness_exe,self.runtime_a_dir / "libamdhip64_7.a",@@ -2424,5 +2468,107 @@ class LoaderBackendSelectionTest(unittest.TestCase):self.assertEqual(result.returncode, 2)+class LoaderPreInitAvailabilityTest(unittest.TestCase):+ """The device count qwen36's tier reads BEFORE it calls coli_cuda_init.++ qwen36_tier.c asks how many devices are usable so it knows which indices to+ hand to coli_cuda_init, so the answer cannot depend on that init having+ happened. Gating the wrapper on g_cuda.available answered 0 on every+ Windows host and the tier silently took the CPU path unless COLI_GPUS was+ set (#1577). Nothing here needs a GPU: the backend is the generated stub.+ """++ fixture = None++ @classmethod+ def setUpClass(cls):+ reason = _fixture_toolchain_skip()+ if reason:+ raise unittest.SkipTest(reason)+ cls.fixture = _StubFixture()++ @classmethod+ def tearDownClass(cls):+ if cls.fixture is not None:+ cls.fixture.cleanup()+ cls.fixture = None++ def setUp(self):+ self.cases = tempfile.TemporaryDirectory(prefix="coli loader probe ")+ self.addCleanup(self.cases.cleanup)++ def _probe(self, name, **kwargs):+ """Run the harness with the stub backend's own dependency present.++ The generated backend imports its marker from amdhip64_7.dll by+ basename, and a CUDA host loads the DLL from the engine's own directory,+ so the dependency has to sit beside it for the load to succeed at all+ (otherwise Windows error 126, which is a fixture setup mistake rather+ than a loader behaviour). The HIP control keeps its sandbox untouched:+ there the loader refuses on COLI_HIP_RUNTIME_DIR before any load.+ """+ if kwargs.get("vendor") == "cuda" and kwargs.get("with_backend", True):+ kwargs.setdefault("extra_files", (self.fixture.runtime_a,))+ return self.fixture.run_harness(Path(self.cases.name) / name, "NONE",+ **kwargs)++ @staticmethod+ def _detail(proc):+ return "\nstdout:\n%s\nstderr:\n%s" % (proc.stdout, proc.stderr)++ def test_the_probe_loads_the_dll_it_reports_on(self):+ """No init yet, backend present: the count comes from the DLL itself."""+ proc, out = self._probe("probe_present", vendor="cuda")+ detail = self._detail(proc)+ self.assertEqual(proc.returncode, 0, "harness failed" + detail)+ self.assertEqual(out.get("backend_loaded_before_probe"), "0",+ "the probe was measured with the DLL already loaded"+ + detail)+ self.assertEqual(out.get("available_before_init"),+ str(_AVAILABLE_PROBE),+ "the pre-init count did not come from the DLL" + detail)+ self.assertEqual(out.get("backend_loaded_after_probe"), "1",+ "the probe never loaded the backend DLL" + detail)+ # The harness still reaches the normal init afterwards: loading on+ # demand must not consume the one load the process is allowed.+ self.assertEqual(out.get("loader_init"), "1", detail)++ def test_the_probe_is_zero_when_no_backend_is_present(self):+ """The miss path stays a miss: 0 devices, nothing loaded, no crash."""+ proc, out = self._probe("probe_absent", vendor="cuda",+ with_backend=False)+ detail = self._detail(proc)+ self.assertEqual(proc.returncode, 0, "harness failed" + detail)+ self.assertEqual(out.get("available_before_init"), "0", detail)+ self.assertEqual(out.get("backend_loaded_after_probe"), "0", detail)+ self.assertEqual(out.get("backend_loaded"), "0", detail)+ self.assertEqual(out.get("loader_init"), "0", detail)++ def test_the_probe_falls_back_for_a_dll_without_the_export(self):+ """An older DLL answers through device_count(), not through a crash."""+ f = self.fixture+ proc, out = self._probe("probe_older_dll", vendor="cuda",+ backend_override=f.cuda_backend_old)+ detail = self._detail(proc)+ self.assertEqual(proc.returncode, 0, "harness failed" + detail)+ self.assertEqual(out.get("backend_loaded_after_probe"), "1", detail)+ self.assertEqual(out.get("available_before_init"),+ str(_DEVICE_COUNT_PROBE),+ "the fallback did not reach device_count()" + detail)++ def test_the_probe_keeps_the_hip_refusal_before_init(self):+ """A HIP host still refuses without a runtime directory, before init.++ The count is asked before init on every host, so the HIP control proves+ the earlier call neither bypasses COLI_HIP_RUNTIME_DIR validation nor+ leaves the backend mapped when that validation fails.+ """+ proc, out = self._probe("probe_hip_unconfigured", vendor="hip")+ detail = self._detail(proc)+ self.assertEqual(proc.returncode, 0, "harness failed" + detail)+ self.assertEqual(out.get("available_before_init"), "0", detail)+ self.assertEqual(out.get("backend_loaded_after_probe"), "0", detail)++if __name__ == "__main__":unittest.main()