Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
win32: auto-enable the GPU in bare coli chat
woolcoxm0b05f6a76bc8
diff --git a/c/coli b/c/coliindex d6728fe2..b00a1cd9 100755--- a/c/coli+++ b/c/coli@@ -237,6 +237,43 @@ def env_for(a):gpu=f" Β· VRAM {format_bytes(vt['budget_bytes'])}" if has_cuda and vt["devices"] else " Β· CPU"print(f" {C.dim}[PLAN] RAM {format_bytes(rt['budget_bytes'])} Β· cap {rt['cache_slots_per_layer']}/layer{gpu}{C.r}",file=sys.stderr)else:+ # Windows: a bare `coli chat` (no --gpu/--vram/--auto-tier) used to ALWAYS+ # run CPU-only, even on a CUDA build with a GPU present β cuda_binary()+ # returned False on Windows (see above), and nothing set COLI_CUDA without+ # an explicit flag. Now that detection works, auto-enable the GPU when one+ # is detected so `coli chat` Just Works. Scoped to Windows: Linux already+ # has working detection + the explicit-flag UX, and changing bare-chat+ # semantics there is out of scope. Falls back to CPU with a warning if+ # nvidia-smi is missing (discover_gpus can't size VRAM without it).+ if (sys.platform == "win32" and a.gpu is None and not a.vram):+ if cuda_binary():+ from resource_plan import discover_gpus, build_plan, environment_for_plan, format_bytes+ gpus = discover_gpus()+ if gpus:+ e["COLI_CUDA"]="1"+ e.setdefault("COLI_GPUS", ",".join(str(g["index"]) for g in gpus))+ # Reuse the planner so the expert-tier VRAM budget is the real+ # free VRAM minus the 2 GB reserve β not a guess. Same machinery+ # as --auto-tier, just without requiring the user to pass it.+ ram,ctx,devices,vram_req = resource_request(a, e)+ try:+ plan=build_plan(a.model,ram,ctx,devices,vram_req,policy=a.policy)+ e.update(environment_for_plan(plan,e,cuda_enabled=True))+ vt=plan["tiers"]["vram"]+ names=",".join(g["name"].strip() for g in gpus)+ print(f" {C.dim}[GPU] auto-enabled CUDA Β· {names} Β· "+ f"{format_bytes(vt['budget_bytes'])} expert tier{C.r}", file=sys.stderr)+ except (OSError,ValueError,json.JSONDecodeError) as error:+ # Plan failed (e.g. model dir unreadable): don't block the+ # run, just leave the unsized COLI_CUDA=1 and let the engine+ # pick its own budget. Engine handles a missing budget.+ print(f" {C.yel}[GPU] auto-enable: could not size VRAM ({error}); "+ f"using engine default{C.r}", file=sys.stderr)+ else:+ print(f" {C.yel}[GPU] coli_cuda.dll present but nvidia-smi not found on PATH "+ f"(cannot size VRAM); running CPU-only. Add nvidia-smi to PATH or pass "+ f"--vram N to enable CUDA.{C.r}", file=sys.stderr)+ # else: CPU build (no coli_cuda.dll) β stay silent, CPU is correct.# --gpu/--vram SENZA --auto-tier: prima venivano ignorati in silenzio e il run# partiva CPU-only senza alcun avviso β benchmark "GPU" pubblicati per errore (#121).if a.gpu is not None:diff --git a/c/tests/test_env_defaults.py b/c/tests/test_env_defaults.pyindex 12a3c999..3bb08fc8 100644--- a/c/tests/test_env_defaults.py+++ b/c/tests/test_env_defaults.py@@ -32,9 +32,16 @@ def args(**over):class EnvDefaultsTest(unittest.TestCase):- def env_for_with(self, environ, platform):+ def env_for_with(self, environ, platform, cuda=False):+ """Run env_for on a bare-chat args() under a faked env + platform.++ cuda=False by default so the existing default-I/O tests stay+ deterministic: the Windows auto-enable branch calls cuda_binary() and+ (if True) discover_gpus(), both of which reach the real machine β faking+ False keeps these tests independent of the host's GPU."""with mock.patch.dict(os.environ, environ, clear=True), \- mock.patch.object(sys, "platform", platform):+ mock.patch.object(sys, "platform", platform), \+ mock.patch.object(coli, "cuda_binary", return_value=cuda):return coli.env_for(args())def test_win32_sets_measured_defaults(self):@@ -64,5 +71,80 @@ class EnvDefaultsTest(unittest.TestCase):self.assertNotIn(k, e)+class CudaAutoEnableTest(unittest.TestCase):+ """Windows bare `coli chat` (no --gpu/--vram/--auto-tier) used to ALWAYS run+ CPU-only even on a CUDA build with a GPU present. env_for now auto-enables+ CUDA on win32 when cuda_binary() is True and a GPU is discoverable; falls+ back to CPU with a warning if nvidia-smi (discover_gpus) is missing; stays+ silent on a CPU build; and never touches the Linux path."""++ def _env_for(self, platform, cuda, gpus, plan=None):+ # Patch discover_gpus / build_plan / environment_for_plan at the+ # resource_plan module (env_for imports them lazily on each call, so the+ # patches are live when those imports run). Stubbing the planner keeps+ # the test independent of a real model dir (args().model == "X").+ import resource_plan+ a = args()+ GPB = 1024 ** 3+ if plan is None:+ plan = {"tiers": {"ram": {"budget_bytes": 16 * GPB, "cache_slots_per_layer": 4},+ "vram": {"budget_bytes": int(8.0 * GPB), "devices": gpus}}}++ def fake_environment_for_plan(p, env, cuda_enabled=True):+ # Mirror the real contract: size CUDA_EXPERT_GB from the plan's VRAM+ # budget (this is the value env_for propagates into the engine env).+ r = dict(env)+ if cuda_enabled and p["tiers"]["vram"]["devices"] and p["tiers"]["vram"]["budget_bytes"] > 0:+ r["CUDA_EXPERT_GB"] = f"{p['tiers']['vram']['budget_bytes'] / GPB:.3f}"+ return r++ with mock.patch.dict(os.environ, {}, clear=True), \+ mock.patch.object(sys, "platform", platform), \+ mock.patch.object(coli, "cuda_binary", return_value=cuda), \+ mock.patch.object(resource_plan, "discover_gpus", return_value=gpus), \+ mock.patch.object(resource_plan, "build_plan", return_value=plan), \+ mock.patch.object(resource_plan, "environment_for_plan",+ side_effect=fake_environment_for_plan):+ return coli.env_for(a)++ def _fake_gpu(self, index=0, name="NVIDIA GeForce RTX 5070 Ti",+ total_mib=16384, free_mib=15000):+ return {"index": index, "name": name,+ "total_bytes": total_mib * 1024 * 1024,+ "free_bytes": free_mib * 1024 * 1024}++ def test_win32_auto_enables_cuda_when_gpu_present(self):+ e = self._env_for("win32", cuda=True, gpus=[self._fake_gpu()])+ self.assertEqual(e["COLI_CUDA"], "1")+ self.assertEqual(e["COLI_GPUS"], "0")+ # VRAM budget is sized from free VRAM by build_plan (real minus reserve),+ # so it must be present and positive β never a guess or zero.+ self.assertIn("CUDA_EXPERT_GB", e)+ self.assertGreater(float(e["CUDA_EXPERT_GB"]), 0.0)+ # Dense offload is an explicit opt-in (matches --auto-tier): not set here.+ self.assertNotIn("CUDA_DENSE", e)++ def test_win32_falls_back_to_cpu_when_nvidia_smi_missing(self):+ # coli_cuda.dll present (cuda=True) but nvidia-smi absent (no GPUs found)+ # -> warn + CPU-only, never crash, never set COLI_CUDA.+ e = self._env_for("win32", cuda=True, gpus=[])+ self.assertNotIn("COLI_CUDA", e)+ self.assertNotIn("COLI_GPUS", e)+ self.assertNotIn("CUDA_EXPERT_GB", e)++ def test_win32_cpu_build_stays_silent(self):+ # No coli_cuda.dll (cuda=False) -> CPU build, nothing GPU-related emitted.+ e = self._env_for("win32", cuda=False, gpus=[self._fake_gpu()])+ self.assertNotIn("COLI_CUDA", e)+ self.assertNotIn("COLI_GPUS", e)++ def test_linux_bare_chat_not_auto_enabled(self):+ # The auto-enable is scoped to win32: a Linux bare chat with a GPU+ # present must NOT turn CUDA on (Linux keeps the explicit-flag UX).+ e = self._env_for("linux", cuda=True, gpus=[self._fake_gpu()])+ self.assertNotIn("COLI_CUDA", e)+ self.assertNotIn("CUDA_EXPERT_GB", e)++if __name__ == "__main__":unittest.main()diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex 9f0d7490..c3694c6d 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -105,6 +105,15 @@ Format: `VAR` β default β effect.| `COLI_CUDA_SHARED_W4A16_MIN_ROWS` | `32` | Min row count to engage the shared-MLP W4A16 kernel. || `COLI_METAL_UNTRACKED` | off (Metal only) | `=1` sets `MTLResourceHazardTrackingModeUntracked` on Metal buffers (reduces hazard-tracking overhead). |+> **Windows note.** On Windows, a bare `coli chat` / `coli run` / `coli serve`+> (no `--gpu`/`--vram`/`--auto-tier`) **auto-enables the GPU** when it detects a+> CUDA build (`coli_cuda.dll` next to the engine) and at least one GPU via+> `nvidia-smi`. The expert-tier VRAM budget is then sized automatically from the+> card's free VRAM (same computation as `--auto-tier`). If `nvidia-smi` is not on+> `PATH` the run falls back to CPU with a warning β pass `--vram N` (or add+> `nvidia-smi` to `PATH`) to enable CUDA in that case. `--gpu none` forces+> CPU-only. (Linux/macOS behaviour is unchanged: pass a flag to enable CUDA.)+---## Advanced / experimental / debug