Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
qwen38: planner support (supports_accelerator) and the GPU section of the docs
kreuzzelg768fc6ce4e8a
diff --git a/c/family_registry.py b/c/family_registry.pyindex e13fb9b1..8343af0c 100644--- a/c/family_registry.py+++ b/c/family_registry.py@@ -1268,7 +1268,10 @@ FAMILIES = ("and prioritize correctness, consistency, and clarity in the final answer.""<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n""<|im_start|>assistant\n<think>\n"),- supports_accelerator=False,+ # CUDA VRAM expert tier (qwen36_tier.c, fp8 streaming mode): hot+ # routed experts get VRAM copies above the RAM LRU; the dense trunk+ # stays on the CPU for now (docs/qwen38.md, "GPU").+ supports_accelerator=True,),FamilyDescriptor(id="deepseek_v4",diff --git a/c/tests/test_cli_output.py b/c/tests/test_cli_output.pyindex 61cd3c45..330b736f 100644--- a/c/tests/test_cli_output.py+++ b/c/tests/test_cli_output.py@@ -368,13 +368,19 @@ class OmpThreadsForEveryEngineTest(unittest.TestCase):"OMP_DYNAMIC", "OMP_PROC_BIND", "OMP_PLACES"):self.assertNotIn(key, env)- def test_qwen38_rejects_gpu_and_vram_flags_before_binary_detection(self):- gpu_args = self.args();gpu_args.gpu="0";gpu_args.vram=0- with self.assertRaisesRegex(SystemExit, "CPU only.*--gpu"):- self.coli.env_for_engine(gpu_args, "qwen38")- vram_args = self.args();vram_args.gpu=None;vram_args.vram=4- with self.assertRaisesRegex(SystemExit, "CPU only.*--vram"):- self.coli.env_for_engine(vram_args, "qwen38")+ def test_qwen38_gpu_flags_need_the_cuda_build(self):+ """qwen38 has a GPU path now (expert tier, dense trunk), so --gpu and+ --vram are no longer refused up front as "CPU only". On a CPU-only+ binary they fail the way every accelerated engine's do: by asking for+ the CUDA build. cuda_binary() is mocked so the test does not depend on+ what happens to be built next to it."""+ with mock.patch.object(self.coli, "cuda_binary", return_value=None):+ gpu_args = self.args();gpu_args.gpu="0";gpu_args.vram=0+ with self.assertRaisesRegex(SystemExit, "--gpu needs the CUDA build"):+ self.coli.env_for_engine(gpu_args, "qwen38")+ vram_args = self.args();vram_args.gpu=None;vram_args.vram=4+ with self.assertRaisesRegex(SystemExit, "--vram needs the CUDA build"):+ self.coli.env_for_engine(vram_args, "qwen38")class ContextFlagHonestyTest(unittest.TestCase):diff --git a/c/tests/test_qwen38_tier_engine.c b/c/tests/test_qwen38_tier_engine.cindex 81178f70..b998f818 100644--- a/c/tests/test_qwen38_tier_engine.c+++ b/c/tests/test_qwen38_tier_engine.c@@ -52,7 +52,7 @@ int main(void) {setenv("COLI_PLACE", "off", 1);setenv("HEAT_FILE", "", 1);/* room for six of the sixteen experts, so promotion has to swap */- size_t exp_bytes = 3ull * 32 * 8 + 3ull * 1 * sizeof(float) + 4096;+ size_t exp_bytes = 3 * dev_alloc_footprint(32 * 8) + 3 * dev_alloc_footprint(1 * sizeof(float));char gb[64]; snprintf(gb, sizeof gb, "%.15f", (double)(6 * exp_bytes + exp_bytes / 2) / 1073741824.0);setenv("CUDA_EXPERT_GB", gb, 1);fake_ndev = 1; fake_uploads = 0; fake_lut_published = 0;@@ -66,7 +66,7 @@ int main(void) {tk(G.on, "tier started from q38_tier_start on the FP8 fixture");tk(G.wfmt == 8, "streaming mode, weight format 8");- tk(G.exp_bytes == exp_bytes, "exp_bytes = 3 x D x Ih + 3 block scales + slack");+ tk(G.exp_bytes == exp_bytes, "exp_bytes charged at cudaMalloc granularity (three matrices, three scale tables)");tk(fake_lut_published, "e4m3 LUT published before the first upload");tk(fake_uploads > 0 && fake_uploads % 3 == 0, "uploads come in gate/up/down triples");tk(last_fmt == 8 && last_bytes == (size_t)G.D * G.Ih, "fmt 8 uploads carry one byte per element");diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.pyindex ede6e88c..255fcb35 100644--- a/c/tests/test_resource_plan.py+++ b/c/tests/test_resource_plan.py@@ -687,7 +687,7 @@ memInfo.free: 23.50 GB (97%)self.assertEqual(plan["tiers"]["vram"]["devices"], [])self.assertIn("not detected", plan["warnings"][0])- def test_qwen38_cpu_only_plan_prices_heterogeneous_cache_and_exports_cap(self):+ def test_qwen38_plan_prices_heterogeneous_cache_exports_cap_and_plans_vram(self):config = {"model_type": "qwen4_exp","text_config": {@@ -735,12 +735,19 @@ memInfo.free: 23.50 GB (97%)"free_bytes": 14 * GB, "unified_memory": True}plan = build_plan(self.model, context=64, available_memory=16 * GB,available_disk=16 * GB, gpus=[gpu])- self.assertEqual(plan["tiers"]["vram"]["devices"], [])- self.assertEqual(plan["tiers"]["vram"]["budget_bytes"], 0)- self.assertFalse(any(item["target"] == "VRAM" for item in plan["decisions"]))+ # Qwen3.8 has the CUDA VRAM expert tier (fp8 streaming mode): a+ # qualified device is planned, the environment names it, and the RAM+ # cache cap is still exported -- VRAM is a stage above the LRU, not+ # a replacement for it.+ self.assertEqual([device["index"] for device in plan["tiers"]["vram"]["devices"]], [0])+ self.assertGreater(plan["tiers"]["vram"]["budget_bytes"], 0)+ self.assertTrue(any(item["target"] == "VRAM" for item in plan["decisions"]))cap = plan["tiers"]["ram"]["cache_slots_per_layer"]self.assertGreaterEqual(cap, 1)- self.assertEqual(environment_for_plan(plan)["COLI_PLAN_CAP"], str(cap))+ environment = environment_for_plan(plan)+ self.assertEqual(environment["COLI_PLAN_CAP"], str(cap))+ self.assertEqual(environment["COLI_CUDA"], "1")+ self.assertEqual(environment["COLI_GPU"], "0")for variable in ("Q38_NATIVE_FP8", "Q38_NATIVE_BF16"):with self.subTest(variable=variable), self.assertRaisesRegex(ValueError, "requires native expert storage"):@@ -748,12 +755,12 @@ memInfo.free: 23.50 GB (97%)plan["tiers"]["ram"]["cache_slots_per_layer"] = 0with self.assertRaisesRegex(ValueError, "one expert slot"):environment_for_plan(plan)- with self.assertRaisesRegex(ValueError, "CPU only"):- build_plan(self.model, context=64, gpu_indices=[0], available_memory=16 * GB,- available_disk=16 * GB, gpus=[gpu])- with self.assertRaisesRegex(ValueError, "CPU only"):- build_plan(self.model, context=64, vram_gb=4, available_memory=16 * GB,- available_disk=16 * GB, gpus=[gpu])+ selected = build_plan(self.model, context=64, gpu_indices=[0], available_memory=16 * GB,+ available_disk=16 * GB, gpus=[gpu])+ self.assertEqual([device["index"] for device in selected["tiers"]["vram"]["devices"]], [0])+ capped = build_plan(self.model, context=64, vram_gb=4, available_memory=16 * GB,+ available_disk=16 * GB, gpus=[gpu])+ self.assertLessEqual(capped["tiers"]["vram"]["budget_bytes"], 4 * GB)def test_cli_emits_versioned_json(self):cli = Path(__file__).parents[1] / "coli"diff --git a/docs/qwen36-cuda-tier.md b/docs/qwen36-cuda-tier.mdindex e2c7bd00..281da378 100644--- a/docs/qwen36-cuda-tier.md+++ b/docs/qwen36-cuda-tier.md@@ -6,14 +6,14 @@ in RAM, the **hot** ones are promoted into DEVICE_LOCAL VRAM across one ormore GPUs and computed there through the existing shared CUDA backend(`backend_cuda.cu` expert-group API — no new backend).-> **A second mode.** Besides the Qwen3.6 modes below, the tier has an *fp8-> streaming mode* (`qt_init_fp8`) for an engine whose experts do not all live-> in RAM: `cap` may be smaller than the expert count, the tier copies each-> expert's e4m3 slab and block scales when the engine reports it and keeps no-> pointer into the engine's slot, and promotion happens at report time instead-> of a warmstart. It is exercised by `tests/test_qwen36_tier_fp8.c`; the-> Qwen3.8 engine that uses it comes in its own PR. Everything below describes-> the Qwen3.6 modes unless it says otherwise.+> **Two engines.** The tier also serves Qwen3.8-Flash-Next (`c/qwen38.c`,+> [qwen38.md](qwen38.md#gpu-cuda-vram-expert-tier)) in its *fp8 streaming+> mode* (`qt_init_fp8`): experts do not all live in RAM there, `cap` may be+> smaller than the expert count, the tier copies each expert's e4m3 slab and+> block scales when the engine reports it and keeps no pointer into the+> engine's slot, and promotion happens at report time instead of a warmstart.+> Everything below describes the Qwen3.6 modes unless it says otherwise.+## How it worksdiff --git a/docs/qwen38.md b/docs/qwen38.mdindex 507b6389..92e07e05 100644--- a/docs/qwen38.md+++ b/docs/qwen38.md@@ -153,6 +153,63 @@ when nothing is cached and roughly half that at the cap-32 hit rate, so at thiscache size the engine spends about two thirds of every request waiting on thedisk, and the planner labels cold expert reads as the expected bottleneck.+## GPU: CUDA VRAM expert tier++With `CUDA=1` the engine links the same expert tier as Qwen3.6+(`c/qwen36_tier.c`, [qwen36-cuda-tier.md](qwen36-cuda-tier.md)) in its+**fp8 streaming mode**. Nothing about the model or the RAM cache changes:+experts still stream from disk into the per-layer LRU (`cap`), and the tier+keeps its own copies of the hot ones in VRAM as a third stage above it,+disk -> RAM LRU -> VRAM. Routed experts that are resident are computed on+the GPU (`grouped_hidden_f8w_dual` / `grouped_down_f8w` from #817: e4m3+bytes plus the checkpoint's `[ceil(O/128), ceil(I/128)]` block scales, no+conversion); the rest run on the CPU as before. Every expert the CPU+computes is reported to the tier, which copies the 4.7 MiB slab and the+three scale tables during that call -- the RAM slot may be recycled by the+next token -- and promotes it when it has room or when it is hotter than+the coldest resident expert on its device (budget-neutral swap). The dense+trunk, prefill, QSA and the PLE table stay on the CPU.++```bash+make -C c qwen38 CUDA=1 CUDA_ARCH=native # NVCC=/usr/bin/nvcc on distro CUDA+COLI_CUDA=1 COLI_GPUS=0,1 CUDA_EXPERT_GB=auto COLI_TIMERS=1 \+OMP_NUM_THREADS=<physical cores - 1> OMP_WAIT_POLICY=ACTIVE OMP_PROC_BIND=close \+SNAP=<checkpoint> N_NEW=200 ./c/qwen38 128 8 prompt.txt+```++`coli chat --gpu <n>` does the same through the planner (`supports_accelerator`+in the `qwen38` descriptor). The tier reports next to the cache hit rate:+resident experts, uploads, VRAM hits and misses, LFRU swaps. Requirements:+native FP8 routed experts (`Q38_NATIVE_FP8=1`, the default) with every layer's+block-scale bank resident; a checkpoint that falls back to the per-matrix+scale loader stays on the CPU and says so.++**What a VRAM budget buys.** The routing of Qwen3.8-Flash-Next has no hot set+that carries over between prompts: the 1,400 experts (5.8 % of the model) that+were hottest in one 100-token run covered a different prompt's routes at+chance level (4.7 %; two cards, 2,800 experts: 9.3 % against 11.5 % chance;+rank correlation of expert frequency per layer between the two runs -0.2). A+heat file from an earlier run therefore does not help, and the engine does no+warmstart. Inside a run the locality is strong -- on the route trace of a+315-token prompt plus 100 generated tokens, an LRU of 32 slots per layer+serves 55 % of decode routes, 64 serve 79 %, 128 serve 90 % -- so the tier+earns its VRAM by promotion at touch time: with the RAM LRU at cap 128 and one+8 GB card, 49 % of decode routes were served from VRAM on the real checkpoint.++**VRAM per expert.** The tier charges what `cudaMalloc` takes, not the payload:+an allocation above 1 MiB rounds up to a multiple of 2 MiB, so each 1.56 MiB+expert matrix occupies 2 MiB and an expert costs 6.03 MiB of VRAM for 4.69 MiB+of bytes. A 6.5 GB budget holds about 1,080 experts. Pooling experts into one+arena per device would recover the 22 %; it is not done yet.++**Where the time goes** (this checkpoint, Threadripper 3945WX 12 cores, 12+threads, cap 32, `COLI_TIMERS=1`): 4.6 s per decode token were 1.4 s dense+kernels (BF16 trunk and DeltaNet projections), 1.8 s expert disk reads, 0.9 s+routed-expert GEMV on the CPU (1.9 ms per expert: e4m3 decode, not bandwidth),+0.3 s the rest. The VRAM tier attacks the routed-expert GEMV and, together+with a larger cap, the disk reads; the dense trunk is the largest item and is+not on the GPU yet.+## Performance telemetryEvery served request reports its own routed-expert cache hit rate; persistent