Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

qwen38: planner support (supports_accelerator) and the GPU section of the docs

kreuzzelg768fc6ce4e8a
diff --git a/c/family_registry.py b/c/family_registry.py
index e13fb9b1..8343af0c 100644
--- a/c/family_registry.py
+++ b/c/family_registry.py
@@ -1268,7 +1268,10 @@ FAMILIES = (
"and prioritize correctness, consistency, and clarity in the final answer."
"<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n"
"<|im_start|>assistant\n<think>\n"),
- supports_accelerator=False,
+ # CUDA VRAM expert tier (qwen36_tier.c, fp8 streaming mode): hot
+ # routed experts get VRAM copies above the RAM LRU; the dense trunk
+ # stays on the CPU for now (docs/qwen38.md, "GPU").
+ supports_accelerator=True,
),
FamilyDescriptor(
id="deepseek_v4",
diff --git a/c/tests/test_cli_output.py b/c/tests/test_cli_output.py
index 61cd3c45..330b736f 100644
--- a/c/tests/test_cli_output.py
+++ b/c/tests/test_cli_output.py
@@ -368,13 +368,19 @@ class OmpThreadsForEveryEngineTest(unittest.TestCase):
"OMP_DYNAMIC", "OMP_PROC_BIND", "OMP_PLACES"):
self.assertNotIn(key, env)
- def test_qwen38_rejects_gpu_and_vram_flags_before_binary_detection(self):
- gpu_args = self.args();gpu_args.gpu="0";gpu_args.vram=0
- with self.assertRaisesRegex(SystemExit, "CPU only.*--gpu"):
- self.coli.env_for_engine(gpu_args, "qwen38")
- vram_args = self.args();vram_args.gpu=None;vram_args.vram=4
- with self.assertRaisesRegex(SystemExit, "CPU only.*--vram"):
- self.coli.env_for_engine(vram_args, "qwen38")
+ def test_qwen38_gpu_flags_need_the_cuda_build(self):
+ """qwen38 has a GPU path now (expert tier, dense trunk), so --gpu and
+ --vram are no longer refused up front as "CPU only". On a CPU-only
+ binary they fail the way every accelerated engine's do: by asking for
+ the CUDA build. cuda_binary() is mocked so the test does not depend on
+ what happens to be built next to it."""
+ with mock.patch.object(self.coli, "cuda_binary", return_value=None):
+ gpu_args = self.args();gpu_args.gpu="0";gpu_args.vram=0
+ with self.assertRaisesRegex(SystemExit, "--gpu needs the CUDA build"):
+ self.coli.env_for_engine(gpu_args, "qwen38")
+ vram_args = self.args();vram_args.gpu=None;vram_args.vram=4
+ with self.assertRaisesRegex(SystemExit, "--vram needs the CUDA build"):
+ self.coli.env_for_engine(vram_args, "qwen38")
class ContextFlagHonestyTest(unittest.TestCase):
diff --git a/c/tests/test_qwen38_tier_engine.c b/c/tests/test_qwen38_tier_engine.c
index 81178f70..b998f818 100644
--- a/c/tests/test_qwen38_tier_engine.c
+++ b/c/tests/test_qwen38_tier_engine.c
@@ -52,7 +52,7 @@ int main(void) {
setenv("COLI_PLACE", "off", 1);
setenv("HEAT_FILE", "", 1);
/* room for six of the sixteen experts, so promotion has to swap */
- size_t exp_bytes = 3ull * 32 * 8 + 3ull * 1 * sizeof(float) + 4096;
+ size_t exp_bytes = 3 * dev_alloc_footprint(32 * 8) + 3 * dev_alloc_footprint(1 * sizeof(float));
char gb[64]; snprintf(gb, sizeof gb, "%.15f", (double)(6 * exp_bytes + exp_bytes / 2) / 1073741824.0);
setenv("CUDA_EXPERT_GB", gb, 1);
fake_ndev = 1; fake_uploads = 0; fake_lut_published = 0;
@@ -66,7 +66,7 @@ int main(void) {
tk(G.on, "tier started from q38_tier_start on the FP8 fixture");
tk(G.wfmt == 8, "streaming mode, weight format 8");
- tk(G.exp_bytes == exp_bytes, "exp_bytes = 3 x D x Ih + 3 block scales + slack");
+ tk(G.exp_bytes == exp_bytes, "exp_bytes charged at cudaMalloc granularity (three matrices, three scale tables)");
tk(fake_lut_published, "e4m3 LUT published before the first upload");
tk(fake_uploads > 0 && fake_uploads % 3 == 0, "uploads come in gate/up/down triples");
tk(last_fmt == 8 && last_bytes == (size_t)G.D * G.Ih, "fmt 8 uploads carry one byte per element");
diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.py
index ede6e88c..255fcb35 100644
--- a/c/tests/test_resource_plan.py
+++ b/c/tests/test_resource_plan.py
@@ -687,7 +687,7 @@ memInfo.free: 23.50 GB (97%)
self.assertEqual(plan["tiers"]["vram"]["devices"], [])
self.assertIn("not detected", plan["warnings"][0])
- def test_qwen38_cpu_only_plan_prices_heterogeneous_cache_and_exports_cap(self):
+ def test_qwen38_plan_prices_heterogeneous_cache_exports_cap_and_plans_vram(self):
config = {
"model_type": "qwen4_exp",
"text_config": {
@@ -735,12 +735,19 @@ memInfo.free: 23.50 GB (97%)
"free_bytes": 14 * GB, "unified_memory": True}
plan = build_plan(self.model, context=64, available_memory=16 * GB,
available_disk=16 * GB, gpus=[gpu])
- self.assertEqual(plan["tiers"]["vram"]["devices"], [])
- self.assertEqual(plan["tiers"]["vram"]["budget_bytes"], 0)
- self.assertFalse(any(item["target"] == "VRAM" for item in plan["decisions"]))
+ # Qwen3.8 has the CUDA VRAM expert tier (fp8 streaming mode): a
+ # qualified device is planned, the environment names it, and the RAM
+ # cache cap is still exported -- VRAM is a stage above the LRU, not
+ # a replacement for it.
+ self.assertEqual([device["index"] for device in plan["tiers"]["vram"]["devices"]], [0])
+ self.assertGreater(plan["tiers"]["vram"]["budget_bytes"], 0)
+ self.assertTrue(any(item["target"] == "VRAM" for item in plan["decisions"]))
cap = plan["tiers"]["ram"]["cache_slots_per_layer"]
self.assertGreaterEqual(cap, 1)
- self.assertEqual(environment_for_plan(plan)["COLI_PLAN_CAP"], str(cap))
+ environment = environment_for_plan(plan)
+ self.assertEqual(environment["COLI_PLAN_CAP"], str(cap))
+ self.assertEqual(environment["COLI_CUDA"], "1")
+ self.assertEqual(environment["COLI_GPU"], "0")
for variable in ("Q38_NATIVE_FP8", "Q38_NATIVE_BF16"):
with self.subTest(variable=variable), self.assertRaisesRegex(
ValueError, "requires native expert storage"):
@@ -748,12 +755,12 @@ memInfo.free: 23.50 GB (97%)
plan["tiers"]["ram"]["cache_slots_per_layer"] = 0
with self.assertRaisesRegex(ValueError, "one expert slot"):
environment_for_plan(plan)
- with self.assertRaisesRegex(ValueError, "CPU only"):
- build_plan(self.model, context=64, gpu_indices=[0], available_memory=16 * GB,
- available_disk=16 * GB, gpus=[gpu])
- with self.assertRaisesRegex(ValueError, "CPU only"):
- build_plan(self.model, context=64, vram_gb=4, available_memory=16 * GB,
- available_disk=16 * GB, gpus=[gpu])
+ selected = build_plan(self.model, context=64, gpu_indices=[0], available_memory=16 * GB,
+ available_disk=16 * GB, gpus=[gpu])
+ self.assertEqual([device["index"] for device in selected["tiers"]["vram"]["devices"]], [0])
+ capped = build_plan(self.model, context=64, vram_gb=4, available_memory=16 * GB,
+ available_disk=16 * GB, gpus=[gpu])
+ self.assertLessEqual(capped["tiers"]["vram"]["budget_bytes"], 4 * GB)
def test_cli_emits_versioned_json(self):
cli = Path(__file__).parents[1] / "coli"
diff --git a/docs/qwen36-cuda-tier.md b/docs/qwen36-cuda-tier.md
index e2c7bd00..281da378 100644
--- a/docs/qwen36-cuda-tier.md
+++ b/docs/qwen36-cuda-tier.md
@@ -6,14 +6,14 @@ in RAM, the **hot** ones are promoted into DEVICE_LOCAL VRAM across one or
more GPUs and computed there through the existing shared CUDA backend
(`backend_cuda.cu` expert-group API — no new backend).
-> **A second mode.** Besides the Qwen3.6 modes below, the tier has an *fp8
-> streaming mode* (`qt_init_fp8`) for an engine whose experts do not all live
-> in RAM: `cap` may be smaller than the expert count, the tier copies each
-> expert's e4m3 slab and block scales when the engine reports it and keeps no
-> pointer into the engine's slot, and promotion happens at report time instead
-> of a warmstart. It is exercised by `tests/test_qwen36_tier_fp8.c`; the
-> Qwen3.8 engine that uses it comes in its own PR. Everything below describes
-> the Qwen3.6 modes unless it says otherwise.
+> **Two engines.** The tier also serves Qwen3.8-Flash-Next (`c/qwen38.c`,
+> [qwen38.md](qwen38.md#gpu-cuda-vram-expert-tier)) in its *fp8 streaming
+> mode* (`qt_init_fp8`): experts do not all live in RAM there, `cap` may be
+> smaller than the expert count, the tier copies each expert's e4m3 slab and
+> block scales when the engine reports it and keeps no pointer into the
+> engine's slot, and promotion happens at report time instead of a warmstart.
+> Everything below describes the Qwen3.6 modes unless it says otherwise.
+
## How it works
diff --git a/docs/qwen38.md b/docs/qwen38.md
index 507b6389..92e07e05 100644
--- a/docs/qwen38.md
+++ b/docs/qwen38.md
@@ -153,6 +153,63 @@ when nothing is cached and roughly half that at the cap-32 hit rate, so at this
cache size the engine spends about two thirds of every request waiting on the
disk, and the planner labels cold expert reads as the expected bottleneck.
+## GPU: CUDA VRAM expert tier
+
+With `CUDA=1` the engine links the same expert tier as Qwen3.6
+(`c/qwen36_tier.c`, [qwen36-cuda-tier.md](qwen36-cuda-tier.md)) in its
+**fp8 streaming mode**. Nothing about the model or the RAM cache changes:
+experts still stream from disk into the per-layer LRU (`cap`), and the tier
+keeps its own copies of the hot ones in VRAM as a third stage above it,
+disk -> RAM LRU -> VRAM. Routed experts that are resident are computed on
+the GPU (`grouped_hidden_f8w_dual` / `grouped_down_f8w` from #817: e4m3
+bytes plus the checkpoint's `[ceil(O/128), ceil(I/128)]` block scales, no
+conversion); the rest run on the CPU as before. Every expert the CPU
+computes is reported to the tier, which copies the 4.7 MiB slab and the
+three scale tables during that call -- the RAM slot may be recycled by the
+next token -- and promotes it when it has room or when it is hotter than
+the coldest resident expert on its device (budget-neutral swap). The dense
+trunk, prefill, QSA and the PLE table stay on the CPU.
+
+```bash
+make -C c qwen38 CUDA=1 CUDA_ARCH=native # NVCC=/usr/bin/nvcc on distro CUDA
+COLI_CUDA=1 COLI_GPUS=0,1 CUDA_EXPERT_GB=auto COLI_TIMERS=1 \
+OMP_NUM_THREADS=<physical cores - 1> OMP_WAIT_POLICY=ACTIVE OMP_PROC_BIND=close \
+SNAP=<checkpoint> N_NEW=200 ./c/qwen38 128 8 prompt.txt
+```
+
+`coli chat --gpu <n>` does the same through the planner (`supports_accelerator`
+in the `qwen38` descriptor). The tier reports next to the cache hit rate:
+resident experts, uploads, VRAM hits and misses, LFRU swaps. Requirements:
+native FP8 routed experts (`Q38_NATIVE_FP8=1`, the default) with every layer's
+block-scale bank resident; a checkpoint that falls back to the per-matrix
+scale loader stays on the CPU and says so.
+
+**What a VRAM budget buys.** The routing of Qwen3.8-Flash-Next has no hot set
+that carries over between prompts: the 1,400 experts (5.8 % of the model) that
+were hottest in one 100-token run covered a different prompt's routes at
+chance level (4.7 %; two cards, 2,800 experts: 9.3 % against 11.5 % chance;
+rank correlation of expert frequency per layer between the two runs -0.2). A
+heat file from an earlier run therefore does not help, and the engine does no
+warmstart. Inside a run the locality is strong -- on the route trace of a
+315-token prompt plus 100 generated tokens, an LRU of 32 slots per layer
+serves 55 % of decode routes, 64 serve 79 %, 128 serve 90 % -- so the tier
+earns its VRAM by promotion at touch time: with the RAM LRU at cap 128 and one
+8 GB card, 49 % of decode routes were served from VRAM on the real checkpoint.
+
+**VRAM per expert.** The tier charges what `cudaMalloc` takes, not the payload:
+an allocation above 1 MiB rounds up to a multiple of 2 MiB, so each 1.56 MiB
+expert matrix occupies 2 MiB and an expert costs 6.03 MiB of VRAM for 4.69 MiB
+of bytes. A 6.5 GB budget holds about 1,080 experts. Pooling experts into one
+arena per device would recover the 22 %; it is not done yet.
+
+**Where the time goes** (this checkpoint, Threadripper 3945WX 12 cores, 12
+threads, cap 32, `COLI_TIMERS=1`): 4.6 s per decode token were 1.4 s dense
+kernels (BF16 trunk and DeltaNet projections), 1.8 s expert disk reads, 0.9 s
+routed-expert GEMV on the CPU (1.9 ms per expert: e4m3 decode, not bandwidth),
+0.3 s the rest. The VRAM tier attacks the routed-expert GEMV and, together
+with a larger cap, the disk reads; the dense trunk is the largest item and is
+not on the GPU yet.
+
## Performance telemetry
Every served request reports its own routed-expert cache hit rate; persistent