Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
tests(qwen38), ci, docs: test_qwen38_dn_gpu on the tiny FP8 fixture; the layer on the card documented
kreuzzelge1a789865e2a
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 0d587f7f..2c532fb6 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -1077,7 +1077,7 @@ jobs:- name: Generate the upstream text fixture and run the token-and-logit oraclerun: make -C c qwen38-tiny-check- name: Routed experts as the release ships them (FP8 shards), then the engine on the fake CUDA tier- run: make -C c qwen38-tiny-fp8-check qwen38-tier-engine-check+ run: make -C c qwen38-tiny-fp8-check qwen38-tier-engine-check qwen38-dn-gpu-check- name: Routed experts as int4-g64 from the converter's sidecarrun: make -C c qwen38-tiny-int4-check- name: MTP speculative decoding against plain decoding (lossless gate)diff --git a/c/Makefile b/c/Makefileindex d1ac0f7f..eee85c61 100644--- a/c/Makefile+++ b/c/Makefile@@ -521,12 +521,13 @@ TEST_RULES := $(patsubst tests/%,%,$(filter tests/test_%,$(RULE_NAMES)))# since a bare invocation exits 2 -- and tests/test_e8x4g64_mint_load.py is what# actually drives it end to end.# test_qwen38_tier_engine drives qwen38.c over the generated FP8 fixture-# (qwen38_tiny_fp8, gitignored); it runs from qwen38-tier-engine-check.+# (qwen38_tiny_fp8, gitignored); it runs from qwen38-tier-engine-check. test_qwen38_dn_gpu does the same from+# qwen38-dn-gpu-check.TEST_EXCLUDE = test_uring test_deepseek_v4 test_v4_ownership test_v4_serve_framing \test_segment_adapters_registration test_segment_adapters_real \test_edge_adapters_registration test_edge_adapters_real \test_xdna_execution test_xdna_failure \- test_gsgemv_sse41 test_qgemv_sse41 test_olmoe_dot_i8_16_sse41 test_st_f16_bf16_simd_sse41 test_qwen38_tier_engine \+ test_gsgemv_sse41 test_qgemv_sse41 test_olmoe_dot_i8_16_sse41 test_st_f16_bf16_simd_sse41 test_qwen38_tier_engine test_qwen38_dn_gpu \test_e8x4g64_loader test_qwenimage_vae test_vk_tier \test_i4_grouped_sse41 test_i4_grouped_sse41_o1 test_i4_grouped_sse41_no_contractTEST_EXCLUDE += test_vk_chain@@ -1404,6 +1405,9 @@ qwen38-tiny-fp8-check: qwen38-tiny-fp8-generate qwen38$(EXE)qwen38-tier-engine-check: qwen38-tiny-fp8-generate tests/test_qwen38_tier_engine$(EXE)./tests/test_qwen38_tier_engine$(EXE)+qwen38-dn-gpu-check: qwen38-tiny-fp8-generate tests/test_qwen38_dn_gpu$(EXE)+ ./tests/test_qwen38_dn_gpu$(EXE)+# The routed experts as int4-g64: tools/convert_qwen38_experts_int4.py writes# the experts-int4g64/ sidecar next to the FP8 shards and qwen38 picks it up.# The fixture is the FP8 one with its experts scaled by 3 (at the default@@ -2454,6 +2458,10 @@ tests/test_qwen36_tier_int8_decode$(EXE): tests/test_qwen36_tier_int8_decode.c qtests/test_qwen38_tier_engine$(EXE): tests/test_qwen38_tier_engine.c qwen38.c qwen36_tier.c $(VK_OBJ)$(CC) $(CFLAGS) -DCOLI_CUDA $< $(VK_OBJ) -o $@ $(LDFLAGS)+# Q38_DN_GPU=1 on the tiny FP8 fixture (fake backend, oracle): runs from qwen38-dn-gpu-check.+tests/test_qwen38_dn_gpu$(EXE): tests/test_qwen38_dn_gpu.c qwen38.c qwen36_tier.c $(VK_OBJ)+ $(CC) $(CFLAGS) -DCOLI_CUDA $< $(VK_OBJ) -o $@ $(LDFLAGS)+tests/test_serve_poll$(EXE): tests/test_serve_poll.c$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)diff --git a/c/tests/test_qwen38_dn_gpu.c b/c/tests/test_qwen38_dn_gpu.cnew file mode 100644index 00000000..806ae0af--- /dev/null+++ b/c/tests/test_qwen38_dn_gpu.c@@ -0,0 +1,67 @@+/* Q38_DN_GPU=1: Qwen3.8's DeltaNet layer on the card gives the oracle's tokens.+ *+ * The tiny FP8 fixture (make qwen38-tiny-fp8-generate: 4 layers, two of them+ * DeltaNet, hidden 32, 4 value heads of 4) runs the real engine on the fake+ * CUDA tier with the whole dense trunk placed and computed by the fake+ * (fake_dense_compute), once with the DeltaNet layers on the card and once+ * without; both must stay within the oracle's limits (ref.json), the GPU run+ * must have stepped every DeltaNet layer on the card, and the two runs' outputs+ * must agree. The fake's coli_cuda_dn_step is the host-side reference of the+ * CUDA kernels, with the sigmoid gate Qwen3.8's norm uses. */+#define main qwen38_main_unused+#include "../qwen38.c"+#undef main++#ifdef _WIN32+#undef setenv+#undef unsetenv+#define unsetenv(name) _putenv_s(name, "")+#endif++#include "qwen36_fake_cuda.h"++#include "../qwen36_tier.c"++static int t_fails;+static void tk(int ok, const char *what) { if (ok) { printf(" ok %s\n", what); return; } printf(" FAIL %s\n", what); t_fails++; }++int main(void) {+ setenv("SNAP", "./qwen38_tiny_fp8", 1);+ setenv("OMP_NUM_THREADS", "2", 1);+ setenv("NOSTREAM", "1", 1);+ setenv("USAGE_SAVE", "0", 1);+ setenv("COLI_CUDA", "1", 1);+ setenv("COLI_GPUS", "0", 1);+ setenv("QT_NO_WARMSTART", "1", 1);+ setenv("QT_UPLOAD_SYNC", "1", 1);+ setenv("HEAT_FILE", "", 1);+ setenv("Q38_TRUNK_GPU", "1", 1);+ setenv("Q38_TRUNK_MIN_KB", "0", 1); /* every dense matrix of the fixture is offered */+ setenv("COLI_PLACE", "auto", 1);+ setenv("CUDA_EXPERT_GB", "0.5", 1);+ setenv("Q38_MTP", "0", 1); /* no draft: the verify path keeps the CPU anyway, keep the run plain */+ fake_dense_compute = 1;+ char *argv[] = { (char *)"qwen38", (char *)"1", (char *)"8", (char *)"./qwen38_tiny_fp8/ref.json" };++ printf(" DeltaNet layers on the fake card\n");+ setenv("Q38_DN_GPU", "1", 1);+ fake_dn_steps = 0; fake_uploads = 0;+ int rc = qwen38_main_unused(4, argv);+ tk(rc == 0, "engine with the DeltaNet layers on the card stays within the oracle's limits");+ tk(fake_dn_steps > 0, "decode tokens ran the DeltaNet layers on the card");+ int on = 0; for (int l = 0; l < QT_DN_MAX_LAYERS; l++) on += qt_dn_gpu_ready(l);+ tk(on == 2, "both DeltaNet layers of the fixture joined their projections on the card");+ qt_shutdown();++ printf(" the same on the CPU\n");+ setenv("Q38_DN_GPU", "0", 1);+ fake_dn_steps = 0;+ rc = qwen38_main_unused(4, argv);+ tk(rc == 0, "the CPU run stays within the oracle's limits");+ tk(fake_dn_steps == 0, "no DeltaNet step reached the card");+ qt_shutdown();++ if (t_fails) { printf("test_qwen38_dn_gpu: %d failure(s)\n", t_fails); return 1; }+ printf("OK test_qwen38_dn_gpu: Qwen3.8's DeltaNet layer on the card gives the oracle's tokens\n");+ return 0;+}diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex fbd9e5b3..6c2661d2 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -488,6 +488,7 @@ checkpoint layout and the text-only capability boundary.| `Q38_MTP_FORCE`, `Q38_MTP_DUMP` | unset | Tests only: `reject` rolls back every draft, `accept` / `mixed` draft the reference's next token (always, every other one) in `ref.json` mode; `Q38_MTP_DUMP=<file>` writes every draft's row, token and logits. || `Q38_PREFILL_BATCH` | `1` (on) | Route prompt rows in bounded expert-major chunks and batch resident shared-expert/DeltaNet projections. `=0` restores row-at-a-time prompt execution for A/B diagnosis; decode is unchanged. || `Q38_TRUNK_CPU_INT8` | `1` (on) | The dense trunk (DeltaNet and attention projections, hyper-connection mixers, shared expert, router, lm_head; every matrix of at least `Q38_TRUNK_MIN_KB`) is kept on the CPU as int8 rows with one scale per row and the BF16 copy is released; `q38_weight_matmul` quantizes the activation to int8 and uses the integer kernels of `idot.h` for decode and prefill. `=0` keeps the BF16 rows and the f32 kernel (the numeric reference). See [qwen38.md](qwen38.md#the-trunk-on-the-cpu-int8-rows). |+| `Q38_DN_GPU` | `0` (off) | CUDA expert tier with the trunk in VRAM: a decode token runs every DeltaNet layer whose `dnqkv`, `dnz` and `dnout` sit on one card end to end on that card (conv, recurrence, gated norm; state resident in VRAM). See [qwen38.md](qwen38.md#the-deltanet-layer-on-the-card-q38_dn_gpu1). || `COLI_MAP_EXPERTS` | `0` | Point the native-FP8 routed-expert slots (or the int4-g64 sidecar's records) at a read-only mapping of their shard instead of copying 14 MB per miss into a slab. Same variable as in `glm53`. || `Q38_FP8_KERNEL` | vector | The routed experts' e4m3 blocks are decoded eight at a time in registers and multiplied with FMA (AVX2 builds); `scalar` restores `quant.h`'s table kernel, which differs only by float summation order inside a block. || `COLI_TIMERS` | `0` (off) | Set to `1` for the detailed Qwen3.8 phase breakdown on stderr. The shared per-request `PROF` frame is emitted regardless. |diff --git a/docs/qwen38.md b/docs/qwen38.mdindex 1bdbd1d7..e82bbb8b 100644--- a/docs/qwen38.md+++ b/docs/qwen38.md@@ -535,6 +535,31 @@ error, only wrong tokens, worse the more experts were resident. The densepath has its own buffers now ([qwen36-cuda-tier.md](qwen36-cuda-tier.md));qwen36 never called the dense path inside that window.+### The DeltaNet layer on the card (`Q38_DN_GPU=1`)++With the trunk in VRAM a DeltaNet layer still crosses the bus four times per+decode token: the `dnqkv` and `dnz` results come down, the convolution, the+recurrence and the gated norm run on the CPU, and the normed rows go back up+for `dnout`. On qwen36 those round trips were 8 of 39 ms per token+([qwen36-cuda-tier.md](qwen36-cuda-tier.md#the-deltanet-layer-on-the-card-q36_dn_gpu1));+Qwen3.8 has 36 such layers of 48 value heads. `Q38_DN_GPU=1` keeps the layer on+the card: for every DeltaNet layer whose `dnqkv`, `dnz` and `dnout` the placer+put on one device, the conv ring, the recurrent state (48 x 128 x 128 f32, 3 MB+per layer) and the conv/norm weights go there too, and a decode token runs the+layer end to end in one device chain -- x up, the two in_proj GEMVs, causal+conv + SiLU, per value head the L2 norms, the decay, the delta rule, the gated+RMSNorm with `sigmoid(z)` (Qwen3.8's gate; Qwen3.6 uses `silu(z)`), the+out_proj GEMV, out down. The two gates (`a`, `b`) stay on the CPU and travel as+kernel parameters.++The host arrays remain the state's owner: a prompt (`S > 1`), an MTP verify+(`snap_after`: the CPU path takes the snapshot after its first row), a+`--pin` snapshot and the prompt cache pull the state down first; a reset, a pin+restore and a rejected draft's rollback invalidate the card's copy. A failing+GPU step turns the layer off and the CPU continues from what the card holds.+Opt-in; `tests/test_qwen38_dn_gpu.c` (`make qwen38-dn-gpu-check`) runs the tiny+fixture's oracle with the layers on the fake card.+### The trunk on the CPU: int8 rowsWithout a GPU the same trunk is the decode's floor: 8 GiB of BF16 read on