Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

tests(qwen38), ci, docs: test_qwen38_dn_gpu on the tiny FP8 fixture; the layer on the card documented

kreuzzelge1a789865e2a
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0d587f7f..2c532fb6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1077,7 +1077,7 @@ jobs:
- name: Generate the upstream text fixture and run the token-and-logit oracle
run: make -C c qwen38-tiny-check
- name: Routed experts as the release ships them (FP8 shards), then the engine on the fake CUDA tier
- run: make -C c qwen38-tiny-fp8-check qwen38-tier-engine-check
+ run: make -C c qwen38-tiny-fp8-check qwen38-tier-engine-check qwen38-dn-gpu-check
- name: Routed experts as int4-g64 from the converter's sidecar
run: make -C c qwen38-tiny-int4-check
- name: MTP speculative decoding against plain decoding (lossless gate)
diff --git a/c/Makefile b/c/Makefile
index d1ac0f7f..eee85c61 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -521,12 +521,13 @@ TEST_RULES := $(patsubst tests/%,%,$(filter tests/test_%,$(RULE_NAMES)))
# since a bare invocation exits 2 -- and tests/test_e8x4g64_mint_load.py is what
# actually drives it end to end.
# test_qwen38_tier_engine drives qwen38.c over the generated FP8 fixture
-# (qwen38_tiny_fp8, gitignored); it runs from qwen38-tier-engine-check.
+# (qwen38_tiny_fp8, gitignored); it runs from qwen38-tier-engine-check. test_qwen38_dn_gpu does the same from
+# qwen38-dn-gpu-check.
TEST_EXCLUDE = test_uring test_deepseek_v4 test_v4_ownership test_v4_serve_framing \
test_segment_adapters_registration test_segment_adapters_real \
test_edge_adapters_registration test_edge_adapters_real \
test_xdna_execution test_xdna_failure \
- test_gsgemv_sse41 test_qgemv_sse41 test_olmoe_dot_i8_16_sse41 test_st_f16_bf16_simd_sse41 test_qwen38_tier_engine \
+ test_gsgemv_sse41 test_qgemv_sse41 test_olmoe_dot_i8_16_sse41 test_st_f16_bf16_simd_sse41 test_qwen38_tier_engine test_qwen38_dn_gpu \
test_e8x4g64_loader test_qwenimage_vae test_vk_tier \
test_i4_grouped_sse41 test_i4_grouped_sse41_o1 test_i4_grouped_sse41_no_contract
TEST_EXCLUDE += test_vk_chain
@@ -1404,6 +1405,9 @@ qwen38-tiny-fp8-check: qwen38-tiny-fp8-generate qwen38$(EXE)
qwen38-tier-engine-check: qwen38-tiny-fp8-generate tests/test_qwen38_tier_engine$(EXE)
./tests/test_qwen38_tier_engine$(EXE)
+qwen38-dn-gpu-check: qwen38-tiny-fp8-generate tests/test_qwen38_dn_gpu$(EXE)
+ ./tests/test_qwen38_dn_gpu$(EXE)
+
# The routed experts as int4-g64: tools/convert_qwen38_experts_int4.py writes
# the experts-int4g64/ sidecar next to the FP8 shards and qwen38 picks it up.
# The fixture is the FP8 one with its experts scaled by 3 (at the default
@@ -2454,6 +2458,10 @@ tests/test_qwen36_tier_int8_decode$(EXE): tests/test_qwen36_tier_int8_decode.c q
tests/test_qwen38_tier_engine$(EXE): tests/test_qwen38_tier_engine.c qwen38.c qwen36_tier.c $(VK_OBJ)
$(CC) $(CFLAGS) -DCOLI_CUDA $< $(VK_OBJ) -o $@ $(LDFLAGS)
+# Q38_DN_GPU=1 on the tiny FP8 fixture (fake backend, oracle): runs from qwen38-dn-gpu-check.
+tests/test_qwen38_dn_gpu$(EXE): tests/test_qwen38_dn_gpu.c qwen38.c qwen36_tier.c $(VK_OBJ)
+ $(CC) $(CFLAGS) -DCOLI_CUDA $< $(VK_OBJ) -o $@ $(LDFLAGS)
+
tests/test_serve_poll$(EXE): tests/test_serve_poll.c
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
diff --git a/c/tests/test_qwen38_dn_gpu.c b/c/tests/test_qwen38_dn_gpu.c
new file mode 100644
index 00000000..806ae0af
--- /dev/null
+++ b/c/tests/test_qwen38_dn_gpu.c
@@ -0,0 +1,67 @@
+/* Q38_DN_GPU=1: Qwen3.8's DeltaNet layer on the card gives the oracle's tokens.
+ *
+ * The tiny FP8 fixture (make qwen38-tiny-fp8-generate: 4 layers, two of them
+ * DeltaNet, hidden 32, 4 value heads of 4) runs the real engine on the fake
+ * CUDA tier with the whole dense trunk placed and computed by the fake
+ * (fake_dense_compute), once with the DeltaNet layers on the card and once
+ * without; both must stay within the oracle's limits (ref.json), the GPU run
+ * must have stepped every DeltaNet layer on the card, and the two runs' outputs
+ * must agree. The fake's coli_cuda_dn_step is the host-side reference of the
+ * CUDA kernels, with the sigmoid gate Qwen3.8's norm uses. */
+#define main qwen38_main_unused
+#include "../qwen38.c"
+#undef main
+
+#ifdef _WIN32
+#undef setenv
+#undef unsetenv
+#define unsetenv(name) _putenv_s(name, "")
+#endif
+
+#include "qwen36_fake_cuda.h"
+
+#include "../qwen36_tier.c"
+
+static int t_fails;
+static void tk(int ok, const char *what) { if (ok) { printf(" ok %s\n", what); return; } printf(" FAIL %s\n", what); t_fails++; }
+
+int main(void) {
+ setenv("SNAP", "./qwen38_tiny_fp8", 1);
+ setenv("OMP_NUM_THREADS", "2", 1);
+ setenv("NOSTREAM", "1", 1);
+ setenv("USAGE_SAVE", "0", 1);
+ setenv("COLI_CUDA", "1", 1);
+ setenv("COLI_GPUS", "0", 1);
+ setenv("QT_NO_WARMSTART", "1", 1);
+ setenv("QT_UPLOAD_SYNC", "1", 1);
+ setenv("HEAT_FILE", "", 1);
+ setenv("Q38_TRUNK_GPU", "1", 1);
+ setenv("Q38_TRUNK_MIN_KB", "0", 1); /* every dense matrix of the fixture is offered */
+ setenv("COLI_PLACE", "auto", 1);
+ setenv("CUDA_EXPERT_GB", "0.5", 1);
+ setenv("Q38_MTP", "0", 1); /* no draft: the verify path keeps the CPU anyway, keep the run plain */
+ fake_dense_compute = 1;
+ char *argv[] = { (char *)"qwen38", (char *)"1", (char *)"8", (char *)"./qwen38_tiny_fp8/ref.json" };
+
+ printf(" DeltaNet layers on the fake card\n");
+ setenv("Q38_DN_GPU", "1", 1);
+ fake_dn_steps = 0; fake_uploads = 0;
+ int rc = qwen38_main_unused(4, argv);
+ tk(rc == 0, "engine with the DeltaNet layers on the card stays within the oracle's limits");
+ tk(fake_dn_steps > 0, "decode tokens ran the DeltaNet layers on the card");
+ int on = 0; for (int l = 0; l < QT_DN_MAX_LAYERS; l++) on += qt_dn_gpu_ready(l);
+ tk(on == 2, "both DeltaNet layers of the fixture joined their projections on the card");
+ qt_shutdown();
+
+ printf(" the same on the CPU\n");
+ setenv("Q38_DN_GPU", "0", 1);
+ fake_dn_steps = 0;
+ rc = qwen38_main_unused(4, argv);
+ tk(rc == 0, "the CPU run stays within the oracle's limits");
+ tk(fake_dn_steps == 0, "no DeltaNet step reached the card");
+ qt_shutdown();
+
+ if (t_fails) { printf("test_qwen38_dn_gpu: %d failure(s)\n", t_fails); return 1; }
+ printf("OK test_qwen38_dn_gpu: Qwen3.8's DeltaNet layer on the card gives the oracle's tokens\n");
+ return 0;
+}
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index fbd9e5b3..6c2661d2 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -488,6 +488,7 @@ checkpoint layout and the text-only capability boundary.
| `Q38_MTP_FORCE`, `Q38_MTP_DUMP` | unset | Tests only: `reject` rolls back every draft, `accept` / `mixed` draft the reference's next token (always, every other one) in `ref.json` mode; `Q38_MTP_DUMP=<file>` writes every draft's row, token and logits. |
| `Q38_PREFILL_BATCH` | `1` (on) | Route prompt rows in bounded expert-major chunks and batch resident shared-expert/DeltaNet projections. `=0` restores row-at-a-time prompt execution for A/B diagnosis; decode is unchanged. |
| `Q38_TRUNK_CPU_INT8` | `1` (on) | The dense trunk (DeltaNet and attention projections, hyper-connection mixers, shared expert, router, lm_head; every matrix of at least `Q38_TRUNK_MIN_KB`) is kept on the CPU as int8 rows with one scale per row and the BF16 copy is released; `q38_weight_matmul` quantizes the activation to int8 and uses the integer kernels of `idot.h` for decode and prefill. `=0` keeps the BF16 rows and the f32 kernel (the numeric reference). See [qwen38.md](qwen38.md#the-trunk-on-the-cpu-int8-rows). |
+| `Q38_DN_GPU` | `0` (off) | CUDA expert tier with the trunk in VRAM: a decode token runs every DeltaNet layer whose `dnqkv`, `dnz` and `dnout` sit on one card end to end on that card (conv, recurrence, gated norm; state resident in VRAM). See [qwen38.md](qwen38.md#the-deltanet-layer-on-the-card-q38_dn_gpu1). |
| `COLI_MAP_EXPERTS` | `0` | Point the native-FP8 routed-expert slots (or the int4-g64 sidecar's records) at a read-only mapping of their shard instead of copying 14 MB per miss into a slab. Same variable as in `glm53`. |
| `Q38_FP8_KERNEL` | vector | The routed experts' e4m3 blocks are decoded eight at a time in registers and multiplied with FMA (AVX2 builds); `scalar` restores `quant.h`'s table kernel, which differs only by float summation order inside a block. |
| `COLI_TIMERS` | `0` (off) | Set to `1` for the detailed Qwen3.8 phase breakdown on stderr. The shared per-request `PROF` frame is emitted regardless. |
diff --git a/docs/qwen38.md b/docs/qwen38.md
index 1bdbd1d7..e82bbb8b 100644
--- a/docs/qwen38.md
+++ b/docs/qwen38.md
@@ -535,6 +535,31 @@ error, only wrong tokens, worse the more experts were resident. The dense
path has its own buffers now ([qwen36-cuda-tier.md](qwen36-cuda-tier.md));
qwen36 never called the dense path inside that window.
+### The DeltaNet layer on the card (`Q38_DN_GPU=1`)
+
+With the trunk in VRAM a DeltaNet layer still crosses the bus four times per
+decode token: the `dnqkv` and `dnz` results come down, the convolution, the
+recurrence and the gated norm run on the CPU, and the normed rows go back up
+for `dnout`. On qwen36 those round trips were 8 of 39 ms per token
+([qwen36-cuda-tier.md](qwen36-cuda-tier.md#the-deltanet-layer-on-the-card-q36_dn_gpu1));
+Qwen3.8 has 36 such layers of 48 value heads. `Q38_DN_GPU=1` keeps the layer on
+the card: for every DeltaNet layer whose `dnqkv`, `dnz` and `dnout` the placer
+put on one device, the conv ring, the recurrent state (48 x 128 x 128 f32, 3 MB
+per layer) and the conv/norm weights go there too, and a decode token runs the
+layer end to end in one device chain -- x up, the two in_proj GEMVs, causal
+conv + SiLU, per value head the L2 norms, the decay, the delta rule, the gated
+RMSNorm with `sigmoid(z)` (Qwen3.8's gate; Qwen3.6 uses `silu(z)`), the
+out_proj GEMV, out down. The two gates (`a`, `b`) stay on the CPU and travel as
+kernel parameters.
+
+The host arrays remain the state's owner: a prompt (`S > 1`), an MTP verify
+(`snap_after`: the CPU path takes the snapshot after its first row), a
+`--pin` snapshot and the prompt cache pull the state down first; a reset, a pin
+restore and a rejected draft's rollback invalidate the card's copy. A failing
+GPU step turns the layer off and the CPU continues from what the card holds.
+Opt-in; `tests/test_qwen38_dn_gpu.c` (`make qwen38-dn-gpu-check`) runs the tiny
+fixture's oracle with the layers on the fake card.
+
### The trunk on the CPU: int8 rows
Without a GPU the same trunk is the decode's floor: 8 GiB of BF16 read on