Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
qwen36: batch resident DeltaNet input projections
ZacharyZcR0fc47ccdba57
diff --git a/c/Makefile b/c/Makefileindex 341ad971..47eee120 100644--- a/c/Makefile+++ b/c/Makefile@@ -2180,3 +2180,7 @@ tests/test_kv_prefix$(EXE): tests/test_kv_prefix.c kv_prefix.htests/test_kimi_request_state$(EXE): tests/test_kimi_request_state.c kimi_k3.c kv_prefix.h serve_codec.h st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h idot.h omp_tune.h route_trace.h$(CC) $(NOCUDA_CFLAGS) tests/test_kimi_request_state.c $(VK_OBJ) -o tests/test_kimi_request_state$(EXE) $(NOCUDA_LDFLAGS)++# DeltaNet input projection batching, state parity and block failure fallback.+tests/test_qwen36_dnproj_batch$(EXE): tests/test_qwen36_dnproj_batch.c tests/qwen36_fake_cuda.h qwen36.c qwen36_tier.c qwen36_tier.h backend_cuda.h compat.h quant.h+ $(CC) $(CFLAGS) -DCOLI_CUDA $< -o $@ $(LDFLAGS)diff --git a/c/qwen36.c b/c/qwen36.cindex 8fa7c56b..17c8d419 100644--- a/c/qwen36.c+++ b/c/qwen36.c@@ -2422,6 +2422,14 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {* split conv_out -> q_in/k_in/v_in; repeat_interleave q,k by rep; l2norm* (q scaled by 1/sqrt(kdim)); recurrence S[h]*=exp(g); kv=k@S; delta=(v-kv)*beta;* S+=k (x) delta; out=q@S; per-head Gated RMSNorm (plain weight) -> out_proj. */+/* Bound both the host result block and device input/output staging. */+static int dnproj_batch_rows(int S, int H, int O) {+ int64_t rows = (32LL << 20) / (((int64_t)H + O) * sizeof(float));+ if (rows < 1) rows = 1;+ if (rows > 256) rows = 256;+ return S < rows ? S : (int)rows;+}+static void deltanet(Model *m, Layer *l, int layer, float *x, int S, int pos_base, float *out) {(void)pos_base;Cfg *c = &m->c;@@ -2433,12 +2441,12 @@ static void deltanet(Model *m, Layer *l, int layer, float *x, int S, int pos_basfloat scale = 1.f / sqrtf((float)kdim);int H = c->hidden;- /* qkv and z live in ONE buffer: the fused GPU projection writes- * [conv_dim ++ value_dim] in a single GEMV, and the CPU fallback fills the- * same two regions. Either way the code below reads qkv/z unchanged. */- float *qkvz = falloc((int64_t)conv_dim + value_dim);- float *qkv = qkvz;- float *z = qkvz + conv_dim;+ /* Input projections have no recurrent dependency. Keep qkv ++ z for a+ * bounded block, then consume rows in order through conv and recurrence. */+ int proj_dim = conv_dim + value_dim;+ int B = qt_dnproj_ready(layer) ? dnproj_batch_rows(S, H, proj_dim) : 1;+ float *qkvz = falloc((int64_t)B * proj_dim);+ int gpu_block = 0;float *b = falloc(vh);float *a = falloc(vh);float *beta= falloc(vh);@@ -2458,9 +2466,13 @@ static void deltanet(Model *m, Layer *l, int layer, float *x, int S, int pos_basconst float *xs = x + (int64_t)s * H;extern double g_dn_sub[4];double _d0 = tm_now();- /* projections (single-token matmuls). One fused GEMV when this layer's- * dnproj is placed on a GPU, the two CPU matmuls otherwise. */- if (!qt_dnproj_matmul(layer, qkvz, xs, H, conv_dim + value_dim)) {+ if (s % B == 0) {+ int rows = S - s < B ? S - s : B;+ gpu_block = qt_dnproj_matmul_batch(layer, qkvz, xs, rows, H, proj_dim);+ }+ float *qkv = qkvz + (int64_t)(s % B) * proj_dim;+ float *z = qkv + conv_dim;+ if (!gpu_block) {matmul_d(qkv, xs, l->dn_qkv, 1, H, conv_dim);matmul_d(z, xs, l->dn_z, 1, H, value_dim);}diff --git a/c/qwen36_tier.c b/c/qwen36_tier.cindex 2b2e24e5..98cd4dca 100644--- a/c/qwen36_tier.c+++ b/c/qwen36_tier.c@@ -810,15 +810,22 @@ int qt_dense_matmul(int h, float *y, const float *x, int I, int O){}int qt_dense_count(void){ return G_dense_n; }-int qt_dnproj_matmul(int layer, float *y, const float *x, int I, int O){- if(layer < 0 || layer >= QT_DN_MAX_LAYERS || !G_dnp[layer].on) return 0;- if(coli_cuda_matmul(&G_dnp[layer].t,y,x,NULL,NULL,1,1,I,O,G_dnp[layer].dev,0))+int qt_dnproj_ready(int layer){+ return layer >= 0 && layer < QT_DN_MAX_LAYERS && G_dnp[layer].on;+}+int qt_dnproj_matmul_batch(int layer, float *y, const float *x, int S, int I, int O){+ if(!qt_dnproj_ready(layer) || S <= 0) return 0;+ if(coli_cuda_matmul(&G_dnp[layer].t,y,x,NULL,NULL,1,S,I,O,G_dnp[layer].dev,0))return 1;fprintf(stderr,"[dnp] layer %d GPU matmul failed; CPU from here on\n", layer);G_dnp[layer].on = 0;return 0;}+int qt_dnproj_matmul(int layer, float *y, const float *x, int I, int O){+ return qt_dnproj_matmul_batch(layer, y, x, 1, I, O);+}+int qt_lmhead_matmul(float *y, const float *x, int I, int O){if(!G_lmh.on) return 0;/* cached-tensor path: upload params are ignored once *t exists */diff --git a/c/qwen36_tier.h b/c/qwen36_tier.hindex 228288d6..1098a0c3 100644--- a/c/qwen36_tier.h+++ b/c/qwen36_tier.h@@ -78,6 +78,8 @@ void qt_trunk_withdraw(const char *why);int qt_dnproj_init(int layer, const int8_t *q, const float *sc,int I, int O, int device);int qt_dnproj_matmul(int layer, float *y, const float *x, int I, int O);+int qt_dnproj_ready(int layer);+int qt_dnproj_matmul_batch(int layer, float *y, const float *x, int S, int I, int O);/* Generic resident dense matrix (int8 per-row, one GEMV per call), addressed* by a handle: the Qwen3.8 trunk uses this for every matrix it places. Offer* the size with qt_trunk_offer(name, layer, bytes) before qt_init, ask@@ -145,6 +147,8 @@ static inline int qt_place_is_auto(void){return 0;}static inline void qt_trunk_withdraw(const char*a){(void)a;}static inline int qt_dnproj_init(int a,const int8_t*b,const float*c,int d,int e,int f){(void)a;(void)b;(void)c;(void)d;(void)e;(void)f;return 0;}static inline int qt_dnproj_matmul(int a,float*b,const float*c,int d,int e){(void)a;(void)b;(void)c;(void)d;(void)e;return 0;}+static inline int qt_dnproj_ready(int a){(void)a;return 0;}+static inline int qt_dnproj_matmul_batch(int a,float*b,const float*c,int d,int e,int f){(void)a;(void)b;(void)c;(void)d;(void)e;(void)f;return 0;}static inline int qt_dense_init(const int8_t*a,const float*b,int c,int d,int e){(void)a;(void)b;(void)c;(void)d;(void)e;return -1;}static inline int qt_dense_matmul(int a,float*b,const float*c,int d,int e){(void)a;(void)b;(void)c;(void)d;(void)e;return 0;}static inline int qt_dense_count(void){return 0;}diff --git a/c/tests/test_qwen36_dnproj_batch.c b/c/tests/test_qwen36_dnproj_batch.cnew file mode 100644index 00000000..1497316e--- /dev/null+++ b/c/tests/test_qwen36_dnproj_batch.c@@ -0,0 +1,108 @@+/* Run DeltaNet itself: batching must preserve outputs, conv history and state. */+#define main qwen_main_unused+#include "../qwen36.c"+#undef main++/* Define COLI_DNPROJ_REAL_CUDA and link --wrap=coli_cuda_matmul to run+ * the same state-parity test against the real CUDA backend on Linux. */+#ifdef COLI_DNPROJ_REAL_CUDA+#include "../backend_cuda.h"+int __real_coli_cuda_matmul(ColiCudaTensor **,float *,const float *,const void *,+ const float *,int,int,int,int,int,int);+#define dn_fake_matmul __real_coli_cuda_matmul+#define DN_MATMUL __wrap_coli_cuda_matmul+#else+/* Wrap the shared fake so this test can inject a failure at a block boundary. */+#define coli_cuda_matmul dn_fake_matmul+#include "qwen36_fake_cuda.h"+#undef coli_cuda_matmul+#define DN_MATMUL coli_cuda_matmul+#endif+static int calls, fail_call, max_rows, last_rows;+int DN_MATMUL(ColiCudaTensor **t,float *y,const float *x,const void *w,+ const float *sc,int fmt,int S,int I,int O,int dev,int gs) {+ calls++; if (S > max_rows) max_rows = S; last_rows = S;+ if (calls == fail_call) { for(int i=0;i<S*O;i++) y[i] = NAN; return 0; }+ return dn_fake_matmul(t,y,x,w,sc,fmt,S,I,O,dev,gs);+}+#include "../qwen36_tier.c"++enum { H=8, KH=1, VH=2, KD=2, VD=3, C=10, Z=6, O=C+Z, CK=3 };+static int failed;+#define CHECK(c) do { if(!(c)){fprintf(stderr,"FAIL line %d: %s\n",__LINE__,#c);failed++;} } while(0)+static float *weights(int n) {+ float *p = calloc(n,sizeof(float));+ for(int i=0;i<n;i++) p[i] = (i%11 - 5)*0.03125f;+ return p;+}+static void state(Model *m) {+ m->DN_rec = calloc(1,sizeof(float *)); m->DN_conv = calloc(1,sizeof(float *));+ m->DN_rec[0] = calloc(VH*KD*VD,sizeof(float)); m->DN_conv[0] = calloc(C*(CK-1),sizeof(float));+}+static void clear_state(Model *m) {+ memset(m->DN_rec[0],0,VH*KD*VD*sizeof(float));+ memset(m->DN_conv[0],0,C*(CK-1)*sizeof(float));+}+static void agree(const float *a,const float *b,int n) {+ for(int i=0;i<n;i++) CHECK(isfinite(a[i]) && isfinite(b[i]) && fabsf(a[i]-b[i]) < 1e-5f);+}+int main(void) {+#ifdef _OPENMP+ omp_set_num_threads(1);+#endif+ setenv("COLI_DENSE_IDOT","0",1);+ Model m={0}; Layer l={0};+ m.c.hidden=H; m.c.dn_kheads=KH; m.c.dn_vheads=VH;+ m.c.dn_kdim=KD; m.c.dn_vdim=VD; m.c.dn_convk=CK; m.c.dn_conv_dim=C; m.c.eps=1e-6f;+ l.dn_qkv=weights(C*H); l.dn_z=weights(Z*H); l.dn_out=weights(H*Z);+ l.dn_b=weights(VH*H); l.dn_a=weights(VH*H); l.dn_conv=weights(C*CK);+ l.dn_alog=weights(VH); l.dn_dtbias=weights(VH); l.dn_norm=weights(VD);+ qdw_register(l.dn_qkv,H,C); qdw_register(l.dn_z,H,Z);+ int8_t q[O*H]; float sc[O];+ memcpy(q,g_qdw[0].q,C*H); memcpy(q+C*H,g_qdw[1].q,Z*H);+ memcpy(sc,g_qdw[0].sc,C*sizeof(float)); memcpy(sc+C,g_qdw[1].sc,Z*sizeof(float));+ state(&m);+ enum { S=259 };+ float *x=weights(S*H), cpu[S*H], gpu[S*H], rec[VH*KD*VD], conv[C*(CK-1)];+ deltanet(&m,&l,0,x,S,0,cpu);+ memcpy(rec,m.DN_rec[0],sizeof(rec)); memcpy(conv,m.DN_conv[0],sizeof(conv));+#ifdef COLI_DNPROJ_REAL_CUDA+ int device=0;+ if(!coli_cuda_init(&device,1)) return 1;+#else+ fake_dense_compute=1;+#endif+ for(int failure=0;failure<=2;failure++) {+ clear_state(&m); calls=0; max_rows=0; fail_call=failure;+ CHECK(qt_dnproj_init(0,q,sc,H,O,0));+ deltanet(&m,&l,0,x,S,0,gpu);+ CHECK(calls == (failure==1 ? 1 : 2));+ CHECK(max_rows==256);+ if(failure!=1) CHECK(last_rows==3);+ agree(cpu,gpu,S*H); agree(rec,m.DN_rec[0],VH*KD*VD); agree(conv,m.DN_conv[0],C*(CK-1));+ CHECK(qt_dnproj_ready(0) == !failure);+ coli_cuda_tensor_free(G_dnp[0].t); memset(&G_dnp[0],0,sizeof(G_dnp[0]));+ qt_shutdown();+ }+ CHECK(dnproj_batch_rows(10000,65536,65536)==64);+ CHECK(dnproj_batch_rows(1,H,O)==1);+ /* Decode still uses a single projection and the same state transition. */+ clear_state(&m); deltanet(&m,&l,0,x,1,0,cpu);+ memcpy(rec,m.DN_rec[0],sizeof(rec)); memcpy(conv,m.DN_conv[0],sizeof(conv));+ clear_state(&m); calls=0; fail_call=0;+ CHECK(qt_dnproj_init(0,q,sc,H,O,0));+ deltanet(&m,&l,0,x,1,0,gpu);+ CHECK(calls==1 && last_rows==1);+ agree(cpu,gpu,H); agree(rec,m.DN_rec[0],VH*KD*VD); agree(conv,m.DN_conv[0],C*(CK-1));+ coli_cuda_tensor_free(G_dnp[0].t); memset(&G_dnp[0],0,sizeof(G_dnp[0]));+ qt_shutdown();+#ifdef COLI_DNPROJ_REAL_CUDA+ coli_cuda_shutdown();+#endif+ free(x); free(m.DN_rec[0]); free(m.DN_conv[0]); free(m.DN_rec); free(m.DN_conv);+ free(l.dn_qkv); free(l.dn_z); free(l.dn_out); free(l.dn_b); free(l.dn_a); free(l.dn_conv);+ free(l.dn_alog); free(l.dn_dtbias); free(l.dn_norm);+ for(int i=0;i<g_qdw_n;i++){free(g_qdw[i].q);free(g_qdw[i].sc);} g_qdw_n=0;+ printf("DeltaNet batch: %s\n",failed?"FAIL":"outputs and recurrent state match; 259 calls -> 2");+ return failed!=0;+}diff --git a/docs/qwen36-cuda-tier.md b/docs/qwen36-cuda-tier.mdindex edf04a5e..1b69011a 100644--- a/docs/qwen36-cuda-tier.md+++ b/docs/qwen36-cuda-tier.md@@ -83,6 +83,13 @@ out_proj and norm, 3.3 the convolution and 2.4 the recurrence -- the matmuls,not the recurrence, are what the trunk costs. They are served from VRAM ondecode (one GEMV each); a prompt batch keeps the batched CPU matmul.+Placed DeltaNet input projections (`dnproj`, qkv ++ z) also run as CUDA+batches during prefill: at most 256 rows per call, with input/output staging+bounded to 32 MiB (or one row if that alone is larger). The convolution and+recurrent state still advance one token at a time. An unavailable or failed+projection uses the existing per-token CPU path. This changes dispatch count,+not the recurrent update order; it is not a full GPU DeltaNet implementation.+**Measured, not assumed.** The pricing rule below presumes the GPU answers aGEMV faster than the CPU does. Four Tesla M10 (sm_50, no tensor cores, fourGPUs on one PCIe board) said otherwise in #1652: with every expert