Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

qwen36: batch resident DeltaNet input projections

ZacharyZcR0fc47ccdba57
diff --git a/c/Makefile b/c/Makefile
index 341ad971..47eee120 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -2180,3 +2180,7 @@ tests/test_kv_prefix$(EXE): tests/test_kv_prefix.c kv_prefix.h
tests/test_kimi_request_state$(EXE): tests/test_kimi_request_state.c kimi_k3.c kv_prefix.h serve_codec.h st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h idot.h omp_tune.h route_trace.h
$(CC) $(NOCUDA_CFLAGS) tests/test_kimi_request_state.c $(VK_OBJ) -o tests/test_kimi_request_state$(EXE) $(NOCUDA_LDFLAGS)
+
+# DeltaNet input projection batching, state parity and block failure fallback.
+tests/test_qwen36_dnproj_batch$(EXE): tests/test_qwen36_dnproj_batch.c tests/qwen36_fake_cuda.h qwen36.c qwen36_tier.c qwen36_tier.h backend_cuda.h compat.h quant.h
+ $(CC) $(CFLAGS) -DCOLI_CUDA $< -o $@ $(LDFLAGS)
diff --git a/c/qwen36.c b/c/qwen36.c
index 8fa7c56b..17c8d419 100644
--- a/c/qwen36.c
+++ b/c/qwen36.c
@@ -2422,6 +2422,14 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
* split conv_out -> q_in/k_in/v_in; repeat_interleave q,k by rep; l2norm
* (q scaled by 1/sqrt(kdim)); recurrence S[h]*=exp(g); kv=k@S; delta=(v-kv)*beta;
* S+=k (x) delta; out=q@S; per-head Gated RMSNorm (plain weight) -> out_proj. */
+/* Bound both the host result block and device input/output staging. */
+static int dnproj_batch_rows(int S, int H, int O) {
+ int64_t rows = (32LL << 20) / (((int64_t)H + O) * sizeof(float));
+ if (rows < 1) rows = 1;
+ if (rows > 256) rows = 256;
+ return S < rows ? S : (int)rows;
+}
+
static void deltanet(Model *m, Layer *l, int layer, float *x, int S, int pos_base, float *out) {
(void)pos_base;
Cfg *c = &m->c;
@@ -2433,12 +2441,12 @@ static void deltanet(Model *m, Layer *l, int layer, float *x, int S, int pos_bas
float scale = 1.f / sqrtf((float)kdim);
int H = c->hidden;
- /* qkv and z live in ONE buffer: the fused GPU projection writes
- * [conv_dim ++ value_dim] in a single GEMV, and the CPU fallback fills the
- * same two regions. Either way the code below reads qkv/z unchanged. */
- float *qkvz = falloc((int64_t)conv_dim + value_dim);
- float *qkv = qkvz;
- float *z = qkvz + conv_dim;
+ /* Input projections have no recurrent dependency. Keep qkv ++ z for a
+ * bounded block, then consume rows in order through conv and recurrence. */
+ int proj_dim = conv_dim + value_dim;
+ int B = qt_dnproj_ready(layer) ? dnproj_batch_rows(S, H, proj_dim) : 1;
+ float *qkvz = falloc((int64_t)B * proj_dim);
+ int gpu_block = 0;
float *b = falloc(vh);
float *a = falloc(vh);
float *beta= falloc(vh);
@@ -2458,9 +2466,13 @@ static void deltanet(Model *m, Layer *l, int layer, float *x, int S, int pos_bas
const float *xs = x + (int64_t)s * H;
extern double g_dn_sub[4];
double _d0 = tm_now();
- /* projections (single-token matmuls). One fused GEMV when this layer's
- * dnproj is placed on a GPU, the two CPU matmuls otherwise. */
- if (!qt_dnproj_matmul(layer, qkvz, xs, H, conv_dim + value_dim)) {
+ if (s % B == 0) {
+ int rows = S - s < B ? S - s : B;
+ gpu_block = qt_dnproj_matmul_batch(layer, qkvz, xs, rows, H, proj_dim);
+ }
+ float *qkv = qkvz + (int64_t)(s % B) * proj_dim;
+ float *z = qkv + conv_dim;
+ if (!gpu_block) {
matmul_d(qkv, xs, l->dn_qkv, 1, H, conv_dim);
matmul_d(z, xs, l->dn_z, 1, H, value_dim);
}
diff --git a/c/qwen36_tier.c b/c/qwen36_tier.c
index 2b2e24e5..98cd4dca 100644
--- a/c/qwen36_tier.c
+++ b/c/qwen36_tier.c
@@ -810,15 +810,22 @@ int qt_dense_matmul(int h, float *y, const float *x, int I, int O){
}
int qt_dense_count(void){ return G_dense_n; }
-int qt_dnproj_matmul(int layer, float *y, const float *x, int I, int O){
- if(layer < 0 || layer >= QT_DN_MAX_LAYERS || !G_dnp[layer].on) return 0;
- if(coli_cuda_matmul(&G_dnp[layer].t,y,x,NULL,NULL,1,1,I,O,G_dnp[layer].dev,0))
+int qt_dnproj_ready(int layer){
+ return layer >= 0 && layer < QT_DN_MAX_LAYERS && G_dnp[layer].on;
+}
+int qt_dnproj_matmul_batch(int layer, float *y, const float *x, int S, int I, int O){
+ if(!qt_dnproj_ready(layer) || S <= 0) return 0;
+ if(coli_cuda_matmul(&G_dnp[layer].t,y,x,NULL,NULL,1,S,I,O,G_dnp[layer].dev,0))
return 1;
fprintf(stderr,"[dnp] layer %d GPU matmul failed; CPU from here on\n", layer);
G_dnp[layer].on = 0;
return 0;
}
+int qt_dnproj_matmul(int layer, float *y, const float *x, int I, int O){
+ return qt_dnproj_matmul_batch(layer, y, x, 1, I, O);
+}
+
int qt_lmhead_matmul(float *y, const float *x, int I, int O){
if(!G_lmh.on) return 0;
/* cached-tensor path: upload params are ignored once *t exists */
diff --git a/c/qwen36_tier.h b/c/qwen36_tier.h
index 228288d6..1098a0c3 100644
--- a/c/qwen36_tier.h
+++ b/c/qwen36_tier.h
@@ -78,6 +78,8 @@ void qt_trunk_withdraw(const char *why);
int qt_dnproj_init(int layer, const int8_t *q, const float *sc,
int I, int O, int device);
int qt_dnproj_matmul(int layer, float *y, const float *x, int I, int O);
+int qt_dnproj_ready(int layer);
+int qt_dnproj_matmul_batch(int layer, float *y, const float *x, int S, int I, int O);
/* Generic resident dense matrix (int8 per-row, one GEMV per call), addressed
* by a handle: the Qwen3.8 trunk uses this for every matrix it places. Offer
* the size with qt_trunk_offer(name, layer, bytes) before qt_init, ask
@@ -145,6 +147,8 @@ static inline int qt_place_is_auto(void){return 0;}
static inline void qt_trunk_withdraw(const char*a){(void)a;}
static inline int qt_dnproj_init(int a,const int8_t*b,const float*c,int d,int e,int f){(void)a;(void)b;(void)c;(void)d;(void)e;(void)f;return 0;}
static inline int qt_dnproj_matmul(int a,float*b,const float*c,int d,int e){(void)a;(void)b;(void)c;(void)d;(void)e;return 0;}
+static inline int qt_dnproj_ready(int a){(void)a;return 0;}
+static inline int qt_dnproj_matmul_batch(int a,float*b,const float*c,int d,int e,int f){(void)a;(void)b;(void)c;(void)d;(void)e;(void)f;return 0;}
static inline int qt_dense_init(const int8_t*a,const float*b,int c,int d,int e){(void)a;(void)b;(void)c;(void)d;(void)e;return -1;}
static inline int qt_dense_matmul(int a,float*b,const float*c,int d,int e){(void)a;(void)b;(void)c;(void)d;(void)e;return 0;}
static inline int qt_dense_count(void){return 0;}
diff --git a/c/tests/test_qwen36_dnproj_batch.c b/c/tests/test_qwen36_dnproj_batch.c
new file mode 100644
index 00000000..1497316e
--- /dev/null
+++ b/c/tests/test_qwen36_dnproj_batch.c
@@ -0,0 +1,108 @@
+/* Run DeltaNet itself: batching must preserve outputs, conv history and state. */
+#define main qwen_main_unused
+#include "../qwen36.c"
+#undef main
+
+/* Define COLI_DNPROJ_REAL_CUDA and link --wrap=coli_cuda_matmul to run
+ * the same state-parity test against the real CUDA backend on Linux. */
+#ifdef COLI_DNPROJ_REAL_CUDA
+#include "../backend_cuda.h"
+int __real_coli_cuda_matmul(ColiCudaTensor **,float *,const float *,const void *,
+ const float *,int,int,int,int,int,int);
+#define dn_fake_matmul __real_coli_cuda_matmul
+#define DN_MATMUL __wrap_coli_cuda_matmul
+#else
+/* Wrap the shared fake so this test can inject a failure at a block boundary. */
+#define coli_cuda_matmul dn_fake_matmul
+#include "qwen36_fake_cuda.h"
+#undef coli_cuda_matmul
+#define DN_MATMUL coli_cuda_matmul
+#endif
+static int calls, fail_call, max_rows, last_rows;
+int DN_MATMUL(ColiCudaTensor **t,float *y,const float *x,const void *w,
+ const float *sc,int fmt,int S,int I,int O,int dev,int gs) {
+ calls++; if (S > max_rows) max_rows = S; last_rows = S;
+ if (calls == fail_call) { for(int i=0;i<S*O;i++) y[i] = NAN; return 0; }
+ return dn_fake_matmul(t,y,x,w,sc,fmt,S,I,O,dev,gs);
+}
+#include "../qwen36_tier.c"
+
+enum { H=8, KH=1, VH=2, KD=2, VD=3, C=10, Z=6, O=C+Z, CK=3 };
+static int failed;
+#define CHECK(c) do { if(!(c)){fprintf(stderr,"FAIL line %d: %s\n",__LINE__,#c);failed++;} } while(0)
+static float *weights(int n) {
+ float *p = calloc(n,sizeof(float));
+ for(int i=0;i<n;i++) p[i] = (i%11 - 5)*0.03125f;
+ return p;
+}
+static void state(Model *m) {
+ m->DN_rec = calloc(1,sizeof(float *)); m->DN_conv = calloc(1,sizeof(float *));
+ m->DN_rec[0] = calloc(VH*KD*VD,sizeof(float)); m->DN_conv[0] = calloc(C*(CK-1),sizeof(float));
+}
+static void clear_state(Model *m) {
+ memset(m->DN_rec[0],0,VH*KD*VD*sizeof(float));
+ memset(m->DN_conv[0],0,C*(CK-1)*sizeof(float));
+}
+static void agree(const float *a,const float *b,int n) {
+ for(int i=0;i<n;i++) CHECK(isfinite(a[i]) && isfinite(b[i]) && fabsf(a[i]-b[i]) < 1e-5f);
+}
+int main(void) {
+#ifdef _OPENMP
+ omp_set_num_threads(1);
+#endif
+ setenv("COLI_DENSE_IDOT","0",1);
+ Model m={0}; Layer l={0};
+ m.c.hidden=H; m.c.dn_kheads=KH; m.c.dn_vheads=VH;
+ m.c.dn_kdim=KD; m.c.dn_vdim=VD; m.c.dn_convk=CK; m.c.dn_conv_dim=C; m.c.eps=1e-6f;
+ l.dn_qkv=weights(C*H); l.dn_z=weights(Z*H); l.dn_out=weights(H*Z);
+ l.dn_b=weights(VH*H); l.dn_a=weights(VH*H); l.dn_conv=weights(C*CK);
+ l.dn_alog=weights(VH); l.dn_dtbias=weights(VH); l.dn_norm=weights(VD);
+ qdw_register(l.dn_qkv,H,C); qdw_register(l.dn_z,H,Z);
+ int8_t q[O*H]; float sc[O];
+ memcpy(q,g_qdw[0].q,C*H); memcpy(q+C*H,g_qdw[1].q,Z*H);
+ memcpy(sc,g_qdw[0].sc,C*sizeof(float)); memcpy(sc+C,g_qdw[1].sc,Z*sizeof(float));
+ state(&m);
+ enum { S=259 };
+ float *x=weights(S*H), cpu[S*H], gpu[S*H], rec[VH*KD*VD], conv[C*(CK-1)];
+ deltanet(&m,&l,0,x,S,0,cpu);
+ memcpy(rec,m.DN_rec[0],sizeof(rec)); memcpy(conv,m.DN_conv[0],sizeof(conv));
+#ifdef COLI_DNPROJ_REAL_CUDA
+ int device=0;
+ if(!coli_cuda_init(&device,1)) return 1;
+#else
+ fake_dense_compute=1;
+#endif
+ for(int failure=0;failure<=2;failure++) {
+ clear_state(&m); calls=0; max_rows=0; fail_call=failure;
+ CHECK(qt_dnproj_init(0,q,sc,H,O,0));
+ deltanet(&m,&l,0,x,S,0,gpu);
+ CHECK(calls == (failure==1 ? 1 : 2));
+ CHECK(max_rows==256);
+ if(failure!=1) CHECK(last_rows==3);
+ agree(cpu,gpu,S*H); agree(rec,m.DN_rec[0],VH*KD*VD); agree(conv,m.DN_conv[0],C*(CK-1));
+ CHECK(qt_dnproj_ready(0) == !failure);
+ coli_cuda_tensor_free(G_dnp[0].t); memset(&G_dnp[0],0,sizeof(G_dnp[0]));
+ qt_shutdown();
+ }
+ CHECK(dnproj_batch_rows(10000,65536,65536)==64);
+ CHECK(dnproj_batch_rows(1,H,O)==1);
+ /* Decode still uses a single projection and the same state transition. */
+ clear_state(&m); deltanet(&m,&l,0,x,1,0,cpu);
+ memcpy(rec,m.DN_rec[0],sizeof(rec)); memcpy(conv,m.DN_conv[0],sizeof(conv));
+ clear_state(&m); calls=0; fail_call=0;
+ CHECK(qt_dnproj_init(0,q,sc,H,O,0));
+ deltanet(&m,&l,0,x,1,0,gpu);
+ CHECK(calls==1 && last_rows==1);
+ agree(cpu,gpu,H); agree(rec,m.DN_rec[0],VH*KD*VD); agree(conv,m.DN_conv[0],C*(CK-1));
+ coli_cuda_tensor_free(G_dnp[0].t); memset(&G_dnp[0],0,sizeof(G_dnp[0]));
+ qt_shutdown();
+#ifdef COLI_DNPROJ_REAL_CUDA
+ coli_cuda_shutdown();
+#endif
+ free(x); free(m.DN_rec[0]); free(m.DN_conv[0]); free(m.DN_rec); free(m.DN_conv);
+ free(l.dn_qkv); free(l.dn_z); free(l.dn_out); free(l.dn_b); free(l.dn_a); free(l.dn_conv);
+ free(l.dn_alog); free(l.dn_dtbias); free(l.dn_norm);
+ for(int i=0;i<g_qdw_n;i++){free(g_qdw[i].q);free(g_qdw[i].sc);} g_qdw_n=0;
+ printf("DeltaNet batch: %s\n",failed?"FAIL":"outputs and recurrent state match; 259 calls -> 2");
+ return failed!=0;
+}
diff --git a/docs/qwen36-cuda-tier.md b/docs/qwen36-cuda-tier.md
index edf04a5e..1b69011a 100644
--- a/docs/qwen36-cuda-tier.md
+++ b/docs/qwen36-cuda-tier.md
@@ -83,6 +83,13 @@ out_proj and norm, 3.3 the convolution and 2.4 the recurrence -- the matmuls,
not the recurrence, are what the trunk costs. They are served from VRAM on
decode (one GEMV each); a prompt batch keeps the batched CPU matmul.
+Placed DeltaNet input projections (`dnproj`, qkv ++ z) also run as CUDA
+batches during prefill: at most 256 rows per call, with input/output staging
+bounded to 32 MiB (or one row if that alone is larger). The convolution and
+recurrent state still advance one token at a time. An unavailable or failed
+projection uses the existing per-token CPU path. This changes dispatch count,
+not the recurrent update order; it is not a full GPU DeltaNet implementation.
+
**Measured, not assumed.** The pricing rule below presumes the GPU answers a
GEMV faster than the CPU does. Four Tesla M10 (sm_50, no tensor cores, four
GPUs on one PCIe board) said otherwise in #1652: with every expert