Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
qwen38: Q38_DN_GPU=1, decode tokens run the DeltaNet layer on the card end to end
kreuzzelgd0824bf4fddd
diff --git a/c/qwen38.c b/c/qwen38.cindex 92baab5c..895449d8 100644--- a/c/qwen38.c+++ b/c/qwen38.c@@ -1394,6 +1394,7 @@ static void q38_pin_state_free(void *v){static int q38_pin_state_copy(Model *m, Q38PinState **slot, int to_state){const Cfg *c = &m->c;size_t rec = 0, conv = 0, ple = 0;+ if (to_state) q38_dn_gpu_pull_all(m); /* the card may be ahead of the host copy */#ifdef COLI_VULKANif (to_state) q38c_sync_host(m); /* the dense chain keeps the newest state on the device */#endif@@ -1446,6 +1447,7 @@ static int q38_pin_state_copy(Model *m, Q38PinState **slot, int to_state){}}q38_mtp_state_copy(m, st->mtp_pend, &st->mtp_len, &st->mtp_pend_n, &st->mtp_pend_tok, to_state);+ if (!to_state) q38_dn_gpu_invalidate(m); /* restored on the host: the card's copy is from another prompt */#ifdef COLI_VULKANif (!to_state) q38c_host_wrote(m, 0); /* up to the device before the next chain step */#endif@@ -1548,6 +1550,7 @@ static int q38_prefix_ids_reserve(int len){}static void q38_prefix_copy_state(Model *m,int to_cache){+ if(to_cache)q38_dn_gpu_pull_all(m); else q38_dn_gpu_invalidate(m);#ifdef COLI_VULKANif(to_cache)q38c_sync_host(m);else q38c_host_wrote(m,0);diff --git a/c/qwen38_core.h b/c/qwen38_core.hindex bf1e0fc7..2d67b660 100644--- a/c/qwen38_core.h+++ b/c/qwen38_core.h@@ -168,6 +168,10 @@ typedef struct {uint64_t expert_prefetch_ranges, expert_parallel_batches;uint64_t expert_scale_bytes;float **DN_rec, **DN_conv;+ /* Q38_DN_GPU=1 (qt_dn_gpu_*): the host arrays above stay canonical; per+ * layer, dn_dev_fresh says the card holds the host's state, dn_host_stale+ * says the card advanced past the host copy (pull before any CPU use). */+ uint8_t *dn_dev_fresh, *dn_host_stale; int dn_dev;float **K, **V, **IK;int kv_len, kv_cap, max_t;kv_prefix kvp; /* token identity of the live attention rows, not a snapshot */@@ -992,6 +996,7 @@ static void q38_load_ple(Model *m,Layer *l) {static void q38_alloc_state(Model *m) {Cfg *c=&m->c;m->DN_rec=(float**)calloc((size_t)c->layers,sizeof(float*));+ m->dn_dev_fresh=(uint8_t*)calloc((size_t)c->layers,1); m->dn_host_stale=(uint8_t*)calloc((size_t)c->layers,1); m->dn_dev=0;m->DN_conv=(float**)calloc((size_t)c->layers,sizeof(float*));m->K=(float**)calloc((size_t)c->layers+1,sizeof(float*)); /* +1: the MTP head's row */m->V=(float**)calloc((size_t)c->layers+1,sizeof(float*));@@ -2181,6 +2186,27 @@ static int q38_bounded_prefill_rows(int requested,uint64_t fixed,* in bounded chunks. The convolution and recurrent update remain strictly* token-causal inside each chunk, so chunk boundaries cannot change state or* floating-point order. */+/* Q38_DN_GPU=1: host/device hand-over of a DeltaNet layer's state (conv ring+ * and recurrence) when the layer runs on the card (qt_dn_gpu_*). The host+ * arrays stay canonical; the card's copy is fresh or ahead. */+static void q38_dn_gpu_push(Model *m,int layer) {+ if(!m->dn_dev_fresh[layer]&&qt_dn_gpu_set_state(layer,m->DN_conv[layer],m->DN_rec[layer]))m->dn_dev_fresh[layer]=1;+}+static void q38_dn_gpu_pull(Model *m,int layer) {+ if(m->dn_host_stale&&m->dn_host_stale[layer]){+ if(qt_dn_gpu_get_state(layer,m->DN_conv[layer],m->DN_rec[layer]))m->dn_host_stale[layer]=0;+ else fprintf(stderr,"[dn] layer %d: could not read the GPU state back; the CPU continues from a stale copy\n",layer);+ }+}+static void q38_dn_gpu_pull_all(Model *m) {+ if(!m->dn_dev)return;+ for(int i=0;i<m->c.layers;i++)if(!m->c.is_attn[i])q38_dn_gpu_pull(m,i);+}+static void q38_dn_gpu_invalidate(Model *m) { /* the host state was rewritten: the card's copy is old */+ if(!m->dn_dev_fresh)return;+ memset(m->dn_dev_fresh,0,(size_t)m->c.layers); memset(m->dn_host_stale,0,(size_t)m->c.layers);+}+static void q38_deltanet(Model *m,Layer *l,int layer,const float *x,int S,float *out) {double phase_started=now_s();@@ -2208,6 +2234,29 @@ static void q38_deltanet(Model *m,Layer *l,int layer,const float *x,int S,float *core=falloc(V);float *rec=m->DN_rec[layer],*ring=m->DN_conv[layer];+ /* Decode token with the layer on the card: the two gates on the CPU (b and+ * a, 48 outputs each), everything else -- in_proj qkv and z, conv,+ * recurrence, gated norm, out_proj -- in one device chain, host in, host+ * out. Not during an MTP verify (snap_after: the CPU path takes the+ * snapshot after its first row). */+ if(S==1&&m->dn_dev&&m->snap_after==0&&qt_dn_gpu_ready(layer)){+ q38_dense_matmul(m,bb,x,&l->dn_b,1,H,VH);+ q38_dense_matmul(m,aa,x,&l->dn_a,1,H,VH);+ float egh[64],beta[64];+ for(int h=0;h<VH&&h<64;h++){+ egh[h]=expf(-expf(l->dn_alog[h])*q38_softplus(aa[h]+l->dn_dtbias[h]));+ beta[h]=q38_sigmoid(bb[h]);+ }+ q38_dn_gpu_push(m,layer);+ if(m->dn_dev_fresh[layer]&&qt_dn_gpu_step(layer,x,out,egh,beta)){+ m->dn_host_stale[layer]=1;+ free(qkv);free(z);free(bb);free(aa);free(norm);free(conv);free(q);free(k);free(core);+ q38_tm_add(m,Q38_TM_DELTANET,phase_started);+ return;+ }+ q38_dn_gpu_pull(m,layer); /* the tier turned the layer off: continue on the CPU from the card's state */+ } else if(m->dn_dev) q38_dn_gpu_pull(m,layer); /* prefill, verify or a CPU-only layer: the host must be current */+for(int base=0;base<S;) {int rows=S-base<rows_capacity?S-base:rows_capacity;const float *chunk=x+(int64_t)base*H;@@ -2286,6 +2335,7 @@ static void q38_deltanet(Model *m,Layer *l,int layer,const float *x,int S,rows,V,H);base+=rows;}+ if(m->dn_dev_fresh)m->dn_dev_fresh[layer]=0; /* the host advanced: the card's copy is old */free(qkv);free(z);free(bb);free(aa);free(norm);free(conv);free(q);free(k);free(core);q38_tm_add(m,Q38_TM_DELTANET,phase_started);@@ -2585,6 +2635,25 @@ static void q38_tier_start(Model *m,int cap) {atexit(qt_shutdown);fprintf(stderr,"[qtier] qwen38: fp8 expert tier on (RAM LRU %d/layer stays; experts stream to VRAM as they get hot)\n",cap);q38_trunk_place_all(m);+ /* Q38_DN_GPU=1: where dnqkv, dnz and dnout of a DeltaNet layer all sit+ * on one card, the conv ring, the recurrence and the gated norm go there+ * too, and a decode token runs the layer end to end on the device+ * (qwen36 measured 8 of 39 ms/token in these round trips). */+ const char *dg=getenv("Q38_DN_GPU");+ if(dg&&dg[0]=='1'&&!dg[1]){+ int n=0; double vram=0;+ for(int i=0;i<c->layers;i++){+ Layer *L=&m->L[i];+ if(c->is_attn[i]||!L->dn_qkv.gpu||!L->dn_z.gpu||!L->dn_out.gpu)continue;+ if(qt_dn_gpu_init_dense(i,c->dn_vheads,c->dn_kheads,c->dn_kdim,c->dn_vdim,c->dn_conv_dim,c->dn_convk,c->hidden,+ L->dn_conv,L->dn_norm,c->eps,1,L->dn_qkv.gpu,L->dn_z.gpu,L->dn_out.gpu)){+ n++; vram+=(double)c->dn_vheads*c->dn_kdim*c->dn_vdim*4+(double)c->dn_conv_dim*(c->dn_convk-1)*4;+ }+ }+ m->dn_dev=n>0;+ if(n)fprintf(stderr,"[dn] qwen38: %d DeltaNet layers run on the GPU end to end (conv, recurrence, gated norm; %.0f MB of state in VRAM)\n",n,vram/1048576.0);+ else fprintf(stderr,"[dn] Q38_DN_GPU=1 but no DeltaNet layer has dnqkv, dnz and dnout on one card; the CPU path stands\n");+ }}}@@ -3006,6 +3075,7 @@ static void reset_recurrent(Model *m) {}memset(m->PLE_conv_state,0,(size_t)c->hc_width*(c->ple_convk-1)*c->ngram_size*sizeof(float));m->ple_history_len=0;m->mtp_len=0;m->mtp_pend_n=0; /* the MTP head's rows go with the model's */+ q38_dn_gpu_invalidate(m); /* zeros on the host are the truth; the card re-loads before its next step */#ifdef COLI_VULKANq38c_host_wrote(m,1); /* zeros: the dense chain fills its copy with zeros */#endif@@ -3373,6 +3443,7 @@ static void q38_spec_rollback(Model *m,int len) {float *t=m->DN_rec[i];m->DN_rec[i]=m->snap_rec[i];m->snap_rec[i]=t;t=m->DN_conv[i];m->DN_conv[i]=m->snap_conv[i];m->snap_conv[i]=t;}+ q38_dn_gpu_invalidate(m); /* the host swapped its state in: the card's copy is from the rejected draft */if(m->snap_ple){float *t=m->PLE_conv_state;m->PLE_conv_state=m->snap_ple;m->snap_ple=t;memcpy(m->ple_history,m->snap_ple_history,sizeof(m->snap_ple_history));@@ -3615,7 +3686,7 @@ static void q38_model_free(Model *m) {q38_weight_free(&m->mtp_fc_emb); q38_weight_free(&m->mtp_fc_hid);free(m->mtp_mixer.norm); q38_weight_free(&m->mtp_mixer.down); q38_weight_free(&m->mtp_mixer.up);if(m->x4){st_destroy(&m->x4->S);free(m->x4->fd);free(m->x4->off);free(m->x4);}- free(m->L); free(m->cache); free(m->expert_scales); free(m->DN_rec); free(m->DN_conv); free(m->K); free(m->V); free(m->IK);+ free(m->L); free(m->cache); free(m->expert_scales); free(m->DN_rec); free(m->DN_conv); free(m->dn_dev_fresh); free(m->dn_host_stale); free(m->K); free(m->V); free(m->IK);q38_weight_free(&m->embed);q38_weight_free(&m->lm_head);free(m->final_gr.norm);q38_weight_free(&m->final_gr.down);q38_weight_free(&m->final_gr.up);q38_weight_free(&m->final_gr.inject);free(m->ple_history); free(m->PLE_conv_state); free(m->c.is_attn); st_destroy(&m->S);