Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

qwen38: Q38_DN_GPU=1, decode tokens run the DeltaNet layer on the card end to end

kreuzzelgd0824bf4fddd
diff --git a/c/qwen38.c b/c/qwen38.c
index 92baab5c..895449d8 100644
--- a/c/qwen38.c
+++ b/c/qwen38.c
@@ -1394,6 +1394,7 @@ static void q38_pin_state_free(void *v){
static int q38_pin_state_copy(Model *m, Q38PinState **slot, int to_state){
const Cfg *c = &m->c;
size_t rec = 0, conv = 0, ple = 0;
+ if (to_state) q38_dn_gpu_pull_all(m); /* the card may be ahead of the host copy */
#ifdef COLI_VULKAN
if (to_state) q38c_sync_host(m); /* the dense chain keeps the newest state on the device */
#endif
@@ -1446,6 +1447,7 @@ static int q38_pin_state_copy(Model *m, Q38PinState **slot, int to_state){
}
}
q38_mtp_state_copy(m, st->mtp_pend, &st->mtp_len, &st->mtp_pend_n, &st->mtp_pend_tok, to_state);
+ if (!to_state) q38_dn_gpu_invalidate(m); /* restored on the host: the card's copy is from another prompt */
#ifdef COLI_VULKAN
if (!to_state) q38c_host_wrote(m, 0); /* up to the device before the next chain step */
#endif
@@ -1548,6 +1550,7 @@ static int q38_prefix_ids_reserve(int len){
}
static void q38_prefix_copy_state(Model *m,int to_cache){
+ if(to_cache)q38_dn_gpu_pull_all(m); else q38_dn_gpu_invalidate(m);
#ifdef COLI_VULKAN
if(to_cache)q38c_sync_host(m);
else q38c_host_wrote(m,0);
diff --git a/c/qwen38_core.h b/c/qwen38_core.h
index bf1e0fc7..2d67b660 100644
--- a/c/qwen38_core.h
+++ b/c/qwen38_core.h
@@ -168,6 +168,10 @@ typedef struct {
uint64_t expert_prefetch_ranges, expert_parallel_batches;
uint64_t expert_scale_bytes;
float **DN_rec, **DN_conv;
+ /* Q38_DN_GPU=1 (qt_dn_gpu_*): the host arrays above stay canonical; per
+ * layer, dn_dev_fresh says the card holds the host's state, dn_host_stale
+ * says the card advanced past the host copy (pull before any CPU use). */
+ uint8_t *dn_dev_fresh, *dn_host_stale; int dn_dev;
float **K, **V, **IK;
int kv_len, kv_cap, max_t;
kv_prefix kvp; /* token identity of the live attention rows, not a snapshot */
@@ -992,6 +996,7 @@ static void q38_load_ple(Model *m,Layer *l) {
static void q38_alloc_state(Model *m) {
Cfg *c=&m->c;
m->DN_rec=(float**)calloc((size_t)c->layers,sizeof(float*));
+ m->dn_dev_fresh=(uint8_t*)calloc((size_t)c->layers,1); m->dn_host_stale=(uint8_t*)calloc((size_t)c->layers,1); m->dn_dev=0;
m->DN_conv=(float**)calloc((size_t)c->layers,sizeof(float*));
m->K=(float**)calloc((size_t)c->layers+1,sizeof(float*)); /* +1: the MTP head's row */
m->V=(float**)calloc((size_t)c->layers+1,sizeof(float*));
@@ -2181,6 +2186,27 @@ static int q38_bounded_prefill_rows(int requested,uint64_t fixed,
* in bounded chunks. The convolution and recurrent update remain strictly
* token-causal inside each chunk, so chunk boundaries cannot change state or
* floating-point order. */
+/* Q38_DN_GPU=1: host/device hand-over of a DeltaNet layer's state (conv ring
+ * and recurrence) when the layer runs on the card (qt_dn_gpu_*). The host
+ * arrays stay canonical; the card's copy is fresh or ahead. */
+static void q38_dn_gpu_push(Model *m,int layer) {
+ if(!m->dn_dev_fresh[layer]&&qt_dn_gpu_set_state(layer,m->DN_conv[layer],m->DN_rec[layer]))m->dn_dev_fresh[layer]=1;
+}
+static void q38_dn_gpu_pull(Model *m,int layer) {
+ if(m->dn_host_stale&&m->dn_host_stale[layer]){
+ if(qt_dn_gpu_get_state(layer,m->DN_conv[layer],m->DN_rec[layer]))m->dn_host_stale[layer]=0;
+ else fprintf(stderr,"[dn] layer %d: could not read the GPU state back; the CPU continues from a stale copy\n",layer);
+ }
+}
+static void q38_dn_gpu_pull_all(Model *m) {
+ if(!m->dn_dev)return;
+ for(int i=0;i<m->c.layers;i++)if(!m->c.is_attn[i])q38_dn_gpu_pull(m,i);
+}
+static void q38_dn_gpu_invalidate(Model *m) { /* the host state was rewritten: the card's copy is old */
+ if(!m->dn_dev_fresh)return;
+ memset(m->dn_dev_fresh,0,(size_t)m->c.layers); memset(m->dn_host_stale,0,(size_t)m->c.layers);
+}
+
static void q38_deltanet(Model *m,Layer *l,int layer,const float *x,int S,
float *out) {
double phase_started=now_s();
@@ -2208,6 +2234,29 @@ static void q38_deltanet(Model *m,Layer *l,int layer,const float *x,int S,
float *core=falloc(V);
float *rec=m->DN_rec[layer],*ring=m->DN_conv[layer];
+ /* Decode token with the layer on the card: the two gates on the CPU (b and
+ * a, 48 outputs each), everything else -- in_proj qkv and z, conv,
+ * recurrence, gated norm, out_proj -- in one device chain, host in, host
+ * out. Not during an MTP verify (snap_after: the CPU path takes the
+ * snapshot after its first row). */
+ if(S==1&&m->dn_dev&&m->snap_after==0&&qt_dn_gpu_ready(layer)){
+ q38_dense_matmul(m,bb,x,&l->dn_b,1,H,VH);
+ q38_dense_matmul(m,aa,x,&l->dn_a,1,H,VH);
+ float egh[64],beta[64];
+ for(int h=0;h<VH&&h<64;h++){
+ egh[h]=expf(-expf(l->dn_alog[h])*q38_softplus(aa[h]+l->dn_dtbias[h]));
+ beta[h]=q38_sigmoid(bb[h]);
+ }
+ q38_dn_gpu_push(m,layer);
+ if(m->dn_dev_fresh[layer]&&qt_dn_gpu_step(layer,x,out,egh,beta)){
+ m->dn_host_stale[layer]=1;
+ free(qkv);free(z);free(bb);free(aa);free(norm);free(conv);free(q);free(k);free(core);
+ q38_tm_add(m,Q38_TM_DELTANET,phase_started);
+ return;
+ }
+ q38_dn_gpu_pull(m,layer); /* the tier turned the layer off: continue on the CPU from the card's state */
+ } else if(m->dn_dev) q38_dn_gpu_pull(m,layer); /* prefill, verify or a CPU-only layer: the host must be current */
+
for(int base=0;base<S;) {
int rows=S-base<rows_capacity?S-base:rows_capacity;
const float *chunk=x+(int64_t)base*H;
@@ -2286,6 +2335,7 @@ static void q38_deltanet(Model *m,Layer *l,int layer,const float *x,int S,
rows,V,H);
base+=rows;
}
+ if(m->dn_dev_fresh)m->dn_dev_fresh[layer]=0; /* the host advanced: the card's copy is old */
free(qkv);free(z);free(bb);free(aa);free(norm);free(conv);
free(q);free(k);free(core);
q38_tm_add(m,Q38_TM_DELTANET,phase_started);
@@ -2585,6 +2635,25 @@ static void q38_tier_start(Model *m,int cap) {
atexit(qt_shutdown);
fprintf(stderr,"[qtier] qwen38: fp8 expert tier on (RAM LRU %d/layer stays; experts stream to VRAM as they get hot)\n",cap);
q38_trunk_place_all(m);
+ /* Q38_DN_GPU=1: where dnqkv, dnz and dnout of a DeltaNet layer all sit
+ * on one card, the conv ring, the recurrence and the gated norm go there
+ * too, and a decode token runs the layer end to end on the device
+ * (qwen36 measured 8 of 39 ms/token in these round trips). */
+ const char *dg=getenv("Q38_DN_GPU");
+ if(dg&&dg[0]=='1'&&!dg[1]){
+ int n=0; double vram=0;
+ for(int i=0;i<c->layers;i++){
+ Layer *L=&m->L[i];
+ if(c->is_attn[i]||!L->dn_qkv.gpu||!L->dn_z.gpu||!L->dn_out.gpu)continue;
+ if(qt_dn_gpu_init_dense(i,c->dn_vheads,c->dn_kheads,c->dn_kdim,c->dn_vdim,c->dn_conv_dim,c->dn_convk,c->hidden,
+ L->dn_conv,L->dn_norm,c->eps,1,L->dn_qkv.gpu,L->dn_z.gpu,L->dn_out.gpu)){
+ n++; vram+=(double)c->dn_vheads*c->dn_kdim*c->dn_vdim*4+(double)c->dn_conv_dim*(c->dn_convk-1)*4;
+ }
+ }
+ m->dn_dev=n>0;
+ if(n)fprintf(stderr,"[dn] qwen38: %d DeltaNet layers run on the GPU end to end (conv, recurrence, gated norm; %.0f MB of state in VRAM)\n",n,vram/1048576.0);
+ else fprintf(stderr,"[dn] Q38_DN_GPU=1 but no DeltaNet layer has dnqkv, dnz and dnout on one card; the CPU path stands\n");
+ }
}
}
@@ -3006,6 +3075,7 @@ static void reset_recurrent(Model *m) {
}
memset(m->PLE_conv_state,0,(size_t)c->hc_width*(c->ple_convk-1)*c->ngram_size*sizeof(float));m->ple_history_len=0;
m->mtp_len=0;m->mtp_pend_n=0; /* the MTP head's rows go with the model's */
+ q38_dn_gpu_invalidate(m); /* zeros on the host are the truth; the card re-loads before its next step */
#ifdef COLI_VULKAN
q38c_host_wrote(m,1); /* zeros: the dense chain fills its copy with zeros */
#endif
@@ -3373,6 +3443,7 @@ static void q38_spec_rollback(Model *m,int len) {
float *t=m->DN_rec[i];m->DN_rec[i]=m->snap_rec[i];m->snap_rec[i]=t;
t=m->DN_conv[i];m->DN_conv[i]=m->snap_conv[i];m->snap_conv[i]=t;
}
+ q38_dn_gpu_invalidate(m); /* the host swapped its state in: the card's copy is from the rejected draft */
if(m->snap_ple){
float *t=m->PLE_conv_state;m->PLE_conv_state=m->snap_ple;m->snap_ple=t;
memcpy(m->ple_history,m->snap_ple_history,sizeof(m->snap_ple_history));
@@ -3615,7 +3686,7 @@ static void q38_model_free(Model *m) {
q38_weight_free(&m->mtp_fc_emb); q38_weight_free(&m->mtp_fc_hid);
free(m->mtp_mixer.norm); q38_weight_free(&m->mtp_mixer.down); q38_weight_free(&m->mtp_mixer.up);
if(m->x4){st_destroy(&m->x4->S);free(m->x4->fd);free(m->x4->off);free(m->x4);}
- free(m->L); free(m->cache); free(m->expert_scales); free(m->DN_rec); free(m->DN_conv); free(m->K); free(m->V); free(m->IK);
+ free(m->L); free(m->cache); free(m->expert_scales); free(m->DN_rec); free(m->DN_conv); free(m->dn_dev_fresh); free(m->dn_host_stale); free(m->K); free(m->V); free(m->IK);
q38_weight_free(&m->embed);q38_weight_free(&m->lm_head);
free(m->final_gr.norm);q38_weight_free(&m->final_gr.down);q38_weight_free(&m->final_gr.up);q38_weight_free(&m->final_gr.inject);
free(m->ple_history); free(m->PLE_conv_state); free(m->c.is_attn); st_destroy(&m->S);