Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

qwen36: Q36_DN_GPU=1, decode tokens run the DeltaNet layer on the GPU end to end

kreuzzelg237cfab883ce
diff --git a/c/qwen36.c b/c/qwen36.c
index ab6589ad..9e253b51 100644
--- a/c/qwen36.c
+++ b/c/qwen36.c
@@ -798,6 +798,12 @@ typedef struct {
int *active_of; /* [n_layers] original->active idx (Phase 2: identity for all layers) */
float **DN_rec; /* [n_layers] recurrent state S[h]=[kdim,vdim] for DeltaNet layers (NULL for attn) */
float **DN_conv; /* [n_layers] conv ring [conv_dim, convk-1] for DeltaNet layers (NULL for attn) */
+ /* DeltaNet layers on the GPU (Q36_DN_GPU=1, qt_dn_gpu_*): the host arrays
+ * above stay canonical; per layer, dn_dev_fresh says the device holds the
+ * newest state (nothing to upload before a GPU step), dn_host_stale says
+ * the device advanced past the host copy (download before any CPU use:
+ * a CPU step, a snapshot, a reset that must not be undone). */
+ uint8_t *dn_dev_fresh, *dn_host_stale; int dn_dev;
uint64_t clock, hits, miss;
RouteStats route; /* CACHE_ROUTE / ROUTE_AGREE meters */
/* Telemetria per la dashboard (Brain/Profile): tempo di lettura esperti
@@ -2107,6 +2113,7 @@ static void model_init_range(Model *m, const char *snap, int cap, int bits,
/* per-layer DeltaNet recurrent + conv state (only for linear_attention layers) */
m->DN_rec = calloc((size_t)c->n_layers, sizeof(float*));
m->DN_conv = calloc((size_t)c->n_layers, sizeof(float*));
+ m->dn_dev_fresh = calloc((size_t)c->n_layers, 1); m->dn_host_stale = calloc((size_t)c->n_layers, 1); m->dn_dev = 0;
for (int i = layer_begin; allocate_state && i < layer_end; i++) {
if (c->is_attn[i]) { m->DN_rec[i] = NULL; m->DN_conv[i] = NULL; continue; }
if (c->dn_vheads <= 0) { fprintf(stderr, "layer %d is DeltaNet but dn dims missing from meta\n", i); exit(1); }
@@ -3208,6 +3215,36 @@ static int dnproj_batch_rows(int S, int H, int O) {
return S < rows ? S : (int)rows;
}
+/* Host/device state hand-over for a DeltaNet layer that runs on the GPU. The
+ * host arrays are canonical: the device copy is a cache that is fresh (holds
+ * what the host holds) or ahead (host_stale: the device advanced). */
+static void dn_gpu_push(Model *m, int layer) {
+ if (!m->dn_dev_fresh[layer]) {
+ if (qt_dn_gpu_set_state(layer, m->DN_conv[layer], m->DN_rec[layer])) m->dn_dev_fresh[layer] = 1;
+ }
+}
+static void dn_gpu_pull(Model *m, int layer) {
+ if (m->dn_host_stale && m->dn_host_stale[layer]) {
+ if (qt_dn_gpu_get_state(layer, m->DN_conv[layer], m->DN_rec[layer])) m->dn_host_stale[layer] = 0;
+ else fprintf(stderr, "[dn] layer %d: could not read the GPU state back; the CPU continues from a stale copy\n", layer);
+ }
+}
+static void dn_gpu_pull_all(Model *m) {
+ if (!m->dn_dev) return;
+ for (int i = 0; i < m->c.n_layers; i++) if (!m->c.is_attn[i]) dn_gpu_pull(m, i);
+}
+/* the host state was rewritten (reset, restore): the device copy is old */
+static void dn_gpu_invalidate(Model *m) {
+ if (!m->dn_dev_fresh) return;
+ memset(m->dn_dev_fresh, 0, (size_t)m->c.n_layers);
+ memset(m->dn_host_stale, 0, (size_t)m->c.n_layers);
+}
+static int dn_gpu_env_on(void) {
+ static int on = -1;
+ if (on < 0) { const char *p = getenv("Q36_DN_GPU"); on = p && *p == '1'; }
+ return on;
+}
+
static void deltanet(Model *m, Layer *l, int layer, float *x, int S, int pos_base, float *out) {
(void)pos_base;
Cfg *c = &m->c;
@@ -3247,6 +3284,38 @@ static void deltanet(Model *m, Layer *l, int layer, float *x, int S, int pos_bas
float *ring = m->DN_conv[layer]; /* [conv_dim*(convk-1)] */
FILE *dbg = layer == 0 && getenv("DN_DBG") ? fopen(getenv("DN_DBG"), "wb") : NULL;
+ /* Decode token with the layer on the GPU: gates on the CPU (two tiny
+ * matmuls), everything else -- in_proj, conv, recurrence, gated norm,
+ * out_proj -- in one device chain, host in, host out. The state stays on
+ * the card; the host copy is refreshed only when something on the CPU
+ * asks for it (dn_gpu_pull). */
+ if (S == 1 && m->dn_dev && qt_dn_gpu_ready(layer)) {
+ extern double g_dn_sub[4];
+ double _g0 = tm_now();
+ matmul(bb, x, l->dn_b, 1, H, vh);
+ matmul(ab, x, l->dn_a, 1, H, vh);
+ for (int h = 0; h < vh; h++) {
+ beta[h] = 1.f / (1.f + expf(-bb[h]));
+ gg[h] = expf(-expf(l->dn_alog[h]) * softplus_f(ab[h] + l->dn_dtbias[h])); /* egh */
+ }
+ dn_gpu_push(m, layer);
+ int ok = m->dn_dev_fresh[layer] && qt_dn_gpu_step(layer, x, out, gg, beta);
+ if (ok) {
+ m->dn_host_stale[layer] = 1;
+ if (tm_on()) g_dn_sub[0] += tm_now() - _g0;
+ if (dbg) fclose(dbg);
+ free(qkvz); free(qkvb); free(zb); free(bb); free(ab); free(outrb);
+ free(beta); free(gg);
+ free(conv_out); free(q); free(k); free(outv); free(kv); free(delta);
+ return;
+ }
+ /* the tier turned the layer off: continue on the CPU from the state
+ * the card still holds (a failed step may or may not have advanced it) */
+ dn_gpu_pull(m, layer);
+ } else if (m->dn_dev) {
+ dn_gpu_pull(m, layer); /* prefill or a CPU-only layer: the host must be current */
+ }
+
for (int base = 0; base < S; base += B) {
int rows = S - base < B ? S - base : B;
const float *xb = x + (int64_t)base * H;
@@ -3383,6 +3452,7 @@ static void deltanet(Model *m, Layer *l, int layer, float *x, int S, int pos_bas
fclose(dbg); dbg = NULL;
}
}
+ if (m->dn_dev_fresh) m->dn_dev_fresh[layer] = 0; /* the host advanced: the device copy is old */
free(qkvz); free(qkvb); free(zb); free(bb); free(ab); free(outrb);
free(beta); free(gg);
free(conv_out); free(q); free(k); free(outv); free(kv); free(delta);
@@ -3838,6 +3908,7 @@ static Q36PinState *q36_pin_state_save(Model *m, Q36PinState *reuse){
if (!st->rec[i] || !st->conv[i]) { q36_pin_state_free(st); return NULL; }
}
}
+ dn_gpu_pull_all(m); /* the card may be ahead of the host copy */
for (int i = 0; i < c->n_layers; i++){
if (c->is_attn[i]) continue;
if (m->DN_rec[i] && st->rec[i]) memcpy(st->rec[i], m->DN_rec[i], nr * sizeof(float));
@@ -3883,6 +3954,7 @@ static int pin_restore(Model *m, const int *ids, int n){
#ifdef COLI_VULKAN
q36c_host_wrote(m, 0); /* the device's copy goes up again before the next chain step */
#endif
+ dn_gpu_invalidate(m); /* restored on the host: the card's copy is from another prompt */
m->kv_len = k->len;
kv_prefix_clear(&m->kvp);
kv_prefix_record(&m->kvp, k->ids, 0, k->len);
@@ -3910,6 +3982,7 @@ static void reset_recurrent(Model *m){
#ifdef COLI_VULKAN
q36c_host_wrote(m, 1); /* zeros: the dense chain fills its copy with zeros */
#endif
+ dn_gpu_invalidate(m); /* zero on the host is the truth now; the card re-loads it before its next step */
}
/* Allocate (once) or reuse the KV cache across requests. Grows only when a
@@ -4943,6 +5016,26 @@ int main(int argc, char **argv) {
fprintf(stderr, "[dense] %d trunk matrices on GPU (dnout/attnproj/shexp, %.2f GB VRAM)\n",
placed, vram / 1073741824.0);
}
+ /* Q36_DN_GPU=1: where the in_proj and the out_proj of a DeltaNet layer
+ * both sit on one card, the conv ring, the recurrence and the gated
+ * norm go there too, and a decode token runs the layer end to end on
+ * the device (qt_dn_gpu_step). Measured motivation: with the trunk in
+ * VRAM the CPU still spent ~8 of 39 ms/token on these steps -- not
+ * arithmetic, host round trips, thirty per token. */
+ if (dn_gpu_env_on()) {
+ int n = 0; double vram = 0;
+ for (int i = 0; i < m.c.n_layers; i++) {
+ if (m.c.is_attn[i] || !m.L[i].qth_dnout || !qt_dnproj_ready(i)) continue;
+ if (qt_dn_gpu_init(i, m.c.dn_vheads, m.c.dn_kheads, m.c.dn_kdim, m.c.dn_vdim, m.c.dn_conv_dim, m.c.dn_convk,
+ m.c.hidden, m.L[i].dn_conv, m.L[i].dn_norm, m.c.eps, m.L[i].qth_dnout)) {
+ n++; vram += (double)m.c.dn_vheads * m.c.dn_kdim * m.c.dn_vdim * 4 + (double)m.c.dn_conv_dim * (m.c.dn_convk - 1) * 4;
+ }
+ }
+ m.dn_dev = n > 0;
+ if (n) fprintf(stderr, "[dn] %d DeltaNet layers run on the GPU end to end (conv, recurrence, gated norm; %.0f MB of state in VRAM)\n",
+ n, vram / 1048576.0);
+ else fprintf(stderr, "[dn] Q36_DN_GPU=1 but no layer has both projections on one card; the CPU path stands\n");
+ }
/* Warmstart: fill the VRAM budget BEFORE the first token (heat order
* when HEAT_FILE exists, natural order otherwise), loading all RAM
* slots along the way. */
@@ -5127,7 +5220,7 @@ static void qwen36_segment_model_destroy(Qwen36SegmentEngine *engine) {
free(model->attn_sc);
free(model->seen); free(model->is_queued); free(model->is_pinned);
free(model->momentum_logits); free(model->freq);
- free(model->DN_conv); free(model->DN_rec);
+ free(model->DN_conv); free(model->DN_rec); free(model->dn_dev_fresh); free(model->dn_host_stale);
free(model->cache); free(model->active_of); free(model->L);
free(model->c.is_attn);
st_destroy(&model->S);
@@ -5189,7 +5282,7 @@ static int qwen36_segment_engine_open(
(int)options->layer_begin, (int)options->layer_end, 0, 0);
engine->model.quant_bits = container_layer_is_int4(
&engine->model, (int)options->layer_begin) ? 4 : 8;
- free(engine->model.DN_rec); free(engine->model.DN_conv);
+ free(engine->model.DN_rec); free(engine->model.DN_conv); free(engine->model.dn_dev_fresh); free(engine->model.dn_host_stale);
engine->model.DN_rec = NULL; engine->model.DN_conv = NULL;
engine->model.max_t = (int)options->context_tokens;
engine->model.kv_cap = (int)options->context_tokens;