Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

qwen38: speculative decoding with the checkpoint's MTP head (Q38_MTP=1)

JustVuggedb9bbe144fe
diff --git a/c/qwen38.c b/c/qwen38.c
index e6343e9b..59012543 100644
--- a/c/qwen38.c
+++ b/c/qwen38.c
@@ -6,7 +6,8 @@
* (PLE), and every layer has a 512-way sparse MoE plus a shared expert.
*
* Resident tensors are loaded from the official multimodal checkpoint's text
- * namespace. Vision and MTP tensors are indexed but never read. Official
+ * namespace. The MTP head's tensors are read only under Q38_MTP=1, which
+ * turns on speculative decoding with it (qwen38_core.h). Official
* block-FP8 experts remain native in a bounded per-layer LRU; the 51B-parameter
* PLE table remains on disk and only its sixteen 160-byte rows/token are read.
*
@@ -925,13 +926,14 @@ static void emit_openai_result(const int *out, int np, int n_new, int stream){
#include "edge_adapter_internal.h"
#endif
-/* Qwen3.8 routes every transformer layer and has no MTP row. Keep the
+/* Qwen3.8 routes every transformer layer; the MTP row (index c.layers) is
+ * kept only when the head is attached and routes too. Keep the
* process-global route_trace owner in the CLI/serve model only; segment
* adapters can coexist in one process and intentionally leave telemetry
* detached, just like the other range-native engines. */
static void q38_telemetry_init(const char *snap, const Model *m) {
rt_init("qwen38", m->c.layers, m->c.experts);
- rt_drop_row(m->c.layers);
+ if (!m->mtp) rt_drop_row(m->c.layers);
const char *up = getenv("COLI_USAGE");
if (up && *up) snprintf(g_q38_usage, sizeof g_q38_usage, "%s", up);
else snprintf(g_q38_usage, sizeof g_q38_usage, "%s/.coli_usage", snap);
@@ -969,6 +971,38 @@ static void q38_validate_ids(const Cfg *c, const int *ids, int count,
}
}
+/* A decode run's speculation: drafting when the MTP head is attached, in the
+ * mode Q38_MTP_FORCE names (tests: reject, accept, mixed; qwen38_core.h). */
+static Q38Spec q38_spec_begin(const Model *m) {
+ Q38Spec sp; memset(&sp, 0, sizeof sp);
+ sp.on = m->mtp;
+ const char *force = getenv("Q38_MTP_FORCE");
+ if (force && *force) {
+ if (!strcmp(force, "reject")) sp.force = 'r';
+ else if (!strcmp(force, "accept")) sp.force = 'a';
+ else if (!strcmp(force, "mixed")) sp.force = 'm';
+ else { fprintf(stderr, "Q38_MTP_FORCE must be reject, accept or mixed\n"); exit(1); }
+ }
+ return sp;
+}
+
+/* Acceptance and tokens per forward, colibri.c's speculation line: a run's
+ * at the end, a serve turn's after its DONE (stderr, never the wire). Tokens
+ * are the decode tokens fed after the prompt, forwards the ones that fed them. */
+static void q38_spec_report(const Model *m, const Q38Spec *sp, const char *scope) {
+ if (!m->mtp) return;
+ fprintf(stderr, "[qwen38 MTP] %s: %.2f tokens/forward (%llu forwards per %llu tokens) | "
+ "acceptance %.1f%% (%llu/%llu drafts) | wiring %c%s%s\n", scope,
+ sp->forwards ? (double)sp->tokens / sp->forwards : 0.0,
+ (unsigned long long)sp->forwards, (unsigned long long)sp->tokens,
+ sp->drafts ? 100.0 * sp->accepted / sp->drafts : 0.0,
+ (unsigned long long)sp->accepted, (unsigned long long)sp->drafts, m->mtp_wiring,
+ sp->force ? ", Q38_MTP_FORCE=" : "",
+ sp->force == 'r' ? "reject" : sp->force == 'a' ? "accept" : sp->force == 'm' ? "mixed" : "");
+}
+
+static Q38Spec g_q38_run_spec; /* generate()'s, reported by main() */
+
static void generate(Model *m, const int *prompt, int np, int n_new, int *out) {
Cfg *c = &m->c;
/* Same ceiling serve_one() enforces. Past max_position_embeddings the RoPE
@@ -982,6 +1016,8 @@ static void generate(Model *m, const int *prompt, int np, int n_new, int *out) {
for (int i = 0; i < np; i++) out[i] = prompt[i];
float *logit = step(m, prompt, np, 0);
q38_tm_snapshot_prefill(m); /* COLI_TIMERS: decode bank starts here */
+ Q38Spec *sp = &g_q38_run_spec;
+ *sp = q38_spec_begin(m); /* Q38_MTP=1: the head drafts, every token still the argmax below */
int len = np;
for (int s = 0; s < n_new; s++) {
int best = 0; float bv = logit[0];
@@ -999,9 +1035,9 @@ static void generate(Model *m, const int *prompt, int np, int n_new, int *out) {
free(logit); out[len++] = best; break;
}
free(logit); out[len++] = best;
- int one = best;
- logit = step(m, &one, 1, len - 1);
+ logit = q38_spec_step(m, sp, best, len - 1, n_new - 1 - s);
}
+ q38_spec_end(m, sp);
}
static int tf_nll(Model *m, const int *full, int nfull, int np, double *nll_out) {
@@ -1305,6 +1341,10 @@ typedef struct {
float *ple_conv;
int64_t ple_history[2];
int ple_history_len;
+ /* the MTP head where the prompt left it: its settled rows and the
+ * streams still waiting for their next token (Model.mtp_*) */
+ float *mtp_pend;
+ int mtp_len, mtp_pend_n, mtp_pend_tok;
} Q38PrefixCache;
static Q38PrefixCache g_q38_prefix;
@@ -1319,7 +1359,26 @@ static Q38PrefixCache g_q38_prefix;
static int q38_prefix_geometry(const Model *m,size_t *rec_cells,
size_t *conv_cells,size_t *ple_cells);
static ColiPinPool g_q38_pins;
-typedef struct { float **rec, **conv, *ple; int64_t ple_hist[2]; int ple_len; int n_layers; } Q38PinState;
+typedef struct { float **rec, **conv, *ple; int64_t ple_hist[2]; int ple_len; int n_layers;
+ float *mtp_pend; int mtp_len, mtp_pend_n, mtp_pend_tok; } Q38PinState;
+
+/* The MTP head's place in the sequence travels with every snapshot of the
+ * recurrent state: its KV rows below mtp_len stay valid wherever the
+ * model's rows do (the head only writes a row together with the row
+ * after it), and the pending streams are what lets the next prompt's
+ * first token reach it. */
+static void q38_mtp_state_copy(Model *m, float *pend, int *len, int *pend_n,
+ int *pend_tok, int to_state){
+ if(!m->mtp||!pend) return;
+ size_t bytes=(size_t)2*m->c.hc_width*sizeof(float);
+ if(to_state){
+ memcpy(pend,m->mtp_pend,bytes);
+ *len=m->mtp_len; *pend_n=m->mtp_pend_n; *pend_tok=m->mtp_pend_tok;
+ } else {
+ memcpy(m->mtp_pend,pend,bytes);
+ m->mtp_len=*len; m->mtp_pend_n=*pend_n; m->mtp_pend_tok=*pend_tok;
+ }
+}
static void q38_pin_state_free(void *v){
Q38PinState *st = (Q38PinState *)v;
@@ -1328,7 +1387,7 @@ static void q38_pin_state_free(void *v){
if (st->rec) free(st->rec[i]);
if (st->conv) free(st->conv[i]);
}
- free(st->rec); free(st->conv); free(st->ple); free(st);
+ free(st->rec); free(st->conv); free(st->ple); free(st->mtp_pend); free(st);
}
/* Copia la ricorrenza DeltaNet (piu le righe PLE) fra motore e scatto. */
@@ -1356,6 +1415,10 @@ static int q38_pin_state_copy(Model *m, Q38PinState **slot, int to_state){
st->ple = (float*)malloc(ple * sizeof(float));
if (!st->ple){ q38_pin_state_free(st); return 0; }
}
+ if (m->mtp){
+ st->mtp_pend = (float*)malloc((size_t)2 * c->hc_width * sizeof(float));
+ if (!st->mtp_pend){ q38_pin_state_free(st); return 0; }
+ }
*slot = st;
}
for (int i = 0; i < c->layers; i++){
@@ -1379,6 +1442,7 @@ static int q38_pin_state_copy(Model *m, Q38PinState **slot, int to_state){
memcpy(m->ple_history, st->ple_hist, sizeof(st->ple_hist));
}
}
+ q38_mtp_state_copy(m, st->mtp_pend, &st->mtp_len, &st->mtp_pend_n, &st->mtp_pend_tok, to_state);
return 1;
}
@@ -1412,7 +1476,7 @@ static void q38_prefix_cache_dispose(Q38PrefixCache *cache){
if(!cache)return;
if(cache->dn_rec)for(int i=0;i<cache->layers;i++)free(cache->dn_rec[i]);
if(cache->dn_conv)for(int i=0;i<cache->layers;i++)free(cache->dn_conv[i]);
- free(cache->dn_rec);free(cache->dn_conv);free(cache->ple_conv);
+ free(cache->dn_rec);free(cache->dn_conv);free(cache->ple_conv);free(cache->mtp_pend);
free(cache->ids);free(cache->logits);memset(cache,0,sizeof(*cache));
}
@@ -1432,6 +1496,10 @@ static int q38_prefix_cache_layout(Model *m){
g_q38_prefix.rec_cells=rec;g_q38_prefix.conv_cells=conv;
g_q38_prefix.ple_cells=ple;
}
+ if(m->mtp&&!g_q38_prefix.mtp_pend){
+ g_q38_prefix.mtp_pend=(float*)malloc((size_t)2*m->c.hc_width*sizeof(float));
+ if(!g_q38_prefix.mtp_pend){q38_prefix_cache_dispose(&g_q38_prefix);return 0;}
+ }
if(g_q38_prefix.dn_rec&&g_q38_prefix.dn_conv&&g_q38_prefix.logits&&
(!ple||g_q38_prefix.ple_conv))return 1;
size_t logits_bytes=0;
@@ -1495,6 +1563,8 @@ static void q38_prefix_copy_state(Model *m,int to_cache){
memcpy(m->ple_history,g_q38_prefix.ple_history,sizeof(g_q38_prefix.ple_history));
}
}
+ q38_mtp_state_copy(m,g_q38_prefix.mtp_pend,&g_q38_prefix.mtp_len,&g_q38_prefix.mtp_pend_n,
+ &g_q38_prefix.mtp_pend_tok,to_cache);
}
static int q38_prefix_cache_save(Model *m,const int *ids,int len,const float *logits,
@@ -1724,6 +1794,9 @@ static int serve_one(Model *m, ServeReq *q){
fprintf(stderr,"[qwen38 prefix] request=%s reused=%d/%d\n",q->id,reuse,np);
int gen=0, limited=1, cancelled=0, stopped=0, input_eof=0;
int eos_ids[4];int n_eos=serve_eos_ids(eos_ids,4,m->c.eos_id,m->c.vocab);
+ /* Q38_MTP=1: the head drafts and a verify forward checks it; every token
+ * is still sampled from the exact logits below, greedy or not. */
+ Q38Spec spec=q38_spec_begin(m);
double first_token_at=0.0,last_token_at=0.0;
unsigned char sbuf[16]; int sbn=0;
for(int s=0;s<q->max_tok;s++){
@@ -1767,8 +1840,9 @@ static int serve_one(Model *m, ServeReq *q){
* is restored on the next request, so stepping here would only run a
* full discarded decode pass. */
if(s == q->max_tok - 1) break;
- lo = step(m, &tk, 1, np+s);
+ lo = q38_spec_step(m, &spec, tk, np+s, q->max_tok-1-s);
}
+ q38_spec_end(m, &spec);
/* I vettori dell'immagine valgono per QUESTO turno soltanto: lasciarli
* agganciati farebbe rispondere la richiesta successiva sulla foto
* precedente, e la mappa e' per posizione assoluta, quindi combacerebbe
@@ -1798,6 +1872,7 @@ static int serve_one(Model *m, ServeReq *q){
fflush(stdout);
serve_hits(m);
q38_tm_report_bank(&timers,"request");
+ {char scope[96];snprintf(scope,sizeof scope,"turn %s",q->id);q38_spec_report(m,&spec,scope);}
return input_eof?-1:0;
}
@@ -1956,6 +2031,7 @@ int main(int argc, char **argv) {
Model m; model_init(&m, snap, cap, bits);
q38_expert_int4_attach(&m, snap); /* <snap>/experts-int4g64/ when present (Q38_EXPERT_INT4) */
+ q38_mtp_attach(&m, cap); /* Q38_MTP=1: the checkpoint's MTP head drafts (qwen38_core.h) */
q38_tier_start(&m, cap); /* COLI_CUDA=1: hot experts stream to VRAM (qwen36_tier.c) */
q38_trunk_cpu_int8(&m); /* the trunk's int8 rows on the CPU, BF16 released (Q38_TRUNK_CPU_INT8=0 keeps BF16) */
q38_expert_report(&m, cap); /* expert format, bytes per expert, what the cache costs */
@@ -1966,6 +2042,11 @@ int main(int argc, char **argv) {
#endif
if(is_ref)ref_logits=read_reference_logits(ref_root,m.c.vocab);
g_capture_last_logit=ref_logits!=NULL||getenv("DUMP")!=NULL;
+ if(is_ref){g_q38_mtp_oracle=full;g_q38_mtp_oracle_n=nfull;} /* Q38_MTP_FORCE=accept|mixed */
+ if(m.mtp&&getenv("Q38_MTP_DUMP")&&*getenv("Q38_MTP_DUMP")){
+ g_q38_mtp_dump=fopen(getenv("Q38_MTP_DUMP"),"wb");
+ if(!g_q38_mtp_dump){perror(getenv("Q38_MTP_DUMP"));return 1;}
+ }
q38_telemetry_init(snap, &m);
fprintf(stderr, "resident weights loaded in %.1fs | RSS after load: %.2f GB\n", m.dense_load_s, rss_gb());
@@ -2066,6 +2147,8 @@ int main(int argc, char **argv) {
}
double tot = m.hits + m.miss;
if (g_ttft >= 0) fprintf(stderr, "TTFT: %.2f s (time to first token)\n", g_ttft);
+ q38_spec_report(&m, &g_q38_run_spec, "run");
+ if (g_q38_mtp_dump) { fclose(g_q38_mtp_dump); g_q38_mtp_dump = NULL; }
tm_report(&m);
fprintf(stderr, "\nPEAK RSS: %.2f GB\n", rss_gb());
qt_stats(); /* VRAM tier hits/misses/swaps, if on */
diff --git a/c/qwen38_core.h b/c/qwen38_core.h
index 3026f336..2fbe1b86 100644
--- a/c/qwen38_core.h
+++ b/c/qwen38_core.h
@@ -4,8 +4,9 @@
* This header is included once by qwen38.c after its tokenizer and protocol
* helpers. It intentionally consumes the official HF safetensors layout:
* the multimodal wrapper's `model.language_model` namespace and the standalone
- * text model's `model` namespace are both accepted. Vision and MTP tensors are
- * indexed by st.h but are never read.
+ * text model's `model` namespace are both accepted. Vision tensors are read
+ * by the vision tower only, and the MTP head's only under Q38_MTP=1
+ * (q38_mtp_attach).
*/
#ifndef COLI_QWEN38_CORE_H
#define COLI_QWEN38_CORE_H
@@ -35,6 +36,11 @@ typedef struct {
int ple_layer, ple_dim, ple_convk, ngram_size, heads_per_ngram;
int ngram_heads, ngram_head_dim, ngram_parts;
uint8_t *is_attn;
+ /* MTP head: mtp_num_hidden_layers (0 = the checkpoint has none, -1 = one
+ * this engine does not run) and the RoPE base of its attention (config
+ * mtp.rope_theta, else the model's) */
+ int mtp_layers;
+ float mtp_theta;
/* vision: 0 = checkpoint di solo testo, o torre non caricata */
int image_token;
int vis_depth, vis_hidden, vis_heads, vis_inter, vis_patch;
@@ -190,10 +196,41 @@ typedef struct {
float *vis_rows;
int *vis_map, vis_map_len, vis_rows_n;
Q38Timers timers;
+ /* The MTP head (q38_mtp_attach, Q38_MTP=1). Its decoder layer is layer
+ * index c.layers wherever an index names storage: L[], cache[],
+ * expert_scales[], K/V/IK[] and the route-trace row. mtp_len counts the
+ * head's KV rows that hold a verified pair (the streams at position p
+ * with the token at p+1); mtp_pend keeps the streams of the rows after
+ * them whose next token has not reached the head yet (one, or two after
+ * an accepted draft, the first one's next token in mtp_pend_tok). */
+ int mtp, mtp_wiring;
+ char mtp_prefix[32];
+ float *mtp_norm_emb, *mtp_norm_hid;
+ Q38Weight mtp_fc_emb, mtp_fc_hid;
+ GatedResidual mtp_mixer;
+ int mtp_len, mtp_pend_n, mtp_pend_tok;
+ float *mtp_pend;
+ /* A verify forward (S=2) copies the DeltaNet and PLE state as it stands
+ * once its first snap_after rows are in (0 = no copy), so a rejected
+ * draft rolls back by swapping these in instead of running the forward
+ * again. */
+ int snap_after;
+ float **snap_rec, **snap_conv, *snap_ple;
+ int64_t snap_ple_history[2];
+ int snap_ple_history_len;
} Model;
+/* Layer rows of storage: the c.layers decoder layers, plus the MTP head's at
+ * index c.layers once it is attached. model_init_range sizes L[], cache[]
+ * and expert_scales[] for that extra row up front; a loop over the rows
+ * stops here so a model without the head never touches it. */
+static inline int q38_layer_rows(const Model *m){ return m->c.layers+(m->mtp?1:0); }
+
static float *g_last_logit;
static int g_capture_last_logit;
+/* Set for the length of an MTP verify forward (q38_spec_step): its rows must
+ * get the bits a decode step gives them, see q38_weight_matmul and q38_moe. */
+static int g_q38_rowwise;
static double now_s(void) {
struct timespec t;
@@ -454,6 +491,20 @@ static void q38_vk_report(void) {
#endif
static void q38_weight_matmul(float *y,const float *x,const Q38Weight *weight,
int S,int I,int O) {
+ /* Every CPU kernel below computes a row the same way whatever S is, so an
+ * MTP verify forward (S=2) reproduces two decode steps bit for bit. A
+ * device answers S == 1 and S > 1 by different routes (the VRAM trunk
+ * only takes decode rows; a Vulkan batch is its own dispatch), so under
+ * a verify such a matrix runs its rows one at a time, as decode does. */
+ if(S>1&&g_q38_rowwise&&weight&&
+ (weight->gpu
+#ifdef COLI_VULKAN
+ ||(g_vk_ready&&q38_vk_eligible(weight))
+#endif
+ )){
+ for(int s=0;s<S;s++)q38_weight_matmul(y+(int64_t)s*O,x+(int64_t)s*I,weight,1,I,O);
+ return;
+ }
/* A matrix the tier placed in VRAM (q38_trunk_place) answers a decode
* GEMV from there; prefill rows and any failure take the CPU path below,
* so the BF16 copy stays the reference for everything but S == 1. */
@@ -708,6 +759,30 @@ static void q38_load_cfg(Cfg *c,const char *snap) {
else if(!strcmp(s,"full_attention")||!strcmp(s,"qwen_sparse_attention")) c->is_attn[i]=1;
else {fprintf(stderr,"unsupported layer type %s\n",s);exit(1);}
}
+ /* The MTP head is described next to the text model (or at the root of a
+ * multimodal config). Only read here; q38_mtp_attach decides whether the
+ * engine can run it, so a config this engine cannot draft with still
+ * loads for plain decoding. */
+ {
+ jval *at=json_get(tc,"mtp_num_hidden_layers")?tc:root;
+ c->mtp_layers=q38_num_int(at,"mtp_num_hidden_layers",0,0,0,64);
+ c->mtp_theta=c->theta;
+ jval *mc=q38_obj(tc,"mtp"); if(!mc) mc=q38_obj(root,"mtp");
+ if(mc){
+ double mt=q38_num(mc,"rope_theta",theta,0);
+ if(!isfinite(mt)||mt<=0.0||mt>FLT_MAX){fprintf(stderr,"invalid mtp.rope_theta\n");exit(1);}
+ c->mtp_theta=(float)mt;
+ jval *types=json_get(mc,"layer_types");
+ if(types&&types->t==J_ARR)
+ for(int i=0;i<types->len;i++){
+ const char *s=types->kids[i]->t==J_STR?types->kids[i]->str:"";
+ /* the head's one layer is an attention layer; anything else is a head this engine does not have */
+ if(strcmp(s,"full_attention")&&strcmp(s,"qwen_sparse_attention")) c->mtp_layers=-1;
+ }
+ }
+ jval *dedicated=json_get(tc,"mtp_use_dedicated_embeddings"); if(!dedicated) dedicated=json_get(root,"mtp_use_dedicated_embeddings");
+ if(dedicated&&dedicated->t==J_BOOL&&dedicated->boolean) c->mtp_layers=-1; /* mtp.embed_tokens: not this head */
+ }
/* Vision: opzionale. Un checkpoint di solo testo non ha vision_config, e in
* quel caso la torre resta spenta invece di rifiutare il modello. La
* geometria viene dal file, non da costanti qui: una torre di misura diversa
@@ -827,17 +902,25 @@ static Q38Weight q38_load_weight(Model *m,const char *name,int rows,int cols) {
return weight;
}
+/* A layer's tensor name. Index c.layers is the MTP head's decoder layer,
+ * which the checkpoint keeps as <mtp>.layers.0. */
static void q38_name(Model *m,char *out,size_t cap,int layer,const char *suffix) {
- snprintf(out,cap,"%s.layers.%d.%s",m->prefix,layer,suffix);
+ if(layer==m->c.layers&&m->mtp_prefix[0]) snprintf(out,cap,"%s.layers.0.%s",m->mtp_prefix,suffix);
+ else snprintf(out,cap,"%s.layers.%d.%s",m->prefix,layer,suffix);
+}
+
+static void q38_load_gr_at(Model *m,GatedResidual *g,const char *base,int inject) {
+ Cfg *c=&m->c; char nm[384];
+ snprintf(nm,sizeof nm,"%s.hc_norm.weight",base); g->norm=q38_load_tensor(m,nm,c->hc_width);
+ snprintf(nm,sizeof nm,"%s.input_mix_weight_down.weight",base); g->down=q38_load_weight(m,nm,c->hc_rank,c->hc_width);
+ snprintf(nm,sizeof nm,"%s.input_mix_weight_up.weight",base); g->up=q38_load_weight(m,nm,c->hc_width,c->hc_rank);
+ if(inject){snprintf(nm,sizeof nm,"%s.block_inject_weight.weight",base);g->inject=q38_load_weight(m,nm,c->hc_count,c->hc_width);}
}
static void q38_load_gr(Model *m,GatedResidual *g,int layer,const char *kind,int inject) {
- Cfg *c=&m->c; char nm[320],base[180];
- if(layer>=0) snprintf(base,sizeof base,"layers.%d.%s",layer,kind); else snprintf(base,sizeof base,"hyper_connection_mixer");
- snprintf(nm,sizeof nm,"%s.%s.hc_norm.weight",m->prefix,base); g->norm=q38_load_tensor(m,nm,c->hc_width);
- snprintf(nm,sizeof nm,"%s.%s.input_mix_weight_down.weight",m->prefix,base); g->down=q38_load_weight(m,nm,c->hc_rank,c->hc_width);
- snprintf(nm,sizeof nm,"%s.%s.input_mix_weight_up.weight",m->prefix,base); g->up=q38_load_weight(m,nm,c->hc_width,c->hc_rank);
- if(inject){snprintf(nm,sizeof nm,"%s.%s.block_inject_weight.weight",m->prefix,base);g->inject=q38_load_weight(m,nm,c->hc_count,c->hc_width);}
+ char base[320];
+ if(layer>=0) q38_name(m,base,sizeof base,layer,kind); else snprintf(base,sizeof base,"%s.hyper_connection_mixer",m->prefix);
+ q38_load_gr_at(m,g,base,inject);
}
static void q38_load_ple(Model *m,Layer *l) {
@@ -894,9 +977,9 @@ static void q38_alloc_state(Model *m) {
Cfg *c=&m->c;
m->DN_rec=(float**)calloc((size_t)c->layers,sizeof(float*));
m->DN_conv=(float**)calloc((size_t)c->layers,sizeof(float*));
- m->K=(float**)calloc((size_t)c->layers,sizeof(float*));
- m->V=(float**)calloc((size_t)c->layers,sizeof(float*));
- m->IK=(float**)calloc((size_t)c->layers,sizeof(float*));
+ m->K=(float**)calloc((size_t)c->layers+1,sizeof(float*)); /* +1: the MTP head's row */
+ m->V=(float**)calloc((size_t)c->layers+1,sizeof(float*));
+ m->IK=(float**)calloc((size_t)c->layers+1,sizeof(float*));
if(!m->DN_rec||!m->DN_conv||!m->K||!m->V||!m->IK){fprintf(stderr,"OOM model state metadata\n");exit(1);}
for(int i=m->range_begin;i<m->range_end;i++) if(!c->is_attn[i]) {
m->DN_rec[i]=(float*)calloc((size_t)c->dn_vheads*c->dn_kdim*c->dn_vdim,sizeof(float));
@@ -1031,6 +1114,45 @@ static void q38_vision_detach(Model *m) {
m->vis_rows=NULL; m->vis_map=NULL; m->vis_map_len=0; m->vis_rows_n=0;
}
+/* One decoder layer's resident tensors and its expert cache. Index c.layers
+ * is the MTP head's layer (q38_name gives it its checkpoint names): an
+ * attention layer like the model's QSA layers, with the same tensors. */
+static void q38_load_layer(Model *m,int i,int cap) {
+ Cfg *c=&m->c; char nm[320];
+ int attn=i<c->layers?c->is_attn[i]:1;
+ Layer *l=&m->L[i]; q38_load_gr(m,&l->attn_gr,i,"attn_hyper_connection",1); q38_load_gr(m,&l->mlp_gr,i,"mlp_hyper_connection",1);
+ #define WLD(field,suf,o,in) q38_name(m,nm,sizeof nm,i,suf); l->field=q38_load_weight(m,nm,(o),(in))
+ #define VLD(field,suf,n) q38_name(m,nm,sizeof nm,i,suf); l->field=q38_load_tensor(m,nm,(n))
+ WLD(router,"mlp.gate.weight",c->experts,c->hidden);
+ WLD(sh_g,"mlp.shared_expert.gate_proj.weight",c->shared_inter,c->hidden);
+ WLD(sh_u,"mlp.shared_expert.up_proj.weight",c->shared_inter,c->hidden);
+ WLD(sh_d,"mlp.shared_expert.down_proj.weight",c->hidden,c->shared_inter);
+ VLD(sh_gate,"mlp.shared_expert_gate.weight",c->hidden);
+ if(attn){
+ WLD(q,"self_attn.q_proj.weight",c->q_heads*c->head_dim*2,c->hidden);
+ WLD(k,"self_attn.k_proj.weight",c->kv_heads*c->head_dim,c->hidden);
+ WLD(v,"self_attn.v_proj.weight",c->kv_heads*c->head_dim,c->hidden);
+ WLD(o,"self_attn.o_proj.weight",c->hidden,c->q_heads*c->head_dim);
+ VLD(qn,"self_attn.q_norm.weight",c->head_dim);VLD(kn,"self_attn.k_norm.weight",c->head_dim);
+ WLD(idx_qk,"self_attn.indexer.index_qk_proj.weight",(c->idx_qheads+c->idx_kheads)*c->idx_dim,c->hidden);
+ VLD(idx_qn,"self_attn.indexer.q_layernorm.weight",c->idx_dim);VLD(idx_kn,"self_attn.indexer.k_layernorm.weight",c->idx_dim);
+ } else {
+ int vd=c->dn_vheads*c->dn_vdim;
+ WLD(dn_qkv,"linear_attn.in_proj_qkv.weight",c->dn_conv_dim,c->hidden);
+ WLD(dn_z,"linear_attn.in_proj_z.weight",vd,c->hidden);
+ WLD(dn_b,"linear_attn.in_proj_b.weight",c->dn_vheads,c->hidden);
+ WLD(dn_a,"linear_attn.in_proj_a.weight",c->dn_vheads,c->hidden);
+ VLD(dn_conv,"linear_attn.conv1d.weight",(int64_t)c->dn_conv_dim*c->dn_convk);
+ VLD(dn_dtbias,"linear_attn.dt_bias",c->dn_vheads);VLD(dn_alog,"linear_attn.A_log",c->dn_vheads);
+ VLD(dn_norm,"linear_attn.norm.weight",c->dn_vdim);
+ WLD(dn_out,"linear_attn.out_proj.weight",c->hidden,vd);
+ }
+ #undef WLD
+ #undef VLD
+ LCache *lc=&m->cache[i]; lc->cap=cap; lc->slots=(Slot*)calloc((size_t)cap,sizeof(Slot)); lc->by_expert=(int*)malloc((size_t)c->experts*sizeof(int));
+ if(!lc->slots||!lc->by_expert){fprintf(stderr,"OOM expert cache\n");exit(1);} for(int e=0;e<c->experts;e++)lc->by_expert[e]=-1;
+}
+
static void model_init_range(Model *m,const char *snap,int cap,int bits,
int layer_begin,int layer_end,int load_boundaries,
int allocate_state) {
@@ -1053,44 +1175,13 @@ static void model_init_range(Model *m,const char *snap,int cap,int bits,
m->lm_head=q38_load_weight(m,"lm_head.weight",c->vocab,c->hidden);
q38_load_gr(m,&m->final_gr,-1,NULL,0);
}
- m->L=(Layer*)calloc((size_t)c->layers,sizeof(Layer));
- m->cache=(LCache*)calloc((size_t)c->layers,sizeof(LCache));
- m->expert_scales=(Q38ExpertScaleCache*)calloc((size_t)c->layers,
+ /* +1 row each: the MTP head's layer, filled only by q38_mtp_attach */
+ m->L=(Layer*)calloc((size_t)c->layers+1,sizeof(Layer));
+ m->cache=(LCache*)calloc((size_t)c->layers+1,sizeof(LCache));
+ m->expert_scales=(Q38ExpertScaleCache*)calloc((size_t)c->layers+1,
sizeof(*m->expert_scales));
if(!m->L||!m->cache||!m->expert_scales){fprintf(stderr,"OOM model metadata\n");exit(1);}
- for(int i=layer_begin;i<layer_end;i++){
- Layer *l=&m->L[i]; q38_load_gr(m,&l->attn_gr,i,"attn_hyper_connection",1); q38_load_gr(m,&l->mlp_gr,i,"mlp_hyper_connection",1);
- #define WLD(field,suf,o,in) q38_name(m,nm,sizeof nm,i,suf); l->field=q38_load_weight(m,nm,(o),(in))
- #define VLD(field,suf,n) q38_name(m,nm,sizeof nm,i,suf); l->field=q38_load_tensor(m,nm,(n))
- WLD(router,"mlp.gate.weight",c->experts,c->hidden);
- WLD(sh_g,"mlp.shared_expert.gate_proj.weight",c->shared_inter,c->hidden);
- WLD(sh_u,"mlp.shared_expert.up_proj.weight",c->shared_inter,c->hidden);
- WLD(sh_d,"mlp.shared_expert.down_proj.weight",c->hidden,c->shared_inter);
- VLD(sh_gate,"mlp.shared_expert_gate.weight",c->hidden);
- if(c->is_attn[i]){
- WLD(q,"self_attn.q_proj.weight",c->q_heads*c->head_dim*2,c->hidden);
- WLD(k,"self_attn.k_proj.weight",c->kv_heads*c->head_dim,c->hidden);
- WLD(v,"self_attn.v_proj.weight",c->kv_heads*c->head_dim,c->hidden);
- WLD(o,"self_attn.o_proj.weight",c->hidden,c->q_heads*c->head_dim);
- VLD(qn,"self_attn.q_norm.weight",c->head_dim);VLD(kn,"self_attn.k_norm.weight",c->head_dim);
- WLD(idx_qk,"self_attn.indexer.index_qk_proj.weight",(c->idx_qheads+c->idx_kheads)*c->idx_dim,c->hidden);
- VLD(idx_qn,"self_attn.indexer.q_layernorm.weight",c->idx_dim);VLD(idx_kn,"self_attn.indexer.k_layernorm.weight",c->idx_dim);
- } else {
- int vd=c->dn_vheads*c->dn_vdim;
- WLD(dn_qkv,"linear_attn.in_proj_qkv.weight",c->dn_conv_dim,c->hidden);
- WLD(dn_z,"linear_attn.in_proj_z.weight",vd,c->hidden);
- WLD(dn_b,"linear_attn.in_proj_b.weight",c->dn_vheads,c->hidden);
- WLD(dn_a,"linear_attn.in_proj_a.weight",c->dn_vheads,c->hidden);
- VLD(dn_conv,"linear_attn.conv1d.weight",(int64_t)c->dn_conv_dim*c->dn_convk);
- VLD(dn_dtbias,"linear_attn.dt_bias",c->dn_vheads);VLD(dn_alog,"linear_attn.A_log",c->dn_vheads);
- VLD(dn_norm,"linear_attn.norm.weight",c->dn_vdim);
- WLD(dn_out,"linear_attn.out_proj.weight",c->hidden,vd);
- }
- #undef WLD
- #undef VLD
- LCache *lc=&m->cache[i]; lc->cap=cap; lc->slots=(Slot*)calloc((size_t)cap,sizeof(Slot)); lc->by_expert=(int*)malloc((size_t)c->experts*sizeof(int));
- if(!lc->slots||!lc->by_expert){fprintf(stderr,"OOM expert cache\n");exit(1);} for(int e=0;e<c->experts;e++)lc->by_expert[e]=-1;
- }
+ for(int i=layer_begin;i<layer_end;i++) q38_load_layer(m,i,cap);
if(c->ple_layer>=layer_begin&&c->ple_layer<layer_end) q38_load_ple(m,&m->L[c->ple_layer]);
/* La torre solo quando il motore possiede la sequenza intera: uno shard che
* ospita solo alcuni layer non ha da fare niente con le immagini, e
@@ -1187,9 +1278,9 @@ static void q38_decode_scale_tensor(float *out,const unsigned char *raw,
* per-matrix loader. */
static int q38_prepare_expert_scale_bank(Model *m,int layer) {
Cfg *c=&m->c;
- if(!m->native_fp8||layer<0||layer>=c->layers)return 0;
+ if(!m->native_fp8||layer<0||layer>=q38_layer_rows(m))return 0;
if(!m->expert_scales){
- m->expert_scales=(Q38ExpertScaleCache*)calloc((size_t)c->layers,
+ m->expert_scales=(Q38ExpertScaleCache*)calloc((size_t)c->layers+1,
sizeof(*m->expert_scales));
if(!m->expert_scales){fprintf(stderr,"OOM expert scale metadata\n");exit(1);}
}
@@ -1351,12 +1442,19 @@ static int q38_try_load_native_fp8_expert(Model *m,int layer,int expert,Slot *sl
return 1;
}
+/* Whether a layer's routed experts come from the int4-g64 sidecar. It holds
+ * the model's layers only: the MTP head's layer (index c.layers) keeps the
+ * snapshot's FP8 experts with a sidecar or without one. */
+static inline int q38_layer_int4(const Model *m,int layer) {
+ return m->x4&&layer>=0&&layer<m->c.layers;
+}
+
/* Advise the kernel of the experts a route is about to miss: the int4
* sidecar's one record each, or the native FP8 pair and down ranges. */
static void q38_prefetch_experts(Model *m,int layer,const int *experts,int count) {
if(!m->expert_prefetch||!experts||count<1)return;
LCache *cache=&m->cache[layer];
- if(m->x4){
+ if(q38_layer_int4(m,layer)){
for(int index=0;index<count;index++){
int expert=experts[index];
if(expert<0||expert>=m->c.experts||cache->by_expert[expert]>=0)continue;
@@ -1644,7 +1742,7 @@ static void q38_load_int4_record(Model *m,int layer,int expert,Slot *slot) {
static void q38_load_expert(Model *m,int layer,int eid,Slot *s) {
Cfg *c=&m->c; int H=c->hidden,I=c->inter; char nm[320],sn[340];
- if(m->x4){
+ if(q38_layer_int4(m,layer)){
double started=now_s();
q38_load_int4_record(m,layer,eid,s);
q38_tm_add(m,Q38_TM_EXPERT_READ,started);m->expert_weight_reads++;
@@ -1757,7 +1855,8 @@ static int q38_expert_get_batch(Model *m,int layer,const int *experts,int count,
if(count>cache->cap)
return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_CACHE_CAPACITY,
layer,cache->cap,count);
- if(!m->x4&&!q38_prepare_expert_scale_bank(m,layer))
+ int x4=q38_layer_int4(m,layer);
+ if(!x4&&!q38_prepare_expert_scale_bank(m,layer))
return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_SCALE_BANK,layer,0,0);
/* The demand set is no longer bounded by the decode top-k: the MoE prefill
* hands over the whole chunk union (up to the cache cap) so its loads run
@@ -1783,7 +1882,7 @@ static int q38_expert_get_batch(Model *m,int layer,const int *experts,int count,
continue;
}
st_tensor *weight[3];
- if(!m->x4&&!q38_native_fp8_expert_tensors(m,layer,expert,weight)){
+ if(!x4&&!q38_native_fp8_expert_tensors(m,layer,expert,weight)){
free(jobs);
return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_LAYOUT,
layer,expert,0);
@@ -1818,7 +1917,7 @@ static int q38_expert_get_batch(Model *m,int layer,const int *experts,int count,
if(slot->eid>=0)cache->by_expert[slot->eid]=-1;
}
slot->eid=-1;jobs[job_count].expert=expert;jobs[job_count].slot=slot;
- if(!m->x4&&!q38_native_fp8_expert_tensors(m,layer,expert,jobs[job_count].weight)){
+ if(!x4&&!q38_native_fp8_expert_tensors(m,layer,expert,jobs[job_count].weight)){
fprintf(stderr,"Qwen3.8 expert layout changed during batch reservation\n");exit(1);
}
job_count++;
@@ -1837,13 +1936,13 @@ static int q38_expert_get_batch(Model *m,int layer,const int *experts,int count,
double started=now_s();
#pragma omp parallel for schedule(static) num_threads(workers) if(job_count>1)
for(int job=0;job<job_count;job++){
- if(m->x4)q38_load_int4_record(m,layer,jobs[job].expert,jobs[job].slot);
+ if(x4)q38_load_int4_record(m,layer,jobs[job].expert,jobs[job].slot);
else q38_load_native_fp8_ranges(m,layer,jobs[job].expert,jobs[job].slot,
jobs[job].weight);
}
q38_tm_add(m,Q38_TM_EXPERT_READ,started);
- m->expert_weight_reads+=(uint64_t)job_count*(m->x4?1:2);
- if(!m->x4)m->expert_pair_reads+=(uint64_t)job_count;
+ m->expert_weight_reads+=(uint64_t)job_count*(x4?1:2);
+ if(!x4)m->expert_pair_reads+=(uint64_t)job_count;
if(job_count>1)m->expert_parallel_batches++;
for(int job=0;job<job_count;job++){
Slot *slot=jobs[job].slot;slot->eid=jobs[job].expert;
@@ -1975,6 +2074,11 @@ static void q38_ple(Model *m,const int *ids,int S,const float *hyper,float *out)
else if(m->ple_history_len==0){m->ple_history[0]=ids[s];m->ple_history_len=1;}
else if(m->ple_history_len==1){m->ple_history[1]=ids[s];m->ple_history_len=2;}
else {m->ple_history[0]=m->ple_history[1];m->ple_history[1]=ids[s];}
+ if(s+1==m->snap_after){ /* MTP verify: as q38_deltanet */
+ memcpy(m->snap_ple,ring,(size_t)c->hc_width*state_len*sizeof(float));
+ memcpy(m->snap_ple_history,m->ple_history,sizeof(m->snap_ple_history));
+ m->snap_ple_history_len=m->ple_history_len;
+ }
}
free(emb);free(keys);free(value);free(kn);free(qn);free(gated);free(norm);
q38_tm_add(m,Q38_TM_PLE,phase_started);
@@ -2135,6 +2239,12 @@ static void q38_deltanet(Model *m,Layer *l,int layer,const float *x,int S,
for(int h=0;h<VH;h++)
q38_rmsg(norm_row+(int64_t)h*VD,core+(int64_t)h*VD,
z_row+(int64_t)h*VD,l->dn_norm,VD,c->eps,1);
+ /* MTP verify: the state the token before the draft leaves, for
+ * the rollback of a rejected draft (q38_spec_rollback) */
+ if(base+s+1==m->snap_after){
+ memcpy(m->snap_rec[layer],rec,(size_t)VH*KD*VD*sizeof(float));
+ memcpy(m->snap_conv[layer],ring,(size_t)CD*(CK-1)*sizeof(float));
+ }
}
q38_dense_matmul(m,out+(int64_t)base*H,norm,&l->dn_out,
rows,V,H);
@@ -2153,6 +2263,7 @@ static int q38_block_desc(const void *aa,const void *bb){
static void q38_attention(Model *m,Layer *l,int layer,const float *x,int S,int pos_base,float *out) {
Cfg *c=&m->c;int H=c->hidden,QH=c->q_heads,KVH=c->kv_heads,D=c->head_dim;
+ float theta=layer<c->layers?c->theta:c->mtp_theta; /* the MTP head has its own RoPE base */
int IQ=c->idx_qheads,ID=c->idx_dim,R=c->idx_ratio,maxsel=c->idx_budget+R-1;
float *qp=falloc((int64_t)S*QH*2*D),*kp=falloc((int64_t)S*KVH*D),*vp=falloc((int64_t)S*KVH*D);
float *ip=falloc((int64_t)S*(IQ+c->idx_kheads)*ID);
@@ -2166,7 +2277,7 @@ static void q38_attention(Model *m,Layer *l,int layer,const float *x,int S,int p
for(int s=0;s<S;s++){
int pos=pos_base+s;
for(int h=0;h<KVH;h++){
- float *kh=kp+(int64_t)s*KVH*D+(int64_t)h*D;q38_rms0(kh,kh,l->kn,D,c->eps);q38_rope(kh,D,c->rotary_dim,pos,c->theta);
+ float *kh=kp+(int64_t)s*KVH*D+(int64_t)h*D;q38_rms0(kh,kh,l->kn,D,c->eps);q38_rope(kh,D,c->rotary_dim,pos,theta);
memcpy(m->K[layer]+((int64_t)h*m->kv_cap+pos)*D,kh,(size_t)D*sizeof(float));
memcpy(m->V[layer]+((int64_t)h*m->kv_cap+pos)*D,vp+(int64_t)s*KVH*D+(int64_t)h*D,(size_t)D*sizeof(float));
}
@@ -2194,13 +2305,13 @@ static void q38_attention(Model *m,Layer *l,int layer,const float *x,int S,int p
float *qidx=falloc((int64_t)IQ*ID),*pool=falloc(ID);
int *selected=(int*)malloc((size_t)maxsel*sizeof(int));
if(!selected){fprintf(stderr,"OOM QSA selection\n");exit(1);}
- for(int h=0;h<IQ;h++){float *qh=qidx+(int64_t)h*ID;memcpy(qh,ip+(int64_t)s*(IQ+1)*ID+(int64_t)h*ID,(size_t)ID*sizeof(float));q38_rms0(qh,qh,l->idx_qn,ID,c->eps);q38_rope(qh,ID,c->rotary_dim,pos,c->theta);}
+ for(int h=0;h<IQ;h++){float *qh=qidx+(int64_t)h*ID;memcpy(qh,ip+(int64_t)s*(IQ+1)*ID+(int64_t)h*ID,(size_t)ID*sizeof(float));q38_rms0(qh,qh,l->idx_qn,ID,c->eps);q38_rope(qh,ID,c->rotary_dim,pos,theta);}
int take=blocks<c->idx_budget/R?blocks:c->idx_budget/R,nsel=0;
Q38Block *rank=blocks?(Q38Block*)malloc((size_t)blocks*sizeof(Q38Block)):NULL;
if(blocks&&!rank){fprintf(stderr,"OOM QSA block ranking\n");exit(1);}
for(int b=0;b<blocks;b++){
memset(pool,0,(size_t)ID*sizeof(float));for(int r=0;r<R;r++){const float *raw=m->IK[layer]+(int64_t)(b*R+r)*ID;for(int d=0;d<ID;d++)pool[d]+=raw[d]/R;}
- q38_rms0(pool,pool,l->idx_kn,ID,c->eps);q38_rope(pool,ID,c->rotary_dim,b*R,c->theta);
+ q38_rms0(pool,pool,l->idx_kn,ID,c->eps);q38_rope(pool,ID,c->rotary_dim,b*R,theta);
float score=0.f;for(int h=0;h<IQ;h++){float a=0.f;for(int d=0;d<ID;d++)a+=qidx[(int64_t)h*ID+d]*pool[d];if(a>0.f)score+=a;}rank[b]=(Q38Block){score/sqrtf((float)ID),b};
}
if(blocks)qsort(rank,(size_t)blocks,sizeof(Q38Block),q38_block_desc);
@@ -2209,7 +2320,7 @@ static void q38_attention(Model *m,Layer *l,int layer,const float *x,int S,int p
index_dt+=now_s()-phase_started; phase_started=now_s();
for(int h=0;h<QH;h++){
float *qraw=qp+(int64_t)s*QH*2*D+(int64_t)h*2*D;
- float *qh=falloc(D);memcpy(qh,qraw,(size_t)D*sizeof(float));q38_rms0(qh,qh,l->qn,D,c->eps);q38_rope(qh,D,c->rotary_dim,pos,c->theta);
+ float *qh=falloc(D);memcpy(qh,qraw,(size_t)D*sizeof(float));q38_rms0(qh,qh,l->qn,D,c->eps);q38_rope(qh,D,c->rotary_dim,pos,theta);
float *score=falloc(nsel);float mx=-INFINITY;
int khidx=h/(QH/KVH);for(int j=0;j<nsel;j++){const float *kh=m->K[layer]+((int64_t)khidx*m->kv_cap+selected[j])*D;float a=0.f;for(int d=0;d<D;d++)a+=qh[d]*kh[d];a/=sqrtf((float)D);score[j]=a;if(a>mx)mx=a;}
float den=0.f;for(int j=0;j<nsel;j++){score[j]=expf(score[j]-mx);den+=score[j];}
@@ -2256,7 +2367,7 @@ static void q38_tier_note(int layer,int eid,const Slot *ex) {
* 127, the qwen36 dnproj/lmhead format) and uploaded once; the BF16 copy
* stays for prefill and as fallback. Q38_TRUNK_GPU=0 keeps the trunk on the
* CPU (parity runs against the BF16 reference). */
-typedef struct { Q38Weight *w; char name[16]; int layer; } Q38TrunkItem;
+typedef struct { Q38Weight *w; char name[16]; int layer, cpu_only; } Q38TrunkItem;
static Q38TrunkItem *g_trunk; static int g_trunk_n, g_trunk_cap, g_trunk_offer_gpu;
static long g_trunk_min_kb; static const char *g_trunk_skip; /* read once per load in q38_trunk_offer_all */
static void q38_trunk_add(Q38Weight *w,const char *name,int layer) {
@@ -2278,7 +2389,7 @@ static void q38_trunk_add(Q38Weight *w,const char *name,int layer) {
g_trunk=(Q38TrunkItem*)realloc(g_trunk,(size_t)g_trunk_cap*sizeof(*g_trunk));
if(!g_trunk){fprintf(stderr,"OOM trunk table\n");exit(1);}
}
- Q38TrunkItem *it=&g_trunk[g_trunk_n++]; it->w=w; it->layer=layer;
+ Q38TrunkItem *it=&g_trunk[g_trunk_n++]; it->w=w; it->layer=layer; it->cpu_only=0;
snprintf(it->name,sizeof it->name,"%s",name);
if(g_trunk_offer_gpu)qt_trunk_offer(it->name,layer,bytes);
}
@@ -2312,6 +2423,26 @@ static void q38_trunk_offer_all(Model *m) {
q38_trunk_add(&L->sh_g,"shg",l); q38_trunk_add(&L->sh_u,"shu",l); q38_trunk_add(&L->sh_d,"shd",l);
q38_trunk_add(&L->router,"router",l);
}
+ if(m->mtp){
+ /* The MTP head's matrices: int8 on the CPU like the rest, but never
+ * offered to the placer, which plans the model's layers only, and
+ * never placed by a COLI_PLACE spec that names their component. */
+ int offer=g_trunk_offer_gpu,first=g_trunk_n; g_trunk_offer_gpu=0;
+ int l=c->layers; Layer *L=&m->L[l];
+ q38_trunk_add(&L->q,"attnq",l); q38_trunk_add(&L->k,"attnk",l);
+ q38_trunk_add(&L->v,"attnv",l); q38_trunk_add(&L->o,"attno",l);
+ q38_trunk_add(&L->idx_qk,"qsaidx",l);
+ q38_trunk_add(&L->attn_gr.down,"hcad",l); q38_trunk_add(&L->attn_gr.up,"hcau",l);
+ q38_trunk_add(&L->attn_gr.inject,"hcai",l);
+ q38_trunk_add(&L->mlp_gr.down,"hcmd",l); q38_trunk_add(&L->mlp_gr.up,"hcmu",l);
+ q38_trunk_add(&L->mlp_gr.inject,"hcmi",l);
+ q38_trunk_add(&L->sh_g,"shg",l); q38_trunk_add(&L->sh_u,"shu",l); q38_trunk_add(&L->sh_d,"shd",l);
+ q38_trunk_add(&L->router,"router",l);
+ q38_trunk_add(&m->mtp_fc_emb,"mtpfce",l); q38_trunk_add(&m->mtp_fc_hid,"mtpfch",l);
+ q38_trunk_add(&m->mtp_mixer.down,"mtpmixd",l); q38_trunk_add(&m->mtp_mixer.up,"mtpmixu",l);
+ for(int i=first;i<g_trunk_n;i++)g_trunk[i].cpu_only=1;
+ g_trunk_offer_gpu=offer;
+ }
}
/* int8 per row, scale = max|w| / 127 (the qwen36 dnproj/lmhead format) */
static void q38_trunk_quantize(const Q38Weight *w,int8_t **qp,float **scp) {
@@ -2362,9 +2493,10 @@ static void q38_trunk_cpu_int8(Model *m) {
/* after qt_init: quantize and upload what the placer accepted */
static void q38_trunk_place_all(Model *m) {
(void)m;
- int placed=0; size_t placed_bytes=0; double t0=now_s();
+ int placed=0,offered=g_trunk_n; size_t placed_bytes=0; double t0=now_s();
for(int i=0;i<g_trunk_n;i++){
Q38TrunkItem *it=&g_trunk[i]; Q38Weight *w=it->w;
+ if(it->cpu_only){offered--;continue;}
int dev=qt_place_of(it->name,it->layer);
if(dev==QT_PLACE_CPU)continue;
int O=w->rows,I=w->cols;
@@ -2385,7 +2517,7 @@ static void q38_trunk_place_all(Model *m) {
}
if(g_trunk_n)
fprintf(stderr,"[qtier] qwen38 trunk: %d of %d offered matrices resident as int8 (%.2f GiB) in %.1fs; the rest answers from the CPU\n",
- placed,g_trunk_n,placed_bytes/1073741824.0,now_s()-t0);
+ placed,offered,placed_bytes/1073741824.0,now_s()-t0);
}
/* Start the tier after the model is loaded. COLI_CUDA=1 turns it on (the
@@ -2435,8 +2567,10 @@ static void q38_moe_decode(Model *m,Layer *l,int layer,const float *x,int S,floa
for(int z=0;z<K;z++) route_gates[z]=(float)(val[z]/den);
rt_route(layer,s,idx,route_gates,K); /* shared counts + post-normalization trace */
/* Resident experts run on the GPU from here on (asynchronously); the
- * CPU only loads and computes the rest, in route order. */
- uint32_t qmask=qt_issue(layer,idx,K,xs);
+ * CPU only loads and computes the rest, in route order. The tier
+ * knows the model's layers only: the MTP head's stays on the CPU. */
+ int tiered=layer<c->layers;
+ uint32_t qmask=tiered?qt_issue(layer,idx,K,xs):0;
int cpu_idx[Q38_MAX_TOPK],cpu_rank[Q38_MAX_TOPK],cpu_n=0;
for(int z=0;z<K;z++)if(!((qmask>>z)&1u)){cpu_idx[cpu_n]=idx[z];cpu_rank[cpu_n]=z;cpu_n++;}
q38_prefetch_experts(m,layer,cpu_idx,cpu_n);
@@ -2454,11 +2588,11 @@ static void q38_moe_decode(Model *m,Layer *l,int layer,const float *x,int S,floa
for(int j=0;j<I;j++)eh[j]=q38_silu(eg[j])*eu[j];q38_weight_matmul(eo,eh,&ex->down,1,I,H);
for(int d=0;d<H;d++)ys[d]+=route_gates[z]*eo[d];
q38_tm_add(m,Q38_TM_ROUTED_EXPERT,phase_started);
- q38_tier_note(layer,cpu_idx[i],ex);
+ if(tiered)q38_tier_note(layer,cpu_idx[i],ex);
}
/* GPU experts land after the CPU ones: same values, one more group in
* the float sum (that is the only ordering difference to a CPU run). */
- if(!qt_take(qmask,route_gates,K,ys)){
+ if(tiered&&!qt_take(qmask,route_gates,K,ys)){
fprintf(stderr,"qwen38: CUDA expert collection failed at layer %d; stopping inference\n",layer);
exit(1);
}
@@ -2685,7 +2819,10 @@ static void q38_moe_prefill(Model *m,Layer *l,int layer,const float *x,
}
static void q38_moe(Model *m,Layer *l,int layer,const float *x,int S,float *out) {
- if(S<=1||!m->prefill_batch)q38_moe_decode(m,l,layer,x,S,out);
+ /* An MTP verify on the CUDA tier routes row by row as decode does, so
+ * the tier's resident experts serve it (the batched path is CPU only). */
+ if(S<=1||!m->prefill_batch||(g_q38_rowwise&&qt_ready()&&layer<m->c.layers))
+ q38_moe_decode(m,l,layer,x,S,out);
else q38_moe_prefill(m,l,layer,x,S,out);
}
@@ -2697,17 +2834,46 @@ static void reset_recurrent(Model *m) {
memset(m->DN_conv[i],0,(size_t)c->dn_conv_dim*(c->dn_convk-1)*sizeof(float));
}
memset(m->PLE_conv_state,0,(size_t)c->hc_width*(c->ple_convk-1)*c->ngram_size*sizeof(float));m->ple_history_len=0;
+ m->mtp_len=0;m->mtp_pend_n=0; /* the MTP head's rows go with the model's */
}
static void ensure_kv(Model *m) {
Cfg *c=&m->c;if(m->max_t<=m->kv_cap&&m->K)return;
- if(m->K){for(int i=0;i<c->layers;i++){free(m->K[i]);free(m->V[i]);free(m->IK[i]);}free(m->K);free(m->V);free(m->IK);}
- m->K=(float**)calloc((size_t)c->layers,sizeof(float*));m->V=(float**)calloc((size_t)c->layers,sizeof(float*));m->IK=(float**)calloc((size_t)c->layers,sizeof(float*));
- for(int i=0;i<c->layers;i++)if(c->is_attn[i]){
+ int rows=q38_layer_rows(m);
+ if(m->K){for(int i=0;i<rows;i++){free(m->K[i]);free(m->V[i]);free(m->IK[i]);}free(m->K);free(m->V);free(m->IK);}
+ /* +1: the MTP head's attention rows at index c.layers */
+ m->K=(float**)calloc((size_t)c->layers+1,sizeof(float*));m->V=(float**)calloc((size_t)c->layers+1,sizeof(float*));m->IK=(float**)calloc((size_t)c->layers+1,sizeof(float*));
+ for(int i=0;i<rows;i++)if(i==c->layers||c->is_attn[i]){
m->K[i]=falloc((int64_t)c->kv_heads*m->max_t*c->head_dim);m->V[i]=falloc((int64_t)c->kv_heads*m->max_t*c->head_dim);m->IK[i]=falloc((int64_t)m->max_t*c->idx_dim);
}
m->kv_cap=m->max_t;
kv_prefix_alloc(&m->kvp,m->kv_cap); /* ensure_kv discards the old rows */
+ m->mtp_len=0;m->mtp_pend_n=0;
+}
+
+/* One decoder layer over the four hyper-connection streams: PLE where the
+ * model has it, the token mixer, then the MoE, each read from the streams
+ * and written back through its GatedResidual. step(), the Segment range and
+ * the MTP head (index c.layers, an attention layer) all run this. */
+static void q38_layer_forward(Model *m,int i,float *hyper,const int *ids,int S,int pos_base,
+ float *mixed,float *inject,float *block) {
+ Cfg *c=&m->c; Layer *l=&m->L[i]; int W=c->hc_width;
+ if(i==c->ple_layer){
+ float *ple=falloc((int64_t)S*W); q38_ple(m,ids,S,hyper,ple);
+ for(int64_t z=0;z<(int64_t)S*W;z++) hyper[z]+=ple[z];
+ free(ple);
+ /* consumate: il chunk successivo ha altri token, e riusare queste
+ * righe darebbe gli embedding del chunk precedente combaciando in
+ * silenzio invece di dare errore. */
+ free(m->ple_pref); m->ple_pref=NULL; m->ple_pref_rows=0;
+ }
+ q38_gr_read(m,&l->attn_gr,hyper,S,mixed,inject);
+ if(i>=c->layers||c->is_attn[i]) q38_attention(m,l,i,mixed,S,pos_base,block);
+ else q38_deltanet(m,l,i,mixed,S,block);
+ q38_gr_apply(c,hyper,block,inject,S);
+ q38_gr_read(m,&l->mlp_gr,hyper,S,mixed,inject);
+ q38_moe(m,l,i,mixed,S,block);
+ q38_gr_apply(c,hyper,block,inject,S);
}
/* Run only the requested native layer interval over hyper-residual activations.
@@ -2716,23 +2882,10 @@ static void ensure_kv(Model *m) {
static void q38_layers_forward_range(Model *m,float *hyper,const int *ids,
int S,int pos_base,int layer_begin,
int layer_end) {
- Cfg *c=&m->c; int H=c->hidden,W=c->hc_width,C=c->hc_count;
+ Cfg *c=&m->c; int H=c->hidden,C=c->hc_count;
float *mixed=falloc((int64_t)S*H),*inject=falloc((int64_t)S*C),*block=falloc((int64_t)S*H);
- for(int i=layer_begin;i<layer_end;i++){
- Layer *l=&m->L[i];
- if(i==c->ple_layer){
- float *ple=falloc((int64_t)S*W); q38_ple(m,ids,S,hyper,ple);
- for(int64_t z=0;z<(int64_t)S*W;z++) hyper[z]+=ple[z];
- free(ple);
- }
- q38_gr_read(m,&l->attn_gr,hyper,S,mixed,inject);
- if(c->is_attn[i]) q38_attention(m,l,i,mixed,S,pos_base,block);
- else q38_deltanet(m,l,i,mixed,S,block);
- q38_gr_apply(c,hyper,block,inject,S);
- q38_gr_read(m,&l->mlp_gr,hyper,S,mixed,inject);
- q38_moe(m,l,i,mixed,S,block);
- q38_gr_apply(c,hyper,block,inject,S);
- }
+ for(int i=layer_begin;i<layer_end;i++)
+ q38_layer_forward(m,i,hyper,ids,S,pos_base,mixed,inject,block);
free(mixed); free(inject); free(block);
}
@@ -2756,7 +2909,22 @@ static void q38_echo(const char *id, int pos, int token, const float *lo, int V,
free(piece);
}
-static float *step(Model *m,const int *ids,int S,int pos_base) {
+/* The input row of the token at absolute position abs_pos: its embedding,
+ * or the vision tower's row when an image covers that position. */
+static void q38_embed_row(Model *m,int id,int abs_pos,float *out) {
+ int vis_row=-1;
+ if(m->vis_map&&abs_pos>=0&&abs_pos<m->vis_map_len) vis_row=m->vis_map[abs_pos];
+ if(vis_row>=0&&vis_row<m->vis_rows_n)
+ memcpy(out,m->vis_rows+(int64_t)vis_row*m->c.hidden,(size_t)m->c.hidden*sizeof(float));
+ else q38_weight_row(&m->embed,id,out);
+}
+
+/* The forward behind step(): ids[0..S) at pos_base through every layer, the
+ * final mixer and lm_head. The logits of the last `nlogits` rows come back
+ * (step() asks for one, an MTP verify for two), and with `streams` every
+ * row's four streams as the final mixer read them are handed over instead of
+ * freed: that is the state the MTP head reads. */
+static float *q38_forward(Model *m,const int *ids,int S,int pos_base,int nlogits,float **streams) {
Cfg *c=&m->c;int H=c->hidden,W=c->hc_width,C=c->hc_count;
m->timers.forwards++;
float *hyper=falloc((int64_t)S*W);
@@ -2767,32 +2935,18 @@ static float *step(Model *m,const int *ids,int S,int pos_base) {
for(int s=0;s<S;s++){
if(ids[s]<0||ids[s]>=c->vocab){fprintf(stderr,"token id %d outside vocabulary\n",ids[s]);exit(1);}
float *e=hyper+(int64_t)s*W;
- int abs_pos=pos_base+s, vis_row=-1;
- if(m->vis_map&&abs_pos>=0&&abs_pos<m->vis_map_len) vis_row=m->vis_map[abs_pos];
- if(vis_row>=0&&vis_row<m->vis_rows_n)
- memcpy(e,m->vis_rows+(int64_t)vis_row*H,(size_t)H*sizeof(float));
- else q38_weight_row(&m->embed,ids[s],e);
+ q38_embed_row(m,ids[s],pos_base+s,e);
for(int b=1;b<C;b++)memcpy(e+(int64_t)b*H,e,(size_t)H*sizeof(float));
}
float *mixed=falloc((int64_t)S*H),*inject=falloc((int64_t)S*C),*block=falloc((int64_t)S*H);
- for(int i=0;i<c->layers;i++){
- Layer *l=&m->L[i];
- if(i==c->ple_layer){float *ple=falloc((int64_t)S*W);q38_ple(m,ids,S,hyper,ple);for(int64_t z=0;z<(int64_t)S*W;z++)hyper[z]+=ple[z];free(ple);
- /* consumate: il chunk successivo ha altri token, e riusare queste
- * righe darebbe gli embedding del chunk precedente combaciando in
- * silenzio invece di dare errore. */
- free(m->ple_pref); m->ple_pref=NULL; m->ple_pref_rows=0;}
- q38_gr_read(m,&l->attn_gr,hyper,S,mixed,inject);
- if(c->is_attn[i])q38_attention(m,l,i,mixed,S,pos_base,block);else q38_deltanet(m,l,i,mixed,S,block);
- q38_gr_apply(c,hyper,block,inject,S);
- q38_gr_read(m,&l->mlp_gr,hyper,S,mixed,inject);q38_moe(m,l,i,mixed,S,block);q38_gr_apply(c,hyper,block,inject,S);
- }
+ for(int i=0;i<c->layers;i++)
+ q38_layer_forward(m,i,hyper,ids,S,pos_base,mixed,inject,block);
q38_gr_read(m,&m->final_gr,hyper,S,mixed,NULL);m->kv_len=pos_base+S;
/* Rewinding and writing a shorter branch invalidates its old tail. */
if(m->kvp.len>pos_base)m->kvp.len=pos_base;
kv_prefix_record(&m->kvp,ids,pos_base,S);
if(m->vis_map && m->vis_rows_n>0)kv_prefix_taint(&m->kvp);
- float *logit=falloc(c->vocab);double phase_started=now_s();
+ float *logit=falloc((int64_t)nlogits*c->vocab);double phase_started=now_s();
/* Lettura del prefill: la posizione p predice il token p+1. Il primo token
* fresco lo predice la fotografia del prefisso, quando c'e. Pagata solo da
* chi ha chiesto il canale. */
@@ -2804,9 +2958,372 @@ static float *step(Model *m,const int *ids,int S,int pos_base) {
q38_echo(g_echo_id,pos_base+p+1,ids[p+1],logit,c->vocab,g_echo_k);
}
}
- q38_weight_matmul(logit,mixed+(int64_t)(S-1)*H,&m->lm_head,1,H,c->vocab);
+ q38_weight_matmul(logit,mixed+(int64_t)(S-nlogits)*H,&m->lm_head,nlogits,H,c->vocab);
q38_tm_add(m,Q38_TM_LM_HEAD,phase_started);
- free(hyper);free(mixed);free(inject);free(block);return logit;
+ if(streams)*streams=hyper; else free(hyper);
+ free(mixed);free(inject);free(block);return logit;
+}
+
+/* ---- the MTP head (Q38_MTP=1) ---------------------------------------------
+ * The checkpoint carries one more decoder layer under mtp.*, trained to read
+ * the model's state at position p together with the token at p+1 and to say
+ * what comes at p+2. Everything it computes is a block this engine already
+ * runs for the model's own layers and checks against the oracle: the decoder
+ * layer is q38_layer_forward at index c.layers (an attention layer:
+ * q38_attention with its own K/V/indexer rows and RoPE base, then q38_moe
+ * with its own router, shared expert and routed experts), its four streams
+ * leave through a GatedResidual of its own, mtp.hyper_connection_mixer, read
+ * by q38_gr_read exactly as q38_forward reads final_gr, and the shared
+ * lm_head gives the logits. The token at p+1 enters through the model's
+ * embedding (q38_embed_row); the head has none of its own.
+ *
+ * What the tensors leave open is how the two inputs become the four streams
+ * that layer reads. The model's streams at p (the 4 x hidden that the final
+ * mixer reads) meet pre_fc_norm_hidden [4*hidden] and fc_hidden
+ * [hidden, hidden]; the token's embedding meets pre_fc_norm_embedding
+ * [hidden] and fc_embedding [hidden, hidden]. Two square projections add (a
+ * single fc over the concatenation [e; h] is the same sum split in two), and
+ * the sum is a stream. Q38_MTP_WIRING picks one reading of the rest:
+ * a (default) pre_fc_norm_hidden normalizes the 4*hidden vector as one
+ * vector -- q38_rms0 over all of it, the way pre_fc_norm_embedding
+ * normalizes its input; neither norm belongs to a GatedResidual -- then
+ * fc_hidden maps each stream and fc_embedding(norm(e)) is added to all
+ * four, as q38_forward puts a token's embedding in all four streams;
+ * b as a, but the norm is per stream: q38_rms0 over each hidden-wide
+ * stream with its slice of the weight, the grouping hc_norm has in
+ * q38_gr_read;
+ * c the per-stream norm of b, the four streams averaged into one vector,
+ * fc_hidden on that, plus fc_embedding(norm(e)), and the result copied
+ * into the four streams as q38_forward does with an embedding.
+ * The output is the same whichever is chosen (the verify decides every
+ * token); the wirings differ in how often a draft is right, which only the
+ * trained weights can say, and the acceptance rate measures. Each norm is
+ * zero-centered (q38_rms0) like every Qwen4-Exp text RMSNorm but DeltaNet's
+ * gated one; q38_mtp_attach prints the mean of each norm weight, near 0 for
+ * that reading and near 1 for a plain norm. The pair (streams at p, token at
+ * p+1) is the head's row p: its K/V row and its RoPE position are p.
+ * Counting it at p+1 instead would change no score beyond rounding, since
+ * RoPE and the indexer's block scores depend on distances only.
+ *
+ * The head's KV rows hold verified pairs only, so it reads a row of the
+ * model's streams once the token after that row is known: the last row of a
+ * forward waits in mtp_pend, a draft runs the pending rows with the token
+ * the caller just picked, and the rows of a verify wait until the caller's
+ * next pick settles which of them stand. */
+enum { Q38_MTP_WHOLE_NORM='a', Q38_MTP_STREAM_NORM='b', Q38_MTP_STREAM_MEAN='c' };
+
+/* Q38_MTP_FORCE (tests): 'r' rejects every draft, a right one too, so the
+ * rollback runs on every token; 'a' drafts the reference's next token
+ * (ref.json, g_q38_mtp_oracle) so every draft is accepted; 'm' alternates a
+ * reference draft with a wrong one. The head still runs in every mode. */
+static const int *g_q38_mtp_oracle; static int g_q38_mtp_oracle_n;
+static FILE *g_q38_mtp_dump; /* Q38_MTP_DUMP=<file>: per draft, the head's row, its token and its logits (tests) */
+
+/* The four streams the head's layer starts from, for S pairs: row s pairs
+ * the model's streams at pos_base+s with the token next[s] at pos_base+s+1. */
+static void q38_mtp_input(Model *m,const float *streams,const int *next,int S,int pos_base,float *hyper) {
+ Cfg *c=&m->c; int H=c->hidden,W=c->hc_width,C=c->hc_count;
+ float *en=falloc((int64_t)S*H),*fe=falloc((int64_t)S*H),*row=falloc(H),*hn=falloc((int64_t)S*W);
+ for(int s=0;s<S;s++){
+ q38_embed_row(m,next[s],pos_base+s+1,row);
+ q38_rms0(en+(int64_t)s*H,row,m->mtp_norm_emb,H,c->eps);
+ }
+ q38_dense_matmul(m,fe,en,&m->mtp_fc_emb,S,H,H);
+ for(int s=0;s<S;s++){
+ const float *x=streams+(int64_t)s*W; float *y=hn+(int64_t)s*W;
+ if(m->mtp_wiring==Q38_MTP_WHOLE_NORM) q38_rms0(y,x,m->mtp_norm_hid,W,c->eps);
+ else for(int b=0;b<C;b++)
+ q38_rms0(y+(int64_t)b*H,x+(int64_t)b*H,m->mtp_norm_hid+(int64_t)b*H,H,c->eps);
+ }
+ if(m->mtp_wiring==Q38_MTP_STREAM_MEAN){
+ float *mean=falloc((int64_t)S*H),*fh=falloc((int64_t)S*H);
+ for(int s=0;s<S;s++)for(int d=0;d<H;d++){
+ float v=0.f;for(int b=0;b<C;b++)v+=hn[(int64_t)s*W+(int64_t)b*H+d];
+ mean[(int64_t)s*H+d]=v/C;
+ }
+ q38_dense_matmul(m,fh,mean,&m->mtp_fc_hid,S,H,H);
+ for(int s=0;s<S;s++)for(int b=0;b<C;b++)for(int d=0;d<H;d++)
+ hyper[(int64_t)s*W+(int64_t)b*H+d]=fh[(int64_t)s*H+d]+fe[(int64_t)s*H+d];
+ free(mean);free(fh);
+ } else {
+ /* fc_hidden on every stream: S rows of four streams are S*C rows of hidden */
+ float *fh=falloc((int64_t)S*W);
+ q38_dense_matmul(m,fh,hn,&m->mtp_fc_hid,S*C,H,H);
+ for(int s=0;s<S;s++)for(int b=0;b<C;b++)for(int d=0;d<H;d++)
+ hyper[(int64_t)s*W+(int64_t)b*H+d]=fh[(int64_t)s*W+(int64_t)b*H+d]+fe[(int64_t)s*H+d];
+ free(fh);
+ }
+ free(en);free(fe);free(row);free(hn);
+}
+
+/* The head over S pairs at rows pos_base..pos_base+S-1: their K/V/indexer
+ * rows enter its attention and, with `logit`, the last row's logits come
+ * back through the head's mixer and lm_head. Rows go in bounded chunks so a
+ * long prompt's pairs do not need a prompt-sized workspace; the head's
+ * attention is causal, so chunking changes no result. */
+static void q38_mtp_rows(Model *m,const float *streams,const int *next,int S,int pos_base,float *logit) {
+ Cfg *c=&m->c; int H=c->hidden,W=c->hc_width,C=c->hc_count;
+ int cap=q38_bounded_prefill_rows(S,0,(uint64_t)(3*W+2*H+C)*sizeof(float));
+ float *hyper=falloc((int64_t)cap*W),*mixed=falloc((int64_t)cap*H);
+ float *inject=falloc((int64_t)cap*C),*block=falloc((int64_t)cap*H);
+ for(int base=0;base<S;base+=cap){
+ int rows=S-base<cap?S-base:cap;
+ q38_mtp_input(m,streams+(int64_t)base*W,next+base,rows,pos_base+base,hyper);
+ q38_layer_forward(m,c->layers,hyper,next+base,rows,pos_base+base,mixed,inject,block);
+ m->mtp_len=pos_base+base+rows;
+ if(logit&&base+rows==S){
+ double started=now_s();
+ q38_gr_read(m,&m->mtp_mixer,hyper+(int64_t)(rows-1)*W,1,mixed,NULL);
+ q38_weight_matmul(logit,mixed,&m->lm_head,1,H,c->vocab);
+ q38_tm_add(m,Q38_TM_LM_HEAD,started);
+ }
+ }
+ free(hyper);free(mixed);free(inject);free(block);
+}
+
+/* The pending rows meet their next token: the head reads them (the first of
+ * two with the token already known in mtp_pend_tok, the last with `tok`). */
+static void q38_mtp_take_pending(Model *m,int tok,float *logit) {
+ int next[2]={m->mtp_pend_tok,tok},n=m->mtp_pend_n;
+ q38_mtp_rows(m,m->mtp_pend,next+2-n,n,m->mtp_len,logit);
+ m->mtp_pend_n=0;
+}
+
+/* After a forward fed ids[0..S) at pos_base (step()): the pairs it completes
+ * go into the head -- the pending rows with ids[0], then rows 0..S-2 with
+ * ids[1..S-1] -- and row S-1 waits for the token after it. A forward that
+ * does not continue the head's rows (a rewind no restore accompanied) stops
+ * the head until the next reset or restore: it cannot draft past rows it
+ * never read. */
+static void q38_mtp_feed(Model *m,const int *ids,int S,int pos_base,const float *streams) {
+ int W=m->c.hc_width;
+ if(m->mtp_len+m->mtp_pend_n>pos_base){
+ m->mtp_pend_n=0;
+ if(m->mtp_len>pos_base-1)m->mtp_len=pos_base>0?pos_base-1:0;
+ }
+ if(m->mtp_pend_n&&m->mtp_len+m->mtp_pend_n==pos_base)q38_mtp_take_pending(m,ids[0],NULL);
+ if(m->mtp_len!=pos_base){m->mtp_pend_n=0;return;}
+ if(S>1)q38_mtp_rows(m,streams,ids+1,S-1,pos_base,NULL);
+ memcpy(m->mtp_pend,streams+(int64_t)(S-1)*W,(size_t)W*sizeof(float));
+ m->mtp_pend_n=1;
+}
+
+static int q38_argmax(const float *lo,int V) {
+ int best=0;float value=lo[0];
+ for(int i=1;i<V;i++)if(lo[i]>value){value=lo[i];best=i;}
+ return best;
+}
+
+/* The head's draft for the token after `tok`, which is about to be fed at
+ * `pos`: the pending rows take `tok`, and the last one's logits name the
+ * token at pos+1. -1 when the head has not read up to pos. */
+static int q38_mtp_draft(Model *m,int tok,int pos) {
+ if(!m->mtp_pend_n||m->mtp_len+m->mtp_pend_n!=pos)return -1;
+ float *logit=falloc(m->c.vocab);
+ q38_mtp_take_pending(m,tok,logit);
+ int draft=q38_argmax(logit,m->c.vocab);
+ if(g_q38_mtp_dump){
+ int32_t head[2]={pos-1,tok};
+ if(fwrite(head,sizeof head,1,g_q38_mtp_dump)!=1||
+ fwrite(logit,sizeof(float),(size_t)m->c.vocab,g_q38_mtp_dump)!=(size_t)m->c.vocab){
+ fprintf(stderr,"Q38_MTP_DUMP: write failed\n");exit(1);
+ }
+ }
+ free(logit);
+ return draft;
+}
+
+static float *step(Model *m,const int *ids,int S,int pos_base) {
+ float *streams=NULL;
+ float *logit=q38_forward(m,ids,S,pos_base,1,m->mtp?&streams:NULL);
+ if(streams){q38_mtp_feed(m,ids,S,pos_base,streams);free(streams);}
+ return logit;
+}
+
+/* ---- speculative decoding with the MTP head -------------------------------
+ * colibri.c's DRAFT loop on a model with recurrent state. The caller asks
+ * for the logits that follow the token it just picked; q38_spec_step has
+ * the head draft the token after that one and feeds both in one forward
+ * (S=2, the verify), returns the first row's logits as step() would and
+ * keeps the second row's. The caller's next pick settles the draft. Equal:
+ * the kept logits answer with no forward. Not equal: the second row is
+ * undone -- the DeltaNet and PLE state go back to the copy the verify took
+ * after its first row (q38_deltanet and q38_ple take it when snap_after
+ * says so; the rollback swaps pointers), kv_len and the kv_prefix record go
+ * back one row, and the attention and indexer rows past it are a stale tail
+ * the next forward overwrites, as every rewind here treats them. The head's
+ * own rows never need undoing: it reads settled rows only.
+ *
+ * Every verify row is computed the way a decode step computes it (the CPU
+ * kernels give a row the same bits whatever S is; device matrices run row by
+ * row, see q38_weight_matmul), so the logits the caller sees, and its pick,
+ * are those of plain decoding, greedy or sampled: the draft never reaches
+ * the sampler, and a sample that equals it is simply an accepted draft. The
+ * one exception is the CUDA expert tier, whose float order already depends
+ * on which experts are resident when, with drafts or without. */
+typedef struct {
+ int on, force; /* drafting; Q38_MTP_FORCE's mode letter or 0 */
+ int ahead, draft, ahead_pos; /* a verify whose second row (the draft at ahead_pos) awaits the caller's pick */
+ float *ahead_logit, *ahead_streams;
+ uint64_t drafts, accepted, forwards, tokens;
+} Q38Spec;
+
+/* Back to the state after the verify's first row, `len` positions fed. */
+static void q38_spec_rollback(Model *m,int len) {
+ Cfg *c=&m->c;
+ for(int i=0;i<c->layers;i++)if(!c->is_attn[i]){
+ float *t=m->DN_rec[i];m->DN_rec[i]=m->snap_rec[i];m->snap_rec[i]=t;
+ t=m->DN_conv[i];m->DN_conv[i]=m->snap_conv[i];m->snap_conv[i]=t;
+ }
+ if(m->snap_ple){
+ float *t=m->PLE_conv_state;m->PLE_conv_state=m->snap_ple;m->snap_ple=t;
+ memcpy(m->ple_history,m->snap_ple_history,sizeof(m->snap_ple_history));
+ m->ple_history_len=m->snap_ple_history_len;
+ }
+ m->kv_len=len;
+ if(m->kvp.len>len)m->kvp.len=len;
+}
+
+/* The verify's second row did not stand: undo it, and its first row's
+ * streams wait for the token the caller picked instead. */
+static void q38_spec_drop_ahead(Model *m,Q38Spec *sp) {
+ q38_spec_rollback(m,sp->ahead_pos);
+ memcpy(m->mtp_pend,sp->ahead_streams,(size_t)m->c.hc_width*sizeof(float));
+ m->mtp_pend_n=1;
+ free(sp->ahead_logit);free(sp->ahead_streams);
+ sp->ahead_logit=sp->ahead_streams=NULL;sp->ahead=0;
+}
+
+/* The logits that follow `tok`, fed at `pos`, exactly as step(m,&tok,1,pos)
+ * gives them. `more` is how many tokens the caller may still want after
+ * `tok`: a draft saves a forward only when there is a second one. The
+ * returned buffer is the caller's (it may hold a second row past the first
+ * vocab floats). */
+static float *q38_spec_step(Model *m,Q38Spec *sp,int tok,int pos,int more) {
+ Cfg *c=&m->c; int V=c->vocab,W=c->hc_width;
+ sp->tokens++;
+ if(sp->ahead){
+ if(tok==sp->draft&&pos==sp->ahead_pos&&sp->force!='r'){
+ /* both verify rows stand: the first one's next token is the
+ * draft, the second waits for the caller's next pick */
+ float *logit=sp->ahead_logit;
+ memcpy(m->mtp_pend,sp->ahead_streams,(size_t)2*W*sizeof(float));
+ m->mtp_pend_n=2;m->mtp_pend_tok=tok;
+ free(sp->ahead_streams);
+ sp->ahead_logit=sp->ahead_streams=NULL;sp->ahead=0;sp->accepted++;
+ return logit;
+ }
+ q38_spec_drop_ahead(m,sp);
+ }
+ int draft=-1;
+ if(sp->on&&more>=2&&pos+1<m->kv_cap){
+ draft=q38_mtp_draft(m,tok,pos);
+ if(draft>=0&&(sp->force=='a'||sp->force=='m')){
+ int truth=pos+1<g_q38_mtp_oracle_n?g_q38_mtp_oracle[pos+1]:-1;
+ if(truth>=0)draft=sp->force=='m'&&(sp->drafts&1)?(truth+1)%V:truth;
+ }
+ }
+ if(draft<0){sp->forwards++;return step(m,&tok,1,pos);}
+ int ids[2]={tok,draft};float *streams=NULL;
+ m->snap_after=1;g_q38_rowwise=1;
+ float *logit=q38_forward(m,ids,2,pos,2,&streams);
+ m->snap_after=0;g_q38_rowwise=0;
+ sp->drafts++;sp->forwards++;
+ sp->ahead=1;sp->draft=draft;sp->ahead_pos=pos+1;
+ sp->ahead_logit=falloc(V);memcpy(sp->ahead_logit,logit+V,(size_t)V*sizeof(float));
+ sp->ahead_streams=streams;
+ return logit;
+}
+
+/* End of a generation: a verify row the caller never consumed is undone, so
+ * the state is the one plain decoding leaves after the same tokens. */
+static void q38_spec_end(Model *m,Q38Spec *sp) {
+ if(sp->ahead)q38_spec_drop_ahead(m,sp);
+}
+
+static float q38_mean(const float *x,int n) {
+ double sum=0.0; for(int i=0;i<n;i++) sum+=x[i];
+ return n?(float)(sum/n):0.f;
+}
+
+/* Q38_MTP=1: load the checkpoint's MTP head and draft with it. Unset or 0
+ * leaves it unread, as every run before it. The CLI and serve engine attach
+ * it after the model (the Segment and Edge adapters never do); a checkpoint
+ * without a head this engine runs is refused rather than decoded without
+ * one, so a benchmark cannot quietly measure plain decoding. The head's
+ * routed experts stay the snapshot's (FP8 on the release): the int4-g64
+ * sidecar holds the model's layers, and the head's one layer, read on at
+ * most two rows per draft, is a small part of the expert traffic. */
+static void q38_mtp_attach(Model *m,int cap) {
+ if(!q38_env_bool("Q38_MTP",0))return;
+ Cfg *c=&m->c;int H=c->hidden,W=c->hc_width;
+ if(m->range_begin!=0||m->range_end!=c->layers||!m->lm_head.rows){
+ fprintf(stderr,"Q38_MTP=1 needs the whole model loaded -- refusing\n");exit(1);
+ }
+ if(c->mtp_layers!=1){
+ if(c->mtp_layers==0)
+ fprintf(stderr,"Q38_MTP=1 but config.json has no MTP head (mtp_num_hidden_layers) -- refusing\n");
+ else if(c->mtp_layers<0)
+ fprintf(stderr,"Q38_MTP=1 but the config's MTP head is not one attention layer over the "
+ "model's embedding -- refusing\n");
+ else fprintf(stderr,"Q38_MTP=1 but the MTP head has %d layers; this engine runs one -- refusing\n",
+ c->mtp_layers);
+ exit(1);
+ }
+ const char *wiring=getenv("Q38_MTP_WIRING");
+ m->mtp_wiring=wiring&&*wiring?wiring[0]:Q38_MTP_WHOLE_NORM;
+ if((wiring&&*wiring&&wiring[1])||(m->mtp_wiring!=Q38_MTP_WHOLE_NORM&&
+ m->mtp_wiring!=Q38_MTP_STREAM_NORM&&m->mtp_wiring!=Q38_MTP_STREAM_MEAN)){
+ fprintf(stderr,"Q38_MTP_WIRING must be a, b or c (qwen38_core.h, the MTP head)\n");exit(1);
+ }
+ /* the release keeps the head at the top level, next to model.* */
+ char probe[96];const char *found=NULL;
+ const char *bases[3]={"mtp","model.mtp",NULL};
+ char nested[48];snprintf(nested,sizeof nested,"%s.mtp",m->prefix);bases[2]=nested;
+ for(int i=0;i<3&&!found;i++){
+ snprintf(probe,sizeof probe,"%s.fc_embedding.weight",bases[i]);
+ if(st_has(&m->S,probe))found=bases[i];
+ }
+ if(!found){fprintf(stderr,"Q38_MTP=1 but the checkpoint has no mtp.fc_embedding.weight -- refusing\n");exit(1);}
+ snprintf(m->mtp_prefix,sizeof m->mtp_prefix,"%s",found);
+ uint64_t resident_before=m->resident_weight_bytes;
+ char nm[384],base[320];
+ snprintf(nm,sizeof nm,"%s.pre_fc_norm_embedding.weight",found); m->mtp_norm_emb=q38_load_tensor(m,nm,H);
+ snprintf(nm,sizeof nm,"%s.pre_fc_norm_hidden.weight",found); m->mtp_norm_hid=q38_load_tensor(m,nm,W);
+ snprintf(nm,sizeof nm,"%s.fc_embedding.weight",found); m->mtp_fc_emb=q38_load_weight(m,nm,H,H);
+ snprintf(nm,sizeof nm,"%s.fc_hidden.weight",found); m->mtp_fc_hid=q38_load_weight(m,nm,H,H);
+ snprintf(base,sizeof base,"%s.hyper_connection_mixer",found); q38_load_gr_at(m,&m->mtp_mixer,base,0);
+ m->mtp=1; /* from here on index c.layers is the head's layer */
+ /* Q38_MTP_CAP: the head's expert cache, by default the cap every layer
+ * has. Its experts are FP8 even beside the int4 sidecar, so on a cap
+ * sized to fill RAM with int4 experts it costs more than a model layer. */
+ int mtp_cap=q38_env_positive_int("Q38_MTP_CAP",cap,c->experts);
+ q38_load_layer(m,c->layers,mtp_cap);
+ m->mtp_pend=falloc((int64_t)2*W);
+ /* the verify's rollback copies (q38_spec_rollback) */
+ m->snap_rec=(float**)calloc((size_t)c->layers,sizeof(float*));
+ m->snap_conv=(float**)calloc((size_t)c->layers,sizeof(float*));
+ if(!m->snap_rec||!m->snap_conv){fprintf(stderr,"OOM MTP rollback state\n");exit(1);}
+ for(int i=0;i<c->layers;i++)if(!c->is_attn[i]){
+ m->snap_rec[i]=falloc((int64_t)c->dn_vheads*c->dn_kdim*c->dn_vdim);
+ m->snap_conv[i]=falloc((int64_t)c->dn_conv_dim*(c->dn_convk-1));
+ }
+ if(m->PLE_conv_state)m->snap_ple=falloc((int64_t)W*(c->ple_convk-1)*c->ngram_size);
+ snprintf(nm,sizeof nm,"%s.layers.0.mlp.experts.0.gate_proj.weight",found);
+ st_tensor *expert=st_find(&m->S,nm);
+ double expert_bytes=expert?3.0*(double)expert->nbytes:0.0; /* gate, up and down are the same size */
+ const char *how=m->mtp_wiring==Q38_MTP_WHOLE_NORM?"one norm over the four streams, fc_hidden per stream":
+ m->mtp_wiring==Q38_MTP_STREAM_NORM?"a norm per stream, fc_hidden per stream":
+ "a norm per stream, the streams averaged, one fc_hidden";
+ fprintf(stderr,"[qwen38] MTP head %s.*: wiring %c (%s), RoPE base %g, %.2f MiB resident; "
+ "routed experts %s from the snapshot%s, cache %d (Q38_MTP_CAP) x %.2f MiB = %.2f GiB full\n",
+ found,m->mtp_wiring,how,(double)c->mtp_theta,
+ (m->resident_weight_bytes-resident_before)/1048576.0,
+ expert?st_dtype_name(expert->dtype):"(fused)",m->x4?" (the int4-g64 sidecar covers the model's layers)":"",
+ mtp_cap,expert_bytes/1048576.0,expert_bytes*mtp_cap/1073741824.0);
+ fprintf(stderr,"[qwen38] MTP norm weights, mean: embedding %.4f, hidden %.4f, mixer %.4f "
+ "(q38_rms0 scales by 1+w: near 0 fits it, near 1 would mean a plain norm)\n",
+ q38_mean(m->mtp_norm_emb,H),q38_mean(m->mtp_norm_hid,W),q38_mean(m->mtp_mixer.norm,W));
}
static int q38_tm_enabled(void) {
@@ -2877,7 +3394,8 @@ static void q38_layer_free(Layer *l) {
static void q38_model_free(Model *m) {
if(!m) return;
kv_prefix_free(&m->kvp);
- for(int i=0;i<m->c.layers;i++) {
+ int rows=q38_layer_rows(m);
+ for(int i=0;i<rows;i++) {
if(m->L)q38_layer_free(&m->L[i]);
if(m->cache) {
if(m->cache[i].slots) {
@@ -2892,9 +3410,14 @@ static void q38_model_free(Model *m) {
free(m->cache[i].slots); free(m->cache[i].by_expert);
}
if(m->expert_scales)free(m->expert_scales[i].values);
- free(m->DN_rec ? m->DN_rec[i] : NULL); free(m->DN_conv ? m->DN_conv[i] : NULL);
+ if(i<m->c.layers){free(m->DN_rec ? m->DN_rec[i] : NULL); free(m->DN_conv ? m->DN_conv[i] : NULL);}
+ if(i<m->c.layers){free(m->snap_rec ? m->snap_rec[i] : NULL); free(m->snap_conv ? m->snap_conv[i] : NULL);}
free(m->K ? m->K[i] : NULL); free(m->V ? m->V[i] : NULL); free(m->IK ? m->IK[i] : NULL);
}
+ free(m->snap_rec); free(m->snap_conv); free(m->snap_ple); free(m->mtp_pend);
+ free(m->mtp_norm_emb); free(m->mtp_norm_hid);
+ q38_weight_free(&m->mtp_fc_emb); q38_weight_free(&m->mtp_fc_hid);
+ free(m->mtp_mixer.norm); q38_weight_free(&m->mtp_mixer.down); q38_weight_free(&m->mtp_mixer.up);
if(m->x4){st_destroy(&m->x4->S);free(m->x4->fd);free(m->x4->off);free(m->x4);}
free(m->L); free(m->cache); free(m->expert_scales); free(m->DN_rec); free(m->DN_conv); free(m->K); free(m->V); free(m->IK);
q38_weight_free(&m->embed);q38_weight_free(&m->lm_head);
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index 10ddc986..e80baa85 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -17,7 +17,7 @@ what follows, but the sister engines read their own:
| `kimi_k3` | `c/kimi_k3.c` | the `K3_*` family β€” see [Kimi K3 engine](#kimi-k3-engine-kimi_k3) |
| `inkling` | `c/inkling.c` | `INK_*`, plus `CTX_MAX`, `PIN_N`, `REP_PEN`, `GPU_DEV`, `NOGPU` β€” see [Inkling engine](#inkling-engine-inkling) |
| `qwen36` | `c/qwen36.c` | `QWEN_*`, `Q36_*`, its dense/CUDA-tier controls, and the `CACHE_ROUTE` family (VRAM tier over RAM cache) β€” see [Qwen3.6 engine](#qwen36-engine-qwen36) |
-| `qwen38` | `c/qwen38.c` | `Q38_MAXT`, `Q38_EOS`, `Q38_NATIVE_FP8`, `Q38_NATIVE_BF16`, `Q38_EXPERT_INT4`, `Q38_PREFILL_BATCH`, `Q38_TRUNK_CPU_INT8`, `Q38_FP8_KERNEL`, `COLI_TIMERS` β€” see [Qwen3.8 engine](#qwen38-engine-qwen38) |
+| `qwen38` | `c/qwen38.c` | `Q38_MAXT`, `Q38_EOS`, `Q38_NATIVE_FP8`, `Q38_NATIVE_BF16`, `Q38_EXPERT_INT4`, `Q38_MTP`, `Q38_PREFILL_BATCH`, `Q38_TRUNK_CPU_INT8`, `Q38_FP8_KERNEL`, `COLI_TIMERS` β€” see [Qwen3.8 engine](#qwen38-engine-qwen38) |
| `olmoe` | `c/olmoe.c` | `HOT`, `WIDE`, `SMOOTH`, `CONF_LIMIT`, `MAX_NEW`, `CHAT`, `EXPERT_DROP`, `WARMUP` β€” see [OLMoE engine](#olmoe-engine-olmoe) |
| `deepseek_v4` | `c/deepseek_v4.c` | `CTX`, the `V4_*` / `DSV4_*` families and the two `COLI_CUDA_*_BATCH` gates β€” see [DeepSeek V4 engine](#deepseek-v4-engine-deepseek_v4); note that the CUDA section below describes `colibri.c` knobs (`COLI_CUDA`, `CUDA_DENSE`, ...) which the V4 engine does not read β€” its GPU switch is `DSV4_CUDA` |
| `deepseek_v41` | `c/deepseek_v41.c` | the `V41_*` family: see [DeepSeek V4.1 engine](#deepseek-v41-engine-deepseek_v41) |
@@ -445,6 +445,10 @@ checkpoint layout and the text-only capability boundary.
| `Q38_NATIVE_FP8` | `1` (on) | Keep routed E4M3 expert bytes and their F32 128Γ—128 block scales native in the LRU. `=0` restores expanded-FP32 slots for A/B validation. |
| `Q38_NATIVE_BF16` | `1` (on) | Keep resident and routed BF16 matrices in two-byte storage while retaining FP32 activations/accumulation. `=0` restores the expanded-FP32 reference. |
| `Q38_EXPERT_INT4` | auto | Routed experts from `<snap>/experts-int4g64/`, the int4-g64 sidecar `tools/convert_qwen38_experts_int4.py` writes next to the FP8 shards: 2.76 MB per expert instead of 4.92 MB, so the same RAM holds 1.78x the experts and a miss reads 56% of the bytes. Unset: used when its `index.json` says the conversion is complete (an incomplete one is skipped with a line on stderr; one that disagrees with the model is refused). `=0` keeps the snapshot's FP8 experts; `=1` refuses to start without a complete sidecar. The CUDA expert tier streams FP8 only and stays off with int4 experts; `coli plan` / `--auto-tier` size the cap with the sidecar's records. See [qwen38.md](qwen38.md#routed-experts-as-int4-g64). |
+| `Q38_MTP` | `0` (off) | `=1` loads the checkpoint's MTP head and decodes speculatively: the head drafts the next token and one forward over two rows verifies it; a rejected draft rolls the DeltaNet/PLE state back to a copy taken after the first row. The output is that of plain decoding, greedy or sampled. A checkpoint without an MTP head is refused. Acceptance and tokens/forward go to stderr at the end of a run and after every served turn. See [qwen38.md](qwen38.md#speculative-decoding-with-the-mtp-head). |
+| `Q38_MTP_WIRING` | `a` | How the head's two inputs become its four streams, which the tensors leave open: `a` one norm over the 4 x hidden streams, `b` one norm per stream, `c` per-stream norms averaged before `fc_hidden` (`c/qwen38_core.h`, "the MTP head"). Only the acceptance rate depends on it. |
+| `Q38_MTP_CAP` | the cap | Expert cache of the MTP head's layer. Its experts are the snapshot's FP8 even beside the int4-g64 sidecar; the startup line prints what the cache costs full. |
+| `Q38_MTP_FORCE`, `Q38_MTP_DUMP` | unset | Tests only: `reject` rolls back every draft, `accept` / `mixed` draft the reference's next token (always, every other one) in `ref.json` mode; `Q38_MTP_DUMP=<file>` writes every draft's row, token and logits. |
| `Q38_PREFILL_BATCH` | `1` (on) | Route prompt rows in bounded expert-major chunks and batch resident shared-expert/DeltaNet projections. `=0` restores row-at-a-time prompt execution for A/B diagnosis; decode is unchanged. |
| `Q38_TRUNK_CPU_INT8` | `1` (on) | The dense trunk (DeltaNet and attention projections, hyper-connection mixers, shared expert, router, lm_head; every matrix of at least `Q38_TRUNK_MIN_KB`) is kept on the CPU as int8 rows with one scale per row and the BF16 copy is released; `q38_weight_matmul` quantizes the activation to int8 and uses the integer kernels of `idot.h` for decode and prefill. `=0` keeps the BF16 rows and the f32 kernel (the numeric reference). See [qwen38.md](qwen38.md#the-trunk-on-the-cpu-int8-rows). |
| `COLI_MAP_EXPERTS` | `0` | Point the native-FP8 routed-expert slots (or the int4-g64 sidecar's records) at a read-only mapping of their shard instead of copying 14 MB per miss into a slab. Same variable as in `glm53`. |
diff --git a/docs/qwen38.md b/docs/qwen38.md
index dd3e61be..bea6d1dc 100644
--- a/docs/qwen38.md
+++ b/docs/qwen38.md
@@ -4,8 +4,9 @@
[`Qwen/Qwen3.8-Flash-Next-FP8`](https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8)
directly from the official safetensors shards. No conversion or second copy of
the weights is required. The engine supports text and images through the
-checkpoint's vision encoder; see **Vision** below. It does not use the optional
-MTP layer.
+checkpoint's vision encoder; see **Vision** below. The checkpoint's optional
+MTP layer drafts tokens for speculative decoding under `Q38_MTP=1`; see
+[Speculative decoding with the MTP head](#speculative-decoding-with-the-mtp-head).
The upstream language model has 125B ordinary parameters with 6B activated,
plus a 51B hashed n-gram embedding. It has 48 layers arranged as 12 repetitions
@@ -261,6 +262,56 @@ reference of the dequantized sidecar through every expert path with the last
logits identical to the bit across them, and the CUDA tier declining the int4
experts on the fake backend.
+## Speculative decoding with the MTP head
+
+Optional, off by default. The release carries one more decoder layer under
+`mtp.*` that reads the model's four hyper-connection streams at a position
+together with the next token's embedding and predicts the token after it.
+`Q38_MTP=1` loads it and decodes speculatively: after each token the head
+drafts the next one, and one forward over both (S=2) returns the logits of the
+first as a plain decode step would while it checks the draft. When the token
+picked from those logits equals the draft, the second row's logits answer the
+next step with no forward; when it does not, the second row is undone: the
+Gated DeltaNet and PLE state go back to a copy taken after the first row (a
+pointer swap, nothing is recomputed) and the attention, indexer and prefix
+records go back one position.
+
+Every row of the verify forward is computed as a decode step computes it, so
+the output is the output of plain decoding, token for token and logit for logit,
+greedy or sampled (a sampled token that happens to equal the draft is an
+accepted draft). The CUDA expert tier is the exception it already is: its float
+order follows which experts are resident when.
+
+The head is built only from blocks the engine already runs for its own layers:
+the decoder layer is a QSA attention layer with its indexer, gated residuals,
+the MoE with its shared expert, the head's own hyper-connection mixer and the
+shared `lm_head`. How its two inputs become the four streams that layer reads
+is not fixed by the tensors, so `Q38_MTP_WIRING` selects a reading (`c/qwen38_core.h`,
+"the MTP head"): `a` (default) normalizes the 4 x 2560 streams as one vector,
+`b` normalizes each stream, `c` averages the normalized streams before
+`fc_hidden`. The output does not depend on the choice; the acceptance rate does,
+and it is what tells which reading the trained weights agree with.
+
+Every run prints its line, and a served request prints one per turn, on stderr:
+
+```
+[qwen38 MTP] run: 1.75 tokens/forward (4 forwards per 7 tokens) | acceptance 100.0% (3/3 drafts) | wiring a
+```
+
+Tokens are the decode tokens after the prompt and forwards the forwards that
+produced them. The head's routed experts stay the snapshot's FP8 with or
+without the int4-g64 sidecar (one layer read for at most two rows per draft is
+a small share of the expert traffic); their cache holds `Q38_MTP_CAP` experts,
+by default the cap every layer has, and the startup line prints what it costs.
+
+`make -C c qwen38-tiny-mtp-check` adds a head with random weights under the
+release's names to the three tiny fixtures and gates on the head's draft
+logits against a float32 reference of each wiring
+(`c/tools/qwen38_mtp_ref.py`), and on byte-identical output with drafts and
+without them across caps, prefill batching, both trunk formats, BF16, FP8 and
+int4 experts, text and serve, with drafts all rejected, all accepted and
+alternating.
+
## GPU: CUDA VRAM expert tier
With `CUDA=1` the engine links the same expert tier as Qwen3.6
@@ -519,7 +570,9 @@ vector, and runs both native-BF16 and expanded-FP32 resident modes at cache
capacities one and four. CI repeats the capacity-one path under ASan and UBSan
and verifies that a config/tensor shape disagreement is refused.
`make -C c qwen38-tiny-int4-check` gates the optional int4-g64 experts (see
-[Routed experts as int4-g64](#routed-experts-as-int4-g64)).
+[Routed experts as int4-g64](#routed-experts-as-int4-g64)), and
+`make -C c qwen38-tiny-mtp-check` the MTP head (see
+[Speculative decoding with the MTP head](#speculative-decoding-with-the-mtp-head)).
## Supported checkpoint layouts