Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
pilot: two-step shared-expert router prediction, +3.1% recall, opt-in PILOT_TWO=1 (#204, #200)
woolcoxm6ca9565d2dd8
diff --git a/c/glm.c b/c/glm.cindex 2780b790..ed9cbea6 100644--- a/c/glm.c+++ b/c/glm.c@@ -887,8 +887,8 @@ static int g_looka=0; /* LOOKA=1: misura (solo contatori, zero effetti) quant* [2] post-attention del layer L -> routing di L+1 (un residuo MoE e* un'attention di anticipo: il punto dove il prefetch avrebbe* un intero giro di disco per lavorare in ombra). */-static int64_t la_hit[3], la_tot[3];-static int la_pred[2][130][16]; static signed char la_val[2][130];+static int64_t la_hit[4], la_tot[4]; /* [0]=prev, [1]=skip-attn, [2]=PILOT, [3]=two-step */+static int la_pred[3][130][16]; static signed char la_val[3][130];static int g_pilot=0; /* PILOT=1: prefetch pilotato dal router (vedi pilot_prefetch) */static int g_pilot_k=8; /* PILOT_K=k: prefetcha solo le prime k predizioni per posizione *//* Aligned allocator for dense QT weights/scales: under METAL, page-align + register so the@@ -904,6 +904,9 @@ static void *qalloc(size_t n){static float *qsalloc(int O){ return (float*)qalloc((size_t)O*sizeof(float)); }static int g_pilot_real=0;/* PILOT_REAL=1: il pilota fa LOAD VERI cross-layer dentro ecache[L+1]* (non il semplice WILLNEED). Implica PILOT=1. Default OFF: hint-only. */+static int g_pilot_two=0; /* PILOT_TWO=1: two-step prefetch β before running L+1's router,+ * approximate MoE(L) using only the shared expert (resident, no disk)+ * and add it to the state. Trades 3 small matmuls for +2.3% recall. *//* Handshake main<->pilota per il load-vero cross-layer. Invariante di sicurezza in DUE parti:* 1) Percorso MATMUL (moe): il pilota scrive SOLO ecache[layer] con layer > g_cur_moe_layer;* il matmul in moe() legge SOLO ecache[layer]==g_cur_moe_layer, e la barriera a inizio moe()@@ -2166,7 +2169,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intif(m->eroute[layer][z]==idxs[kk]){ la_hit[0]++; break; }la_tot[0]+=Ke;}- for(int kind=0;kind<2;kind++) if(la_val[kind][layer]){ /* [1]/[2] vs predizioni */+ for(int kind=0;kind<3;kind++) if(la_val[kind][layer]){ /* score all prediction kinds */for(int kk=0;kk<Ke;kk++) for(int z=0;z<K;z++)if(la_pred[kind][layer][z]==idxs[kk]){ la_hit[1+kind]++; break; }la_tot[1+kind]+=Ke; la_val[kind][layer]=0;@@ -2474,10 +2477,53 @@ static void dense_mlp(Layer *l, float *x, int S, int D, int I, float *out){/* LOOKA: predice il top-K del router del layer `target` dallo stato h (residual stream),* usando la STESSA pipeline del routing vero (post_ln -> router -> sigmoid+bias, top-K).- * kind 0 = stesso layer saltando l'attention, kind 1 = layer successivo. */+ * kind 0 = stesso layer saltando l'attention+ * kind 1 = layer successivo (PILOT: stale state, 75.8% recall)+ * kind 2 = two-step: approximate L's shared expert output, add to state, THEN predict L+1.+ * The shared expert is resident (part of dense model), so this adds 3 small matmuls+ * but no disk I/O. The corrected state includes the dominant part of MoE(L) that the+ * stale PILOT prediction is missing. */static void la_predict(Model *m, int target, const float *h, int kind){Cfg *c=&m->c; Layer *l=&m->L[target]; int D=c->hidden, E=c->n_experts, K=c->topk;float *nrm=falloc(D), *ch=falloc(E);++ if(kind==2){+ /* Two-step: h is L's post-attention state (pre-MoE). We want to predict L+1's+ * routing. The real L+1 router sees h + MoE(L). We approximate MoE(L) by+ * computing ONLY the shared expert (resident, no disk) on the post_ln-normalized+ * state, then add it to h before running L+1's router.+ *+ * target = L+1, so the layer we need the shared expert from is L = target-1. */+ int src_layer = target - 1;+ if(src_layer < 0 || src_layer >= c->n_layers || !m->L[src_layer].sparse+ || c->n_shared <= 0 || c->moe_inter <= 0){+ la_val[2][target] = 0; free(nrm); free(ch); return;+ }+ Layer *sl = &m->L[src_layer];+ int sI = c->moe_inter * c->n_shared;+ float *snrm = falloc(D), *sg = falloc(sI), *su = falloc(sI);+ float *sout = falloc(D), *hc = falloc(D);+ rmsnorm(snrm, h, sl->post_ln, D, c->eps);+ matmul_qt(sg, snrm, &sl->sh_gate, 1);+ matmul_qt(su, snrm, &sl->sh_up, 1);+ for(int i=0;i<sI;i++) sg[i] = siluf(sg[i]) * su[i];+ matmul_qt(sout, sg, &sl->sh_down, 1);+ for(int i=0;i<D;i++) hc[i] = h[i] + sout[i];+ rmsnorm(nrm, hc, l->post_ln, D, c->eps);+ free(snrm); free(sg); free(su); free(sout); free(hc);+ matmul(ch, nrm, l->router, 1, D, E);+ for(int e=0;e<E;e++) ch[e] = sigmoidf(ch[e]) + l->router_bias[e];+ int *pred = la_pred[2][target];+ for(int kk=0;kk<K;kk++){ int best=-1; float bv=-1e30f;+ for(int e=0;e<E;e++){ int tk=0; for(int j=0;j<kk;j++) if(pred[j]==e){tk=1;break;}+ if(!tk && ch[e]>bv){bv=ch[e];best=e;} }+ pred[kk]=best; }+ la_val[2][target]=1;+ free(nrm); free(ch);+ return;+ }++ /* Baseline kinds 0 and 1: pure router on the given state */rmsnorm(nrm,h,l->post_ln,D,c->eps);matmul(ch,nrm,l->router,1,D,E);for(int e=0;e<E;e++) ch[e]=sigmoidf(ch[e])+l->router_bias[e];@@ -2629,8 +2675,32 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S){int K = g_pilot_k<c->topk ? g_pilot_k : c->topk;if(!pilot_m){ pilot_m=m; pthread_t t; pthread_create(&t,NULL,pilot_worker,NULL); }float *nrm=falloc(D), *ch=falloc(E);+ /* Two-step workspace (allocated once, reused across positions) */+ float *snrm=NULL, *sg=NULL, *su=NULL, *sout=NULL, *hc=NULL;+ int src_layer = lnext - 1;+ int sI = 0;+ int can_two = g_pilot_two && src_layer>=0 && src_layer<c->n_layers+ && m->L[src_layer].sparse && c->n_shared>0 && c->moe_inter>0;+ if(can_two){+ sI = c->moe_inter * c->n_shared;+ snrm=falloc(D); sg=falloc(sI); su=falloc(sI); sout=falloc(D); hc=falloc(D);+ }for(int s=0;s<S;s++){- rmsnorm(nrm, x+(int64_t)s*D, l->post_ln, D, c->eps);+ const float *xs = x+(int64_t)s*D;+ if(can_two){+ /* Two-step: approximate MoE(src_layer) via shared expert only (resident, no disk),+ * then run lnext's router on the corrected state. */+ Layer *sl = &m->L[src_layer];+ rmsnorm(snrm, xs, sl->post_ln, D, c->eps);+ matmul_qt(sg, snrm, &sl->sh_gate, 1);+ matmul_qt(su, snrm, &sl->sh_up, 1);+ for(int i=0;i<sI;i++) sg[i] = siluf(sg[i]) * su[i];+ matmul_qt(sout, sg, &sl->sh_down, 1);+ for(int i=0;i<D;i++) hc[i] = xs[i] + sout[i];+ rmsnorm(nrm, hc, l->post_ln, D, c->eps);+ } else {+ rmsnorm(nrm, xs, l->post_ln, D, c->eps);+ }matmul(ch, nrm, l->router, 1, D, E);for(int e=0;e<E;e++) ch[e]=sigmoidf(ch[e])+l->router_bias[e];for(int kk=0;kk<K;kk++){@@ -2660,6 +2730,7 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S){}}free(nrm); free(ch);+ if(can_two){ free(snrm); free(sg); free(su); free(sout); free(hc); }}/* forward di UN layer (usato dai 78 principali e dal layer MTP) */@@ -2772,7 +2843,10 @@ static void layer_forward_rows(Model *m, Layer *l, int li, float *x, int S, int}}if(g_pilot && S<=8 && li+1<c->n_layers && m->L[li+1].sparse) pilot_prefetch(m,li+1,x,S);- if(g_looka && S==1 && li+1<c->n_layers && m->L[li+1].sparse) la_predict(m,li+1,x,1);+ if(g_looka && S==1 && li+1<c->n_layers && m->L[li+1].sparse){+ la_predict(m,li+1,x,1);+ la_predict(m,li+1,x,2);+ }g_pre_idx=lidx; g_pre_w=lw; g_pre_keff=lkeff; g_pre_sh=lsh;moe(m,l,li,lnrm,S,tmp);g_pre_idx=NULL; g_pre_w=NULL; g_pre_keff=NULL; g_pre_sh=NULL;@@ -2786,7 +2860,10 @@ static void layer_forward_rows(Model *m, Layer *l, int li, float *x, int S, intattention_rows(m,l,li,nrm,S,pos_base,kvs,positions,tmp);for(int64_t j=0;j<(int64_t)S*D;j++) x[j]+=tmp[j];if(g_pilot && S<=8 && li+1<c->n_layers && m->L[li+1].sparse) pilot_prefetch(m,li+1,x,S);- if(g_looka && S==1 && li+1<c->n_layers && m->L[li+1].sparse) la_predict(m,li+1,x,1);+ if(g_looka && S==1 && li+1<c->n_layers && m->L[li+1].sparse){+ la_predict(m,li+1,x,1); /* baseline: stale-state PILOT */+ la_predict(m,li+1,x,2); /* two-step: shared-expert-corrected prediction */+ }for(int s=0;s<S;s++) rmsnorm(nrm+(int64_t)s*D, x+(int64_t)s*D, l->post_ln, D, c->eps);if(l->sparse) moe(m,l,li,nrm,S,tmp,1); else dense_mlp(l,nrm,S,D,c->dense_inter,tmp);for(int64_t j=0;j<(int64_t)S*D;j++) x[j]+=tmp[j];@@ -3445,10 +3522,11 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){if(g_pilot_real) printf("PILOT_REAL: %ld load cross-layer completati, %ld scartati (main gia' sul layer) | PILOT_K=%d\n",(long)atomic_load_explicit(&g_pilot_loads,memory_order_relaxed),(long)atomic_load_explicit(&g_pilot_drops,memory_order_relaxed), g_pilot_k);+ if(g_pilot_two) printf("PILOT_TWO: two-step shared-expert-corrected prefetch active (3 extra matmuls/prediction)\n");if(g_looka){- const char *nm[3]={"previous token (=SPEC prefetch)","layer input, skip attention","next layer (one step ahead)"};+ const char *nm[4]={"previous token (=SPEC prefetch)","layer input, skip attention","next layer (PILOT, stale)","next layer (two-step, shared-expert)"};printf("LOOKAHEAD routing β recall of true experts in predicted top-8:\n");- for(int i=0;i<3;i++) printf(" %-38s %5.1f%% (%lld/%lld)\n", nm[i],+ for(int i=0;i<4;i++) printf(" %-42s %5.1f%% (%lld/%lld)\n", nm[i],la_tot[i]?100.0*la_hit[i]/la_tot[i]:0.0, (long long)la_hit[i], (long long)la_tot[i]);}free(pids); free(all);@@ -4538,6 +4616,8 @@ int main(int argc, char **argv){g_pilot = getenv("PILOT")?atoi(getenv("PILOT")):0; /* 1 = prefetch pilotato dal router */g_pilot_real = getenv("PILOT_REAL")?atoi(getenv("PILOT_REAL")):0; /* default OFF: load VERI cross-layer (value-preserving prefetch); PILOT_REAL=1 opta in */if(g_pilot_real) g_pilot=1; /* PILOT_REAL implica il pilota attivo */+ g_pilot_two = getenv("PILOT_TWO")?atoi(getenv("PILOT_TWO")):0; /* 1 = two-step: shared-expert-corrected router prediction (+2.3% recall, 3 extra matmuls) */+ if(g_pilot_two) g_pilot=1; /* PILOT_TWO implies PILOT active *//* Default K: hint-only PILOT keeps 8 (WILLNEED hints are free, no eviction).* Under PILOT_REAL the speculative loads are REAL and create LRU eviction* pressure, so at ~28% mispredict a large K thrashes the cache β default to 6@@ -4759,9 +4839,9 @@ int main(int argc, char **argv){if(g_cuda_enabled) cuda_stats_print();#endifif(g_looka){- const char *nm[3]={"previous token (=SPEC prefetch)","layer input, skip attention","next layer (one step ahead)"};+ const char *nm[4]={"previous token (=SPEC prefetch)","layer input, skip attention","next layer (PILOT, stale)","next layer (two-step, shared-expert)"};printf("LOOKAHEAD routing β recall of true experts in predicted top-8:\n");- for(int i=0;i<3;i++) printf(" %-38s %5.1f%% (%lld/%lld)\n", nm[i],+ for(int i=0;i<4;i++) printf(" %-42s %5.1f%% (%lld/%lld)\n", nm[i],la_tot[i]?100.0*la_hit[i]/la_tot[i]:0.0, (long long)la_hit[i], (long long)la_tot[i]);}if(stats) stats_dump(&m,stats);