Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
qwen36 tier: qt_replan, the resident set follows the prompt's own routing
kreuzzelg0fc8834bb030
diff --git a/c/qwen36_tier.c b/c/qwen36_tier.cindex d501471c..e98c9616 100644--- a/c/qwen36_tier.c+++ b/c/qwen36_tier.c@@ -65,9 +65,16 @@ static struct {int *fill_order; int fill_cur; /* warmstart order (heat desc) */int issue_open; /* guard: no tensor_free while a group is in flight */pthread_cond_t cv_take; /* signals qt_take done + queue space */- uint64_t tick, swaps, pf_hits, pf_notes;+ uint64_t tick, swaps, inplace, pf_hits, pf_notes;uint32_t *heat0; /* heat table loaded from HEAT_FILE */+ /* re-plan (qt_replan): pending (candidate, victim) slot-index pairs, drained on the decode ticks */+ int *rp_c, *rp_v; int rp_n, rp_i; uint64_t rp_planned, rp_done;+ /* evicted-expert ring for qt_evicted_take (slot indices) */+ int *ev; int ev_h, ev_n;+ /* qt_stats_mark: counters at the prefill/decode boundary */+ uint64_t mk_hits, mk_miss; int mk_on;} G;+#define QT_EV_CAP 8192/* Count parked callers so shutdown can reclaim their shared storage safely. */static void wait_take_locked(void){@@ -168,8 +175,16 @@ static void *uploader(void *arg){uint8_t *w=G.q[G.qh].w; float *sc=G.q[G.qh].s;G.qh=(G.qh+1)%QT_QCAP; G.qn--;pthread_cond_broadcast(&G.cv_take); /* queue space available */+ /* staging layout: three matrices of mb bytes each; int4 packed = half a+ * byte per element, int8/fp8 = one. Scales follow the format. */+ size_t mb=(size_t)G.D*G.Ih/(G.wfmt==4?2:1);+ const float *sg=sc, *su, *sd;+ if(G.wfmt==8||G.egs){ su=sc+G.sc_gu; sd=sc+2*G.sc_gu; } else { su=sc+G.Ih; sd=sc+2*G.Ih; }+ int dv = G.dev[home2(layer,eid)];+ ColiCudaTensor *tg=NULL,*tu=NULL,*td=NULL;+ int ok=0, inplace=0;if(ve>=0){- /* LFRU swap: free the victim only when no group is in flight */+ /* LFRU swap: touch the victim only when no group is in flight */while(G.issue_open && !G.th_stop) wait_take_locked();QSlot *v=qs(vl,ve);if(G.th_stop && G.issue_open){@@ -187,36 +202,43 @@ static void *uploader(void *arg){ColiCudaTensor *a=v->tg,*b=v->tu,*ct=v->td;v->tg=v->tu=v->td=NULL;pthread_mutex_unlock(&G.mx);- if(a)coli_cuda_tensor_free(a); if(b)coli_cuda_tensor_free(b); if(ct)coli_cuda_tensor_free(ct);+ /* Every expert has the same geometry: write the newcomer into the+ * victim's device buffers instead of cudaFree + cudaMalloc, each+ * of which synchronises the device under the async groups (a+ * re-plan of 1,500 swaps cost 1.3 s of prefill that way). A+ * backend without the entry point, or a failure half-way, falls+ * back to the old free-then-upload. */+ if(a&&b&&ct && coli_cuda_tensor_overwrite(a,w,sg) && coli_cuda_tensor_overwrite(b,w+mb,su)+ && coli_cuda_tensor_overwrite(ct,w+2*mb,sd)){+ tg=a; tu=b; td=ct; ok=1; inplace=1;+ } else {+ if(a)coli_cuda_tensor_free(a); if(b)coli_cuda_tensor_free(b); if(ct)coli_cuda_tensor_free(ct);+ }} else pthread_mutex_unlock(&G.mx);- int dv = G.dev[home2(layer,eid)];- /* passo fra le tre matrici nello staging: int4 impacchettato = mezzo- * byte per elemento, int8 = uno. */- size_t mb=(size_t)G.D*G.Ih/(G.wfmt==4?2:1);- ColiCudaTensor *tg=NULL,*tu=NULL,*td=NULL;- int ok;- if(G.wfmt==8){- /* e4m3 bytes as they came from the checkpoint, block scales- * [ceil(O/128), ceil(I/128)] per matrix -- the layout #817's- * kernels and tensor_upload(fmt=8) already agree on */- ok = coli_cuda_tensor_upload(&tg, w, sc, 8, G.D, G.Ih, dv)- && coli_cuda_tensor_upload(&tu, w+mb, sc+G.sc_gu, 8, G.D, G.Ih, dv)- && coli_cuda_tensor_upload(&td, w+2*mb, sc+2*G.sc_gu, 8, G.Ih, G.D, dv);- } else if(G.wfmt==1){- /* int8, scale per riga: qt_init ha gia' rifiutato il caso raggruppato,- * che questo formato non sa esprimere. */- ok = coli_cuda_tensor_upload(&tg, w, sc, 1, G.D, G.Ih, dv)- && coli_cuda_tensor_upload(&tu, w+mb, sc+G.Ih, 1, G.D, G.Ih, dv)- && coli_cuda_tensor_upload(&td, w+2*mb, sc+2*G.Ih, 1, G.Ih, G.D, dv);- } else if(G.egs){- ok = coli_cuda_tensor_upload_g(&tg, w, sc, 4, G.D, G.Ih, dv, G.egs)- && coli_cuda_tensor_upload_g(&tu, w+mb, sc+G.sc_gu, 4, G.D, G.Ih, dv, G.egs)- && coli_cuda_tensor_upload_g(&td, w+2*mb, sc+2*G.sc_gu, 4, G.Ih, G.D, dv, G.egs);- } else {- ok = coli_cuda_tensor_upload(&tg, w, sc, 2, G.D, G.Ih, dv)- && coli_cuda_tensor_upload(&tu, w+mb, sc+G.Ih, 2, G.D, G.Ih, dv)- && coli_cuda_tensor_upload(&td, w+2*mb, sc+2*G.Ih, 2, G.Ih, G.D, dv);+ if(!inplace){+ if(G.wfmt==8){+ /* e4m3 bytes as they came from the checkpoint, block scales+ * [ceil(O/128), ceil(I/128)] per matrix -- the layout #817's+ * kernels and tensor_upload(fmt=8) already agree on */+ ok = coli_cuda_tensor_upload(&tg, w, sg, 8, G.D, G.Ih, dv)+ && coli_cuda_tensor_upload(&tu, w+mb, su, 8, G.D, G.Ih, dv)+ && coli_cuda_tensor_upload(&td, w+2*mb, sd, 8, G.Ih, G.D, dv);+ } else if(G.wfmt==1){+ /* int8, scale per riga: qt_init ha gia' rifiutato il caso raggruppato,+ * che questo formato non sa esprimere. */+ ok = coli_cuda_tensor_upload(&tg, w, sg, 1, G.D, G.Ih, dv)+ && coli_cuda_tensor_upload(&tu, w+mb, su, 1, G.D, G.Ih, dv)+ && coli_cuda_tensor_upload(&td, w+2*mb, sd, 1, G.Ih, G.D, dv);+ } else if(G.egs){+ ok = coli_cuda_tensor_upload_g(&tg, w, sg, 4, G.D, G.Ih, dv, G.egs)+ && coli_cuda_tensor_upload_g(&tu, w+mb, su, 4, G.D, G.Ih, dv, G.egs)+ && coli_cuda_tensor_upload_g(&td, w+2*mb, sd, 4, G.Ih, G.D, dv, G.egs);+ } else {+ ok = coli_cuda_tensor_upload(&tg, w, sg, 2, G.D, G.Ih, dv)+ && coli_cuda_tensor_upload(&tu, w+mb, su, 2, G.D, G.Ih, dv)+ && coli_cuda_tensor_upload(&td, w+2*mb, sd, 2, G.Ih, G.D, dv);+ }}free(w); free(sc);if(!ok){@@ -226,7 +248,7 @@ static void *uploader(void *arg){}pthread_mutex_lock(&G.mx);QSlot *s=qs(layer,eid);- if(ok){ s->tg=tg; s->tu=tu; s->td=td; s->resident=1; G.uploads++; }+ if(ok){ s->tg=tg; s->tu=tu; s->td=td; s->resident=1; G.uploads++; if(inplace) G.inplace++; }else { int hd=home2(layer,eid); G.used[hd]-=G.exp_bytes;G.budget[hd]=G.used[hd]; /* device genuinely full: stop trying */ }s->queued=0;@@ -1204,11 +1226,105 @@ void qt_fill_wait(void){/* Adaptive swap check (every 16 ticks = tokens): per device, coldest resident* vs hottest non-resident. Decay every 1024 ticks so an old workload cannot* permanently own the tier; admission uses the shared tier.h contract. */+/* G.mx held: remember an evicted slot for qt_evicted_take. */+static void evict_note_locked(int gi){+ if(!G.ev){ G.ev=malloc(QT_EV_CAP*sizeof(int)); if(!G.ev) return; G.ev_h=0; G.ev_n=0; }+ if(G.ev_n==QT_EV_CAP) return; /* full: the miss path rebuilds lazily */+ G.ev[(G.ev_h+G.ev_n)%QT_EV_CAP]=gi; G.ev_n++;+}+int qt_evicted_take(int *layers,int *eids,int max){+ if(!G.on||max<=0) return 0;+ pthread_mutex_lock(&G.mx);+ int n=0;+ while(n<max && G.ev_n>0){+ int gi=G.ev[G.ev_h]; G.ev_h=(G.ev_h+1)%QT_EV_CAP; G.ev_n--;+ layers[n]=gi/G.ne; eids[n]=gi%G.ne; n++;+ }+ pthread_mutex_unlock(&G.mx);+ return n;+}++/* G.mx held: start up to `max` pending re-plan swaps, as many as the queue+ * takes. Flags are re-checked -- the LFRU tick or an earlier drain may have+ * moved a slot since the plan was made. */+static void replan_drain_locked(int max){+ while(max-- > 0 && G.rp_i<G.rp_n && G.qn<QT_QCAP && !G.th_stop){+ int ci=G.rp_c[G.rp_i], vi=G.rp_v[G.rp_i]; G.rp_i++;+ QSlot *cs=&G.slot[ci], *vs=&G.slot[vi];+ if(cs->resident||cs->queued||!cs->g4) continue;+ if(!vs->resident||vs->queued) continue;+ vs->resident=0; /* CPU fallback from now on */+ if(enqueue_locked(ci/G.ne,ci%G.ne,vi/G.ne,vi%G.ne,0)){ G.swaps++; G.rp_done++; evict_note_locked(vi); }+ else { vs->resident=1; G.rp_i--; break; } /* no staging memory: retry later */+ }+ if(G.rp_i==G.rp_n){ free(G.rp_c); free(G.rp_v); G.rp_c=G.rp_v=NULL; G.rp_n=G.rp_i=0; }+}+#define QT_REPLAN_PER_TICK 4 /* a decode tick starts at most this many pending swaps: each swap is a+ * cudaFree + cudaMalloc, which synchronise the device under the async groups */++static const uint32_t *g_rp_cnt; static int g_rp_base; /* counts indexed by slot - base */+static int cmp_rp_cand(const void *a,const void *b){ /* count desc */+ uint32_t fa=g_rp_cnt[*(const int*)a-g_rp_base], fb=g_rp_cnt[*(const int*)b-g_rp_base];+ return fa<fb ? 1 : fa>fb ? -1 : 0;+}+static int cmp_rp_vict(const void *a,const void *b){ /* count asc, heat asc */+ int ia=*(const int*)a, ib=*(const int*)b;+ uint32_t fa=g_rp_cnt[ia-g_rp_base], fb=g_rp_cnt[ib-g_rp_base];+ if(fa!=fb) return fa<fb ? -1 : 1;+ uint32_t ha=G.slot[ia].heat, hb=G.slot[ib].heat;+ return ha<hb ? -1 : ha>hb ? 1 : 0;+}+int qt_replan(int layer,const uint32_t *counts,int max_swaps){+ if(!G.on||!counts||G_fp8_stream||max_swaps<=0||layer>=G.nl) return 0;+ size_t lo = layer>=0 ? (size_t)layer*G.ne : 0, hi = layer>=0 ? lo+G.ne : (size_t)G.nl*G.ne;+ size_t n=hi-lo;+ int *cand=malloc(n*sizeof(int)), *vict=malloc(n*sizeof(int));+ if(!cand||!vict){ free(cand); free(vict); return 0; }+ pthread_mutex_lock(&G.mx);+ g_rp_cnt=counts; g_rp_base=(int)lo;+ int np=0, seen=0;+ int *pc=realloc(G.rp_c,(size_t)(G.rp_n+(int)n)*sizeof(int)), *pv=realloc(G.rp_v,(size_t)(G.rp_n+(int)n)*sizeof(int));+ if(!pc||!pv){ free(pc?pc:G.rp_c); free(pv?pv:G.rp_v); G.rp_c=G.rp_v=NULL; G.rp_n=G.rp_i=0;+ pthread_mutex_unlock(&G.mx); free(cand); free(vict); return 0; }+ G.rp_c=pc; G.rp_v=pv;+ for(int di=0;di<G.ndev && np<max_swaps;di++){+ int nc=0, nv=0;+ for(size_t i=lo;i<hi;i++){+ QSlot *s=&G.slot[i]; int e=(int)(i%G.ne);+ if(home(e)!=di || s->queued) continue;+ if(s->resident) vict[nv++]=(int)i;+ else if(s->g4 && counts[i-lo]>0) cand[nc++]=(int)i;+ }+ seen+=nc;+ qsort(cand,nc,sizeof(int),cmp_rp_cand);+ qsort(vict,nv,sizeof(int),cmp_rp_vict);+ for(int k=0;k<nc && k<nv && np<max_swaps;k++){+ if(counts[cand[k]-lo]<=counts[vict[k]-lo]) break; /* the rest is not worth a swap */+ G.rp_c[G.rp_n+np]=cand[k]; G.rp_v[G.rp_n+np]=vict[k]; np++;+ }+ }+ G.rp_n+=np; G.rp_planned+=np;+ replan_drain_locked(QT_QCAP); /* start what the queue takes */+ pthread_mutex_unlock(&G.mx);+ free(cand); free(vict);+ if(layer<0) fprintf(stderr,"[qtier] replan: %d swaps planned from %d routed non-residents\n",np,seen);+ return np;+}++void qt_stats_mark(void){+ if(!G.on) return;+ pthread_mutex_lock(&G.mx);+ G.mk_hits=0; for(int i=0;i<G.ndev;i++) G.mk_hits+=G.hits[i];+ G.mk_miss=G.miss; G.mk_on=1;+ pthread_mutex_unlock(&G.mx);+}+static void qt_lfru_tick_locked(void){size_t n=(size_t)G.nl*G.ne;G.tick++;if(!(G.tick%1024))for(size_t i=0;i<n;i++) G.slot[i].heat=tier_decay_value(G.slot[i].heat);+ replan_drain_locked(QT_REPLAN_PER_TICK); /* pending re-plan pairs, a few per token */if(G.tick%16) return;for(int di=0;di<G.ndev;di++){int cold=-1, hot=-1; uint32_t ch=0, hh=0;@@ -1223,7 +1339,7 @@ static void qt_lfru_tick_locked(void){if(!tier_should_promote(hh,ch)) continue;QSlot *v=&G.slot[cold];v->resident=0; /* CPU fallback from now on */- if(enqueue_locked(hot/G.ne,hot%G.ne,cold/G.ne,cold%G.ne,0)) G.swaps++;+ if(enqueue_locked(hot/G.ne,hot%G.ne,cold/G.ne,cold%G.ne,0)){ G.swaps++; evict_note_locked(cold); }else v->resident=1; /* queue full: revert */}}@@ -1320,8 +1436,16 @@ void qt_stats(void){G_trunk_bytes[i]/1073741824.0, G.budget[i]/1073741824.0);}double tot=(double)(hits+G.miss);- fprintf(stderr,"[qtier] VRAM hit rate: %.1f %% | LFRU swaps %llu\n",- tot>0? 100.0*hits/tot : 0.0, (unsigned long long)G.swaps);+ fprintf(stderr,"[qtier] VRAM hit rate: %.1f %% | swaps %llu (%llu in place)\n",+ tot>0? 100.0*hits/tot : 0.0, (unsigned long long)G.swaps, (unsigned long long)G.inplace);+ if(G.mk_on){+ uint64_t dh=hits-G.mk_hits, dm=G.miss-G.mk_miss;+ fprintf(stderr,"[qtier] decode VRAM hit rate: %.1f %% (%llu hits, %llu misses after the prefill)\n",+ dh+dm? 100.0*dh/(dh+dm) : 0.0, (unsigned long long)dh, (unsigned long long)dm);+ }+ if(G.rp_planned)+ fprintf(stderr,"[qtier] replan: %llu swaps planned, %llu done, %d still pending\n",+ (unsigned long long)G.rp_planned, (unsigned long long)G.rp_done, G.rp_n-G.rp_i);{ uint64_t calls=0,ex=0,rows=0; double h2d=0,kms=0,d2h=0;coli_cuda_group_stats(&calls,&ex,&rows,&h2d,&kms,&d2h);if(calls) fprintf(stderr,"[qtier] group_stats: %llu calls, %llu experts | h2d %.0f ms, kernel %.0f ms, d2h %.0f ms\n",@@ -1341,6 +1465,8 @@ static void dense_free_all(void){void qt_shutdown(void){dense_free_all();if(!G.on) return;+ free(G.rp_c); free(G.rp_v); G.rp_c=G.rp_v=NULL; G.rp_n=G.rp_i=0;+ free(G.ev); G.ev=NULL; G.ev_n=0;const char *hf=getenv("HEAT_FILE");if(hf){FILE *f=fopen(hf,"wb");diff --git a/c/qwen36_tier.h b/c/qwen36_tier.hindex cee2a85b..13187397 100644--- a/c/qwen36_tier.h+++ b/c/qwen36_tier.h@@ -153,6 +153,27 @@ void qt_note_block(int layer, int eid,const float *gs, const float *us, const float *ds);void qt_fill_wait(void); /* blocks until every enqueued upload is resident (not merely dequeued) */+/* Re-plan the resident set from the prompt's own routing. counts[] are this+ * prompt's routing counts per expert: for layer >= 0 one layer's ne counts+ * (the engine calls this after each prefill layer's routing, so the swaps of+ * layer L upload while layers L+1.. still compute), for layer < 0 the whole+ * nl*ne table. Residents the prompt never routed to are swapped, budget-+ * neutral, for the prompt's most-routed non-residents of the same layer (or+ * device, for the whole-table form), through the victim-first swap the LFRU+ * tick uses, while the newcomer's count strictly exceeds the victim's. As+ * many swaps as the upload queue takes start at once, the rest are pending+ * and drain on later calls and, a few per token, on the decode ticks: nothing+ * blocks. Returns the pairs planned by this call. */+int qt_replan(int layer, const uint32_t *counts, int max_swaps);+/* Experts the tier evicted since the last call (LFRU and re-plan swaps), as+ * (layer, eid) pairs, oldest first; the engine rebuilds their CPU copies ahead+ * of the miss path. Bounded ring: what overflows is simply not reported, the+ * miss path rebuilds lazily as before. Returns the count written. */+int qt_evicted_take(int *layers, int *eids, int max);+/* Snapshot the hit/miss counters; qt_stats then also reports the hit rate+ * from this point on (the engine marks the prefill/decode boundary). */+void qt_stats_mark(void);+/* One telemetry block on stderr: residency, hits/misses, uploads per device. */void qt_stats(void);@@ -191,6 +212,9 @@ static inline void qt_note_planned(int a,int b,const uint8_t*c,const uint8_t*d,cstatic inline int qt_fill_next(int*a,int*b){(void)a;(void)b;return 0;}static inline void qt_note_block(int a,int b,const uint8_t*c,const uint8_t*d,const uint8_t*e,const float*f,const float*g,const float*h){(void)a;(void)b;(void)c;(void)d;(void)e;(void)f;(void)g;(void)h;}static inline void qt_fill_wait(void){}+static inline int qt_replan(int l,const uint32_t*a,int b){(void)l;(void)a;(void)b;return 0;}+static inline int qt_evicted_take(int*a,int*b,int c){(void)a;(void)b;(void)c;return 0;}+static inline void qt_stats_mark(void){}static inline void qt_stats(void){}#endif /* COLI_CUDA */