Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
telemetry: split expert hits into pin-tier vs LRU ecache (#336)
JustVugg679c0742b2ec
diff --git a/c/glm.c b/c/glm.cindex 2ee12ed5..1446606b 100644--- a/c/glm.c+++ b/c/glm.c@@ -196,6 +196,7 @@ typedef struct {uint64_t mtp_prop, mtp_acc; /* statistica acceptance */int **eroute; int *enr; /* metodo C: routing dell'ULTIMO token per layer */uint64_t eclock, hits, miss, ereq;+ uint64_t hit_pin, hit_ecache; /* split di hits per tier (#336): pin vs LRU ecache */uint64_t gpu_expert_calls; int gpu_expert_count; int64_t gpu_expert_bytes;uint64_t n_fw, n_emit; /* metodo E: forward di decode / token emessi */uint64_t route_slots, route_swaps; /* CACHE_ROUTE: slots chosen / substituted vs true top-K */@@ -312,6 +313,7 @@ static void prof_lat(double s){ g_prof_lat[g_prof_nlat++ % PROF_LAT_CAP]=s; }typedef struct {double edisk,ewait,emm,ecpu,egpu,route,p2p,attn,head;int64_t io,cpu_bytes; uint64_t hits,miss,ereq,n_fw,n_emit,nlat,n_p2p,cpu_rows;+ uint64_t hit_pin,hit_ecache;} ProfBase;static void prof_base(Model *m, ProfBase *b){b->edisk=edisk_s(); b->ewait=m->t_ewait; b->emm=m->t_emm;@@ -319,6 +321,7 @@ static void prof_base(Model *m, ProfBase *b){b->attn=m->t_attn; b->head=m->t_head;b->io=atomic_load_explicit(&g_prof_io,memory_order_relaxed);b->hits=m->hits; b->miss=m->miss; b->ereq=m->ereq;+ b->hit_pin=m->hit_pin; b->hit_ecache=m->hit_ecache;b->n_fw=m->n_fw; b->n_emit=m->n_emit; b->nlat=g_prof_nlat; b->n_p2p=m->n_p2p;b->cpu_bytes=m->cpu_expert_bytes;b->cpu_rows=m->cpu_expert_rows;}@@ -3132,9 +3135,9 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intESlot *use[64]; int missk[64]; int qof[64]; int nmiss=0;for(int j=0;j<nb;j++){ int eid=uniq[base+j]; use[j]=NULL; qof[j]=-1;ESlot *P=m->pin[layer];- for(int z=0;z<m->npin[layer];z++) if(P[z].eid==eid){ m->hits++; use[j]=&P[z]; break; }+ for(int z=0;z<m->npin[layer];z++) if(P[z].eid==eid){ m->hits++; m->hit_pin++; use[j]=&P[z]; break; }if(!use[j]){ ESlot *Sl=m->ecache[layer]; int nn=m->ecn[layer];- for(int z=0;z<nn;z++) if(Sl[z].eid==eid){ m->hits++; Sl[z].used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); use[j]=&Sl[z]; break; } }+ for(int z=0;z<nn;z++) if(Sl[z].eid==eid){ m->hits++; m->hit_ecache++; Sl[z].used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); use[j]=&Sl[z]; break; } }if(!use[j]){ qof[j]=nmiss; use[j]=&m->ws[nmiss]; missk[nmiss++]=j; m->miss++;if(g_disk_split){ if(m->ld_ctx==1) m->miss_draft++; else if(m->ld_ctx==2) m->miss_absorb++; } }}@@ -4674,11 +4677,12 @@ static void prof_report(Model *m, const ProfBase *b, double elapsed, int tokens,for(int i=0;i<=c->n_layers;i++){ if(m->npin)pinned+=m->npin[i]; if(m->ecn)lru+=m->ecn[i]; }double io_w=m->t_ewait-b->ewait; /* stall the compute thread felt */double io_svc=edisk_s()-b->edisk; /* read service on the loading threads (overlaps compute) */+ uint64_t dhp=m->hit_pin-b->hit_pin, dhe=m->hit_ecache-b->hit_ecache; /* split #336 */fprintf(f,"[PROF] expert I/O: %.3f GB fetched (%.1f MB/token, %.2f GB/s over the run%s) | "- "hit %.1f%% (%llu hit / %llu load) | %.1f loads/token | %.1fs read service / %.1fs felt wait\n",+ "hit %.1f%% (%llu pin + %llu lru / %llu load) | %.1f loads/token | %.1fs read service / %.1fs felt wait\n",io/1e9, tokens>0?io/1e6/tokens:0.0, io/1e9/elapsed,g_mmap?"; COLI_MMAP=1: page cache may serve part":"",- hitp,(unsigned long long)dh,(unsigned long long)dm, tokens>0?(double)dq/tokens:0.0,+ hitp,(unsigned long long)dhp,(unsigned long long)dhe,(unsigned long long)dm, tokens>0?(double)dq/tokens:0.0,io_svc,io_w);fprintf(f,"[PROF] resident experts: %d pinned (%.1f GB) + %d in LRU (%.1f GB, cap %d/layer)\n",pinned,pinned*eb/1e9,lru,lru*eb/1e9,m->ecap);@@ -4724,7 +4728,7 @@ static void run_replay(Model *m, const int *full, int nfull, int np){if(np<2||nfull<=np){ fprintf(stderr,"REPLAY requires a non-empty prompt and continuation\n"); return; }kv_alloc(m,nfull+2);float *logit=step(m,full,np-1,0); free(logit);- m->hits=m->miss=m->ereq=m->gpu_expert_calls=0;+ m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; m->hit_pin=m->hit_ecache=0;profile_reset(m);ProfBase pb; prof_base(m,&pb);double t0=now_s(); int steps=0;@@ -4774,7 +4778,7 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){}prefill_t=now_s()-prefill_t;printf("PROFILO PREFILL (%.2fs):\n",prefill_t); profile_print(m,prefill_t);- m->hits=m->miss=m->ereq=m->gpu_expert_calls=0;+ m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; m->hit_pin=m->hit_ecache=0;m->n_emit=m->n_fw=0;g_last_repin=0;profile_reset(m);@@ -4787,8 +4791,9 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){double tot=m->hits+m->miss;int nsp=0; for(int i=0;i<c->n_layers;i++) if(m->L[i].sparse) nsp++;printf("\n---\nprefill %d tokens in %.2fs | decode %d tokens in %.2fs (%.2f tok/s) | "- "expert hit rate %.1f%% | RSS %.2f GB",- np,prefill_t,produced,dt,produced/dt,tot?100.0*m->hits/tot:0.0,rss_gb());+ "expert hit rate %.1f%% (pin %.1f%% + lru %.1f%%) | RSS %.2f GB", /* split #336: quale tier serve gli hit */+ np,prefill_t,produced,dt,produced/dt,tot?100.0*m->hits/tot:0.0,+ tot?100.0*m->hit_pin/tot:0.0, tot?100.0*m->hit_ecache/tot:0.0, rss_gb());if(g_cache_route && m->route_slots)printf(" | swap %.1f%% (%llu/%llu)",100.0*m->route_swaps/m->route_slots,@@ -6535,8 +6540,9 @@ int main(int argc, char **argv){double tot=m.hits+m.miss;printf("N-gram speculation (DRAFT=%d): %.2f tokens/forward (%llu forwards per %llu tokens)\n",g_draft, m.n_fw?(double)m.n_emit/m.n_fw:1.0, (unsigned long long)m.n_fw, (unsigned long long)m.n_emit);- printf("Expert cache hit rate: %.1f%% (hit=%llu miss=%llu) | RSS: %.2f GB | %.1f tok/s\n",- tot?100.0*m.hits/tot:0.0, (unsigned long long)m.hits, (unsigned long long)m.miss, rss_gb(), n_new/dt);+ printf("Expert cache hit rate: %.1f%% (%llu pin + %llu lru / %llu miss) | RSS: %.2f GB | %.1f tok/s\n",+ tot?100.0*m.hits/tot:0.0, (unsigned long long)m.hit_pin, (unsigned long long)m.hit_ecache,+ (unsigned long long)m.miss, rss_gb(), n_new/dt);profile_print(&m,dt);if(g_prof) prof_report(&m,&pb,dt,n_new,stdout);#ifdef COLI_CUDA