Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
Merge dev into dual-ssd-mirror: resolve #298/#192-era conflicts
JustVugg87cd0aafed17
diff --cc c/Makefileindex bb49499e,06fe176e..4ad515f2--- a/c/Makefile+++ b/c/Makefile@@@ -166,7 -171,7 +171,7 @@@ elsPYTHON ?= python3endifCUDA_OBJ =- TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_mirror$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE)-TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_tok_o200k$(EXE) tests/test_kv_alloc$(EXE) tests/test_int3$(EXE) tests/test_int3_load$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) tests/test_pipe_block$(EXE)++TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_mirror$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_tok_o200k$(EXE) tests/test_kv_alloc$(EXE) tests/test_int3$(EXE) tests/test_int3_load$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) tests/test_pipe_block$(EXE)ifneq (,$(LINUX))TEST_BINS += tests/test_uring$(EXE)endifdiff --cc c/colibri.cindex e5e08f3d,a94f7951..4a4b2e06--- a/c/colibri.c+++ b/c/colibri.c@@@ -1215,42 -1472,55 +1527,58 @@@ static int expert_load_impl(Model *m, iint contig = tw[ord[0]]->fd==tw[ord[1]]->fd && tw[ord[1]]->fd==tw[ord[2]]->fd&& tw[ord[0]]->off+tw[ord[0]]->nbytes==tw[ord[1]]->off&& tw[ord[1]]->off+tw[ord[1]]->nbytes==tw[ord[2]]->off;- int64_t pos[3]; int done=0;+ int64_t pos[3]; int done=0, dc_direct=0;if(contig){int64_t off0=tw[ord[0]]->off;- int dfd = g_direct ? st_direct_fd(&m->S, tw[ord[0]]->fd) : -1;+ int dfd = g_direct ? st_direct_fd_rep(&m->S, tw[ord[0]]->fd, rep) : -1;if(dfd>=0){ /* O_DIRECT: offset/len allineati a 4K */int64_t base=off0 & ~4095LL, need=(off0-base)+wtot;int64_t len=(need+4095)&~4095LL;ssize_t r=pread(dfd, s->slab, len, base);if(r>=need){pos[ord[0]]=off0-base; pos[ord[1]]=pos[ord[0]]+tw[ord[0]]->nbytes;- pos[ord[2]]=pos[ord[1]]+tw[ord[1]]->nbytes; done=1;+ pos[ord[2]]=pos[ord[1]]+tw[ord[1]]->nbytes; done=1; dc_direct=1;+ atomic_fetch_add_explicit(&g_mir_bytes[rep],(int64_t)r,memory_order_relaxed);+ atomic_fetch_add_explicit(&g_mir_nread[rep],1,memory_order_relaxed);}}if(!done){ /* fallback bufferizzato */- if(mir_pread(&m->S, tw[ord[0]]->fd, rep, s->slab, wtot, off0, "pread expert")){ if(fatal) exit(1); return -1; }- if(pread_full(tw[ord[0]]->fd, s->slab, wtot, off0, "pread expert")){ if(fatal) exit(1);++ if(mir_pread(&m->S, tw[ord[0]]->fd, rep, s->slab, wtot, off0, "pread expert")){ if(fatal) exit(1);+ if(dc_on) dc_wall_exit(dc_cls,now_s()); /* pair the enter on the non-fatal unwind */+ return -1; }pos[ord[0]]=0; pos[ord[1]]=tw[ord[0]]->nbytes; pos[ord[2]]=tw[ord[0]]->nbytes+tw[ord[1]]->nbytes; done=1;}}if(!done){ /* non contigui: 3 pread bufferizzate */int64_t o=0;for(int a=0;a<3;a++){ int k=ord[a];- if(mir_pread(&m->S, tw[k]->fd, rep, s->slab+o, tw[k]->nbytes, tw[k]->off, "pread expert")){ if(fatal) exit(1); return -1; }- if(pread_full(tw[k]->fd, s->slab+o, tw[k]->nbytes, tw[k]->off, "pread expert")){ if(fatal) exit(1);++ if(mir_pread(&m->S, tw[k]->fd, rep, s->slab+o, tw[k]->nbytes, tw[k]->off, "pread expert")){ if(fatal) exit(1);+ if(dc_on) dc_wall_exit(dc_cls,now_s()); /* pair the enter on the non-fatal unwind */+ return -1; }pos[k]=o; o+=tw[k]->nbytes; }}float *fp[3]; int64_t fo=0; /* scale (piccole) */for(int k=0;k<3;k++){- if(mir_pread(&m->S, tq[k]->fd, rep, (char*)(s->fslab+fo), tq[k]->nbytes, tq[k]->off, "pread qs")){ if(fatal) exit(1); return -1; }- if(pread_full(tq[k]->fd, (char*)(s->fslab+fo), tq[k]->nbytes, tq[k]->off, "pread qs")){ if(fatal) exit(1);++ if(mir_pread(&m->S, tq[k]->fd, rep, (char*)(s->fslab+fo), tq[k]->nbytes, tq[k]->off, "pread qs")){ if(fatal) exit(1);+ if(dc_on) dc_wall_exit(dc_cls,now_s()); /* pair the enter on the non-fatal unwind */+ return -1; }fp[k]=s->fslab+fo; fo+=tq[k]->nbytes/4; }atomic_fetch_add_explicit(&g_prof_io,wtot+fo*4,memory_order_relaxed);+ if(dc_on){ /* DISK-CLASS accounting, see dc_needed() */+ double dc_t1=now_s(); /* one clock read for thread-ns AND the wall exit */+ int64_t bytes=wtot+fo*4;+ atomic_fetch_add_explicit(&g_dc_n[dc_cls],1,memory_order_relaxed);+ atomic_fetch_add_explicit(&g_dc_bytes[dc_cls],bytes,memory_order_relaxed);+ atomic_fetch_add_explicit(&g_dc_ns[dc_cls],(int64_t)((dc_t1-dc_t0)*1e9),memory_order_relaxed);+ dc_wall_exit(dc_cls,dc_t1);+ if(dc_direct) /* which fd ACTUALLY served this class */+ atomic_fetch_add_explicit(&g_dc_direct_n[dc_cls],1,memory_order_relaxed);+ }if(g_drop){ /* scarta subito le pagine: evita che la page- * cache in pressione strangoli il throughput */- posix_fadvise(tw[ord[0]]->fd, tw[ord[0]]->off, wtot, POSIX_FADV_DONTNEED);- for(int k=0;k<3;k++) posix_fadvise(tq[k]->fd, tq[k]->off, tq[k]->nbytes, POSIX_FADV_DONTNEED);+ * cache in pressione strangoli il throughput.+ * The drop targets the fd of the replica READ. */+ posix_fadvise(rep_bfd(&m->S,tw[ord[0]]->fd,rep), tw[ord[0]]->off, wtot, POSIX_FADV_DONTNEED);+ for(int k=0;k<3;k++) posix_fadvise(rep_bfd(&m->S,tq[k]->fd,rep), tq[k]->off, tq[k]->nbytes, POSIX_FADV_DONTNEED);}QT *qt[3]={&s->g,&s->u,&s->d}; int OO[3]={I,I,D}, II[3]={D,D,I};for(int k=0;k<3;k++){@@@ -1599,14 -1916,22 +1974,24 @@@ static void expert_host_ensure(Model *m#endif/* prefetch asincrono dei pesi di un expert (e delle sue scale .qs): avvia il readahead- * cosi' le letture sincrone successive trovano la page-cache calda. The readahead- * targets the SAME replica that will serve the pread (expert_route is deterministic). */+ * cosi' le letture sincrone successive trovano la page-cache calda.+ * Sotto g_direct i PESI vengono letti con O_DIRECT (bypassa la page-cache, vedi+ * expert_load): il WILLNEED su di essi scalda pagine che la lettura di domanda non+ * consuma -> readahead sprecato sul disco, la risorsa piu' scarsa nello streaming.+ * Le scale .qs restano SEMPRE bufferizzate (pread sul fd normale), quindi il loro+ * WILLNEED resta utile anche con DIRECT=1. fadvise e' solo consultivo: saltarlo non+ * cambia mai l'output (bit-identico), riduce solo I/O sprecato.++ * The readahead targets the SAME replica that will serve the pread (expert_route++ * is deterministic).+ * EN: under O_DIRECT the weights bypass the page cache, so their WILLNEED is wasted;+ * the .qs scales are always buffered, so keep theirs. Advisory hint -> output-preserving. */static void expert_prefetch(Model *m, int layer, int eid){- char nm[300];+ char nm[300]; int rep=expert_route(layer,eid);const char *suf[3]={"gate_proj.weight","up_proj.weight","down_proj.weight"};for(int k=0;k<3;k++){- snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.%s",layer,eid,suf[k]); st_prefetch_rep(&m->S,nm,rep);+ snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.%s",layer,eid,suf[k]);- if(!g_direct) st_prefetch(&m->S,nm);- char qs[320]; snprintf(qs,sizeof(qs),"%s.qs",nm); st_prefetch(&m->S,qs);++ if(!g_direct) st_prefetch_rep(&m->S,nm,rep);+ char qs[320]; snprintf(qs,sizeof(qs),"%s.qs",nm); st_prefetch_rep(&m->S,qs,rep);}}diff --cc c/tests/test_st_preadindex 00000000,00000000..8cf27103new file mode 100644Binary files differ