Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Merge dev into dual-ssd-mirror: resolve #298/#192-era conflicts

JustVugg87cd0aafed17
diff --cc c/Makefile
index bb49499e,06fe176e..4ad515f2
--- a/c/Makefile
+++ b/c/Makefile
@@@ -166,7 -171,7 +171,7 @@@ els
PYTHON ?= python3
endif
CUDA_OBJ =
- TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_mirror$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE)
-TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_tok_o200k$(EXE) tests/test_kv_alloc$(EXE) tests/test_int3$(EXE) tests/test_int3_load$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) tests/test_pipe_block$(EXE)
++TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_mirror$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_tok_o200k$(EXE) tests/test_kv_alloc$(EXE) tests/test_int3$(EXE) tests/test_int3_load$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) tests/test_pipe_block$(EXE)
ifneq (,$(LINUX))
TEST_BINS += tests/test_uring$(EXE)
endif
diff --cc c/colibri.c
index e5e08f3d,a94f7951..4a4b2e06
--- a/c/colibri.c
+++ b/c/colibri.c
@@@ -1215,42 -1472,55 +1527,58 @@@ static int expert_load_impl(Model *m, i
int contig = tw[ord[0]]->fd==tw[ord[1]]->fd && tw[ord[1]]->fd==tw[ord[2]]->fd
&& tw[ord[0]]->off+tw[ord[0]]->nbytes==tw[ord[1]]->off
&& tw[ord[1]]->off+tw[ord[1]]->nbytes==tw[ord[2]]->off;
- int64_t pos[3]; int done=0;
+ int64_t pos[3]; int done=0, dc_direct=0;
if(contig){
int64_t off0=tw[ord[0]]->off;
- int dfd = g_direct ? st_direct_fd(&m->S, tw[ord[0]]->fd) : -1;
+ int dfd = g_direct ? st_direct_fd_rep(&m->S, tw[ord[0]]->fd, rep) : -1;
if(dfd>=0){ /* O_DIRECT: offset/len allineati a 4K */
int64_t base=off0 & ~4095LL, need=(off0-base)+wtot;
int64_t len=(need+4095)&~4095LL;
ssize_t r=pread(dfd, s->slab, len, base);
if(r>=need){
pos[ord[0]]=off0-base; pos[ord[1]]=pos[ord[0]]+tw[ord[0]]->nbytes;
- pos[ord[2]]=pos[ord[1]]+tw[ord[1]]->nbytes; done=1;
+ pos[ord[2]]=pos[ord[1]]+tw[ord[1]]->nbytes; done=1; dc_direct=1;
+ atomic_fetch_add_explicit(&g_mir_bytes[rep],(int64_t)r,memory_order_relaxed);
+ atomic_fetch_add_explicit(&g_mir_nread[rep],1,memory_order_relaxed);
}
}
if(!done){ /* fallback bufferizzato */
- if(mir_pread(&m->S, tw[ord[0]]->fd, rep, s->slab, wtot, off0, "pread expert")){ if(fatal) exit(1); return -1; }
- if(pread_full(tw[ord[0]]->fd, s->slab, wtot, off0, "pread expert")){ if(fatal) exit(1);
++ if(mir_pread(&m->S, tw[ord[0]]->fd, rep, s->slab, wtot, off0, "pread expert")){ if(fatal) exit(1);
+ if(dc_on) dc_wall_exit(dc_cls,now_s()); /* pair the enter on the non-fatal unwind */
+ return -1; }
pos[ord[0]]=0; pos[ord[1]]=tw[ord[0]]->nbytes; pos[ord[2]]=tw[ord[0]]->nbytes+tw[ord[1]]->nbytes; done=1;
}
}
if(!done){ /* non contigui: 3 pread bufferizzate */
int64_t o=0;
for(int a=0;a<3;a++){ int k=ord[a];
- if(mir_pread(&m->S, tw[k]->fd, rep, s->slab+o, tw[k]->nbytes, tw[k]->off, "pread expert")){ if(fatal) exit(1); return -1; }
- if(pread_full(tw[k]->fd, s->slab+o, tw[k]->nbytes, tw[k]->off, "pread expert")){ if(fatal) exit(1);
++ if(mir_pread(&m->S, tw[k]->fd, rep, s->slab+o, tw[k]->nbytes, tw[k]->off, "pread expert")){ if(fatal) exit(1);
+ if(dc_on) dc_wall_exit(dc_cls,now_s()); /* pair the enter on the non-fatal unwind */
+ return -1; }
pos[k]=o; o+=tw[k]->nbytes; }
}
float *fp[3]; int64_t fo=0; /* scale (piccole) */
for(int k=0;k<3;k++){
- if(mir_pread(&m->S, tq[k]->fd, rep, (char*)(s->fslab+fo), tq[k]->nbytes, tq[k]->off, "pread qs")){ if(fatal) exit(1); return -1; }
- if(pread_full(tq[k]->fd, (char*)(s->fslab+fo), tq[k]->nbytes, tq[k]->off, "pread qs")){ if(fatal) exit(1);
++ if(mir_pread(&m->S, tq[k]->fd, rep, (char*)(s->fslab+fo), tq[k]->nbytes, tq[k]->off, "pread qs")){ if(fatal) exit(1);
+ if(dc_on) dc_wall_exit(dc_cls,now_s()); /* pair the enter on the non-fatal unwind */
+ return -1; }
fp[k]=s->fslab+fo; fo+=tq[k]->nbytes/4; }
atomic_fetch_add_explicit(&g_prof_io,wtot+fo*4,memory_order_relaxed);
+ if(dc_on){ /* DISK-CLASS accounting, see dc_needed() */
+ double dc_t1=now_s(); /* one clock read for thread-ns AND the wall exit */
+ int64_t bytes=wtot+fo*4;
+ atomic_fetch_add_explicit(&g_dc_n[dc_cls],1,memory_order_relaxed);
+ atomic_fetch_add_explicit(&g_dc_bytes[dc_cls],bytes,memory_order_relaxed);
+ atomic_fetch_add_explicit(&g_dc_ns[dc_cls],(int64_t)((dc_t1-dc_t0)*1e9),memory_order_relaxed);
+ dc_wall_exit(dc_cls,dc_t1);
+ if(dc_direct) /* which fd ACTUALLY served this class */
+ atomic_fetch_add_explicit(&g_dc_direct_n[dc_cls],1,memory_order_relaxed);
+ }
if(g_drop){ /* scarta subito le pagine: evita che la page
- * cache in pressione strangoli il throughput */
- posix_fadvise(tw[ord[0]]->fd, tw[ord[0]]->off, wtot, POSIX_FADV_DONTNEED);
- for(int k=0;k<3;k++) posix_fadvise(tq[k]->fd, tq[k]->off, tq[k]->nbytes, POSIX_FADV_DONTNEED);
+ * cache in pressione strangoli il throughput.
+ * The drop targets the fd of the replica READ. */
+ posix_fadvise(rep_bfd(&m->S,tw[ord[0]]->fd,rep), tw[ord[0]]->off, wtot, POSIX_FADV_DONTNEED);
+ for(int k=0;k<3;k++) posix_fadvise(rep_bfd(&m->S,tq[k]->fd,rep), tq[k]->off, tq[k]->nbytes, POSIX_FADV_DONTNEED);
}
QT *qt[3]={&s->g,&s->u,&s->d}; int OO[3]={I,I,D}, II[3]={D,D,I};
for(int k=0;k<3;k++){
@@@ -1599,14 -1916,22 +1974,24 @@@ static void expert_host_ensure(Model *m
#endif
/* prefetch asincrono dei pesi di un expert (e delle sue scale .qs): avvia il readahead
- * cosi' le letture sincrone successive trovano la page-cache calda. The readahead
- * targets the SAME replica that will serve the pread (expert_route is deterministic). */
+ * cosi' le letture sincrone successive trovano la page-cache calda.
+ * Sotto g_direct i PESI vengono letti con O_DIRECT (bypassa la page-cache, vedi
+ * expert_load): il WILLNEED su di essi scalda pagine che la lettura di domanda non
+ * consuma -> readahead sprecato sul disco, la risorsa piu' scarsa nello streaming.
+ * Le scale .qs restano SEMPRE bufferizzate (pread sul fd normale), quindi il loro
+ * WILLNEED resta utile anche con DIRECT=1. fadvise e' solo consultivo: saltarlo non
+ * cambia mai l'output (bit-identico), riduce solo I/O sprecato.
++ * The readahead targets the SAME replica that will serve the pread (expert_route
++ * is deterministic).
+ * EN: under O_DIRECT the weights bypass the page cache, so their WILLNEED is wasted;
+ * the .qs scales are always buffered, so keep theirs. Advisory hint -> output-preserving. */
static void expert_prefetch(Model *m, int layer, int eid){
- char nm[300];
+ char nm[300]; int rep=expert_route(layer,eid);
const char *suf[3]={"gate_proj.weight","up_proj.weight","down_proj.weight"};
for(int k=0;k<3;k++){
- snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.%s",layer,eid,suf[k]); st_prefetch_rep(&m->S,nm,rep);
+ snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.%s",layer,eid,suf[k]);
- if(!g_direct) st_prefetch(&m->S,nm);
- char qs[320]; snprintf(qs,sizeof(qs),"%s.qs",nm); st_prefetch(&m->S,qs);
++ if(!g_direct) st_prefetch_rep(&m->S,nm,rep);
+ char qs[320]; snprintf(qs,sizeof(qs),"%s.qs",nm); st_prefetch_rep(&m->S,qs,rep);
}
}
diff --cc c/tests/test_st_pread
index 00000000,00000000..8cf27103
new file mode 100644
Binary files differ