Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
port uring implementation
Robert Landers5d1eb142ab1f
diff --git a/README.md b/README.mdindex 86c30090..e34508ce 100644--- a/README.md+++ b/README.md@@ -468,7 +468,7 @@ works against the colibrΓ¬ OpenAI-compatible server (in review, #21) or any othecompatible endpoint. Nothing leaves the endpoint you configure. The terminal`coli chat` remains the first-class interface.-Useful knobs (env or flags): `--temp T` token sampling temperature (default 0.7 + nucleus 0.90 β tuned for int4; 0 = greedy), `--topp 0.7` adaptive expert top-p (30β40% less disk), `--ngen N` max tokens per answer (`:more` in chat continues a truncated one), `--repin N` adapt RAM/VRAM hot experts every N emitted tokens, `AUTOPIN=0` disable the learning cache's auto-pin, `THINK=1` enable GLM-5.2's reasoning block, `DRAFT=n` MTP draft depth, `GRAMMAR=g.gbnf` grammar-forced drafts for constrained JSON/NDJSON output (`GRAMMAR_DRAFT=n` caps the forced span), `TF=1` teacher-forcing validation, `PILOT=1` router-lookahead disk prefetch (experimental β see below), `CAP_RAISE=0` don't auto-grow the expert cache.+Useful knobs (env or flags): `--temp T` token sampling temperature (default 0.7 + nucleus 0.90 β tuned for int4; 0 = greedy), `--topp 0.7` adaptive expert top-p (30β40% less disk), `--ngen N` max tokens per answer (`:more` in chat continues a truncated one), `--repin N` adapt RAM/VRAM hot experts every N emitted tokens, `AUTOPIN=0` disable the learning cache's auto-pin, `THINK=1` enable GLM-5.2's reasoning block, `DRAFT=n` MTP draft depth, `GRAMMAR=g.gbnf` grammar-forced drafts for constrained JSON/NDJSON output (`GRAMMAR_DRAFT=n` caps the forced span), `TF=1` teacher-forcing validation, `PILOT=1` router-lookahead disk prefetch (experimental β see below), `URING=1` Linux-only batched expert I/O (implies `PIPE=1`; also batches `PILOT_REAL`), `CAP_RAISE=0` don't auto-grow the expert cache.### Resource policydiff --git a/c/Makefile b/c/Makefileindex 5a53c248..f47f5f39 100644--- a/c/Makefile+++ b/c/Makefile@@ -15,6 +15,7 @@ MINGW := $(findstring mingw,$(TRIPLET))CYGWIN := $(findstring cygwin,$(TRIPLET))DARWIN := $(findstring darwin,$(TRIPLET))PPC64 := $(findstring powerpc64,$(TRIPLET))+LINUX := $(findstring linux,$(TRIPLET))IS_WIN := $(MINGW)$(CYGWIN)# Fallbacks for the rare toolchain that does not answer -dumpmachine: keep the@@ -148,6 +149,9 @@ PYTHON ?= python3endifCUDA_OBJ =TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE)+ifneq (,$(LINUX))+TEST_BINS += tests/test_uring$(EXE)+endif# Windows CUDA DLL path: host links the loader, NOT cudart.ifneq ($(IS_WIN),)@@ -190,7 +194,7 @@ all: glm$(EXE)# phony 'glm' β 'glm.exe' on Windows (so 'make glm' and 'coli build' work on every platform)glm: glm$(EXE)-glm$(EXE): glm.c st.h json.h tok.h tok_unicode.h compat.h grammar.h $(CUDA_OBJ) $(METAL_OBJ)+glm$(EXE): glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h $(CUDA_OBJ) $(METAL_OBJ)$(CC) $(CFLAGS) glm.c $(CUDA_OBJ) $(METAL_OBJ) -o glm$(EXE) $(LDFLAGS)# Windows runtime loader object: resolves coli_cuda_* from coli_cuda.dll.@@ -258,7 +262,7 @@ tests/test_schema_gbnf$(EXE): tests/test_schema_gbnf.c schema_gbnf.h grammar.h jtests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)-tests/test_idot$(EXE): tests/test_idot.c glm.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h+tests/test_idot$(EXE): tests/test_idot.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c glm.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h@@ -270,6 +274,9 @@ tests/test_i4_acc512$(EXE): tests/test_i4_acc512.ctests/test_compat_direct$(EXE): tests/test_compat_direct.c compat.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_uring$(EXE): tests/test_uring.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+test-c: $(TEST_BINS)$(PYTHON) tools/run_tests.py $(TEST_BINS)@@ -303,4 +310,4 @@ clean:bench: iobench$(EXE)@if [ -n "$(ARGS)" ]; then ./iobench$(EXE) $(ARGS); else echo "built iobench$(EXE) β run: ./iobench$(EXE) <file> <MB> <iters> <threads> <direct 0|1>"; fi-.PHONY: all glm cuda-test cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench\ No newline at end of file+.PHONY: all glm cuda-test cuda-bench cuda-dll portable test-c test-python test check clean install uninstall benchdiff --git a/c/glm.c b/c/glm.cindex fb93e59d..0237869e 100644--- a/c/glm.c+++ b/c/glm.c@@ -37,6 +37,9 @@#include <sys/stat.h> /* fstat per mmap degli shard (COLI_MMAP) */#endif#include "st.h"+#ifdef __linux__+#include "uring.h"+#endif#include "tok.h"#include "tier.h"#include "grammar.h" /* metodo F: draft grammaticali (#48) */@@ -989,8 +992,9 @@ static int g_pilot_two=0; /* PILOT_TWO=1: two-step prefetch β before running L* niente torn read di ecn[]/eid. Il pilota non altera MAI il valore di un expert, solo QUALE* expert e' residente: con un load andato a buon fine l'output resta byte-identico all'OFF. */static pthread_mutex_t g_pilot_mx=PTHREAD_MUTEX_INITIALIZER;+static pthread_cond_t g_pilot_cv=PTHREAD_COND_INITIALIZER;static _Atomic int g_cur_moe_layer=-1; /* massimo layer moe in cui il MAIN e' entrato (per forward) */-static _Atomic int g_pilot_inflight=-1; /* layer che il worker sta REAL-caricando adesso (-1 = idle) */+static int g_pilot_inflight[256]; /* protected by g_pilot_mx; URING can load a layer concurrently */static _Atomic long g_pilot_loads=0; /* load cross-layer VERI completati (banda spesa) */static _Atomic long g_pilot_drops=0; /* predizioni scartate perche' il main possiede gia' il layer *//* sceglie il formato da `bits`: >=16 f32, 5..8 int8, <=4 int4-packed */@@ -1587,6 +1591,189 @@ static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal){s->eid=eid; return 0;}+#ifdef __linux__+/* io_uring expert batches. One owner prepares all reads for a block, submits+ * them in one syscall, and reaps CQEs on demand. The kernel, rather than a set+ * of blocking pthreads, owns the I/O concurrency. */+#define URING_LOAD_MAX 64+#define URING_REQ_MAX 512+typedef struct {+ int load, expect;+} UringRead;+typedef struct {+ Model *m; ESlot *s; int layer,eid,fatal;+ st_tensor *tw[3],*tq[3]; int64_t pos[3];+ int pending,done,finalized,error;+} UringLoad;+typedef struct {+ ColiUring ring;+ UringLoad load[URING_LOAD_MAX];+ UringRead req[URING_REQ_MAX];+ int nload,nreq,started;+} UringBatch;+static UringBatch g_ub_pipe, g_ub_pilot;++static int uring_batch_init(UringBatch *b){+ if(b->started) return 0;+ if(coli_uring_init(&b->ring,URING_REQ_MAX)) return -1;+ b->started=1; return 0;+}+static void uring_batch_reset(UringBatch *b){+ b->nload=0; b->nreq=0;+}+static int uring_load_error(UringLoad *l,int err,const char *what){+ l->error=err?err:EIO; l->done=1;+ if(l->fatal){ errno=l->error; perror(what); exit(1); }+ return -1;+}+static int uring_add_read(UringBatch *b,int li,int fd,void *buf,size_t len,+ int64_t off,size_t expect){+ if(b->nreq>=URING_REQ_MAX || expect>INT_MAX){ errno=E2BIG; return -1; }+ int ri=b->nreq++;+ b->req[ri]=(UringRead){li,(int)expect};+ if(coli_uring_prep_read(&b->ring,fd,buf,len,off,(uint64_t)ri+1)) return -1;+ b->load[li].pending++;+ return 0;+}+/* Returns the load index. URING is intentionally a quantized streaming path;+ * unsupported layouts fail instead of silently dropping back to pread. */+static int uring_load_add(UringBatch *b,Model *m,int layer,int eid,ESlot *s,int fatal){+ if(b->nload>=URING_LOAD_MAX){ errno=E2BIG; return -1; }+ int li=b->nload++;+ UringLoad *l=&b->load[li]; memset(l,0,sizeof(*l));+ l->m=m; l->s=s; l->layer=layer; l->eid=eid; l->fatal=fatal;+ char nm[3][288],qn[300]; const char *suf[3]={"gate_proj","up_proj","down_proj"};+ for(int k=0;k<3;k++) snprintf(nm[k],sizeof(nm[k]),"model.layers.%d.mlp.experts.%d.%s.weight",layer,eid,suf[k]);+ snprintf(qn,sizeof(qn),"%s.qs",nm[0]);+ if(g_mmap || !st_has(&m->S,qn))+ return uring_load_error(l,ENOTSUP,"URING requires quantized expert tensors"),li;+#ifdef COLI_CUDA+ if(s->eid!=eid){ qt_cuda_reset(&s->g); qt_cuda_reset(&s->u); qt_cuda_reset(&s->d); }+#endif+ for(int k=0;k<3;k++){+ l->tw[k]=st_find(&m->S,nm[k]);+ size_t n=strnlen(nm[k],sizeof(nm[k]));+ if(n+3>=sizeof(qn)) return uring_load_error(l,ENAMETOOLONG,"io_uring expert metadata"),li;+ memcpy(qn,nm[k],n); memcpy(qn+n,".qs",4); l->tq[k]=st_find(&m->S,qn);+ if(!l->tw[k]||!l->tq[k]) return uring_load_error(l,ENOENT,"io_uring expert metadata"),li;+ }+ int64_t wtot=l->tw[0]->nbytes+l->tw[1]->nbytes+l->tw[2]->nbytes;+ int64_t ftot=(l->tq[0]->nbytes+l->tq[1]->nbytes+l->tq[2]->nbytes)/4;+ if(wtot<=0 || ftot<=0) return uring_load_error(l,EINVAL,"io_uring expert size"),li;+ if(!s->slab || wtot+8192>s->slab_cap){+#ifdef COLI_METAL+ if(s->slab&&g_metal_enabled) coli_metal_unregister(s->slab);+ compat_aligned_free(s->slab);+ size_t need=((size_t)wtot+8192+16383)&~(size_t)16383;+ if(posix_memalign((void**)&s->slab,16384,need)){+ s->slab=NULL; s->slab_cap=0; return uring_load_error(l,ENOMEM,"io_uring expert slab"),li; }+ s->slab_cap=need; if(g_metal_enabled) coli_metal_register(s->slab,need);+#else+ compat_aligned_free(s->slab);+ if(posix_memalign((void**)&s->slab,4096,(size_t)wtot+8192)){+ s->slab=NULL; s->slab_cap=0; return uring_load_error(l,ENOMEM,"io_uring expert slab"),li; }+ s->slab_cap=wtot+8192;+#endif+ }+ if(!s->fslab || ftot>s->fslab_cap){+#ifdef COLI_METAL+ if(s->fslab&&g_metal_enabled) coli_metal_unregister(s->fslab);+ free(s->fslab); size_t fb=(((size_t)ftot*sizeof(float))+16383)&~(size_t)16383;+ if(posix_memalign((void**)&s->fslab,16384,fb)){+ s->fslab=NULL; s->fslab_cap=0; return uring_load_error(l,ENOMEM,"io_uring expert scales"),li; }+ s->fslab_cap=ftot; if(g_metal_enabled) coli_metal_register(s->fslab,fb);+#else+ free(s->fslab); s->fslab=malloc((size_t)ftot*sizeof(float));+ if(!s->fslab){ s->fslab_cap=0; return uring_load_error(l,ENOMEM,"io_uring expert scales"),li; }+ s->fslab_cap=ftot;+#endif+ }+ int ord[3]={0,1,2};+ for(int a=0;a<3;a++) for(int z=a+1;z<3;z++) if(l->tw[ord[z]]->off<l->tw[ord[a]]->off){int t=ord[a];ord[a]=ord[z];ord[z]=t;}+ int contig=l->tw[ord[0]]->fd==l->tw[ord[1]]->fd && l->tw[ord[1]]->fd==l->tw[ord[2]]->fd+ && l->tw[ord[0]]->off+l->tw[ord[0]]->nbytes==l->tw[ord[1]]->off+ && l->tw[ord[1]]->off+l->tw[ord[1]]->nbytes==l->tw[ord[2]]->off;+ if(contig){+ int64_t off0=l->tw[ord[0]]->off;+ int dfd=g_direct?st_direct_fd(&m->S,l->tw[ord[0]]->fd):-1;+ if(dfd>=0){+ int64_t base=off0&~4095LL,need=(off0-base)+wtot,len=(need+4095)&~4095LL;+ l->pos[ord[0]]=off0-base; l->pos[ord[1]]=l->pos[ord[0]]+l->tw[ord[0]]->nbytes;+ l->pos[ord[2]]=l->pos[ord[1]]+l->tw[ord[1]]->nbytes;+ if(uring_add_read(b,li,dfd,s->slab,(size_t)len,base,(size_t)need))+ return uring_load_error(l,errno,"io_uring direct expert read"),li;+ }else{+ l->pos[ord[0]]=0; l->pos[ord[1]]=l->tw[ord[0]]->nbytes;+ l->pos[ord[2]]=l->pos[ord[1]]+l->tw[ord[1]]->nbytes;+ if(uring_add_read(b,li,l->tw[ord[0]]->fd,s->slab,(size_t)wtot,off0,(size_t)wtot))+ return uring_load_error(l,errno,"io_uring expert read"),li;+ }+ }else{+ int64_t o=0;+ for(int a=0;a<3;a++){ int k=ord[a]; l->pos[k]=o;+ if(uring_add_read(b,li,l->tw[k]->fd,s->slab+o,(size_t)l->tw[k]->nbytes,l->tw[k]->off,(size_t)l->tw[k]->nbytes))+ return uring_load_error(l,errno,"io_uring expert read"),li;+ o+=l->tw[k]->nbytes;+ }+ }+ int64_t fo=0;+ for(int k=0;k<3;k++){+ if(uring_add_read(b,li,l->tq[k]->fd,s->fslab+fo,(size_t)l->tq[k]->nbytes,l->tq[k]->off,(size_t)l->tq[k]->nbytes))+ return uring_load_error(l,errno,"io_uring expert scale read"),li;+ fo+=l->tq[k]->nbytes/4;+ }+ return li;+}+static void uring_reap(UringBatch *b){+ struct io_uring_cqe cqe;+ while(coli_uring_peek(&b->ring,&cqe)){+ if(!cqe.user_data || cqe.user_data>(uint64_t)b->nreq) continue;+ UringRead *r=&b->req[cqe.user_data-1]; UringLoad *l=&b->load[r->load];+ if(cqe.res<r->expect && !l->error) l->error=cqe.res<0?-cqe.res:EIO;+ if(l->pending>0) l->pending--;+ if(l->pending==0) l->done=1;+ }+}+static int uring_submit_batch(UringBatch *b){+ if(coli_uring_enter(&b->ring,0)<0) return -1;+ uring_reap(b); return 0;+}+static int uring_wait_load(UringBatch *b,int li){+ UringLoad *l=&b->load[li];+ while(!l->done){+ uring_reap(b); if(l->done) break;+ if(coli_uring_enter(&b->ring,1)<0) return uring_load_error(l,errno,"io_uring wait");+ }+ return l->error?-1:0;+}+static int uring_finalize_load(UringBatch *b,int li,int publish_eid){+ UringLoad *l=&b->load[li]; ESlot *s=l->s;+ if(l->finalized) return 0;+ if(uring_wait_load(b,li)<0){ errno=l->error; if(l->fatal){perror("io_uring expert completion");exit(1);} return -1; }+ if(g_drop){+ int ord0=0; for(int k=1;k<3;k++) if(l->tw[k]->off<l->tw[ord0]->off) ord0=k;+ int64_t wtot=l->tw[0]->nbytes+l->tw[1]->nbytes+l->tw[2]->nbytes;+ posix_fadvise(l->tw[ord0]->fd,l->tw[ord0]->off,wtot,POSIX_FADV_DONTNEED);+ for(int k=0;k<3;k++) posix_fadvise(l->tq[k]->fd,l->tq[k]->off,l->tq[k]->nbytes,POSIX_FADV_DONTNEED);+ }+ Cfg *c=&l->m->c; int I=c->moe_inter,D=c->hidden; float *fp[3]; int64_t fo=0;+ QT *qt[3]={&s->g,&s->u,&s->d}; int OO[3]={I,I,D},II[3]={D,D,I};+ for(int k=0;k<3;k++){+ fp[k]=s->fslab+fo; fo+=l->tq[k]->nbytes/4;+ int64_t nb=l->tw[k]->nbytes;+ int fmt=(nb==(int64_t)OO[k]*II[k])?1:(nb==(int64_t)OO[k]*((II[k]+1)/2))?2:3;+ qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->qf=NULL;+ qt[k]->q8=(int8_t*)(s->slab+l->pos[k]); qt[k]->q4=s->slab+l->pos[k]; qt[k]->s=fp[k];+ }+ if(publish_eid) s->eid=l->eid;+ l->finalized=1; return 0;+}+static int uring_wait_all(UringBatch *b){+ for(int i=0;i<b->nload;i++) if(uring_wait_load(b,i)<0) return -1;+ return 0;+}+#endif+/* ============================ PIPE: load β matmul ============================* Overlap NVMe expert-weight loads with expert matmul. A small persistent pool* of I/O worker pthreads runs the misses' pread (expert_load) into distinct@@ -1616,6 +1803,7 @@ static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal){* byte-identically. */static int g_pipe=0; /* PIPE=1: async expert-load pipeline (default OFF) */static int g_pipe_nw=8; /* PIPE_WORKERS=n: I/O worker threads (disk-parallel reads) */+static int g_uring=0; /* URING=1: Linux io_uring load/completion backend; implies PIPE */typedef struct {_Atomic uint64_t cur; /* (gen<<8)|index; gen main-only, index 0..njobs (β€64) */_Atomic int njobs; /* current batch job count */@@ -1655,6 +1843,12 @@ static void *pipe_worker(void *arg){}static void pipe_init(Model *m){if(g_pp.started) return;+#ifdef __linux__+ if(g_uring){+ if(uring_batch_init(&g_ub_pipe)){ perror("URING=1 io_uring_setup"); exit(1); }+ g_pp.m=m; g_pp.started=1; return;+ }+#endifg_pp.m=m; g_pp.nw=g_pipe_nw; if(g_pp.nw>16) g_pp.nw=16; if(g_pp.nw<1) g_pp.nw=1;atomic_store(&g_pp.cur,0); atomic_store(&g_pp.njobs,0);pthread_mutex_init(&g_pp.mx,NULL); pthread_cond_init(&g_pp.cv,NULL);@@ -1665,6 +1859,17 @@ static void pipe_init(Model *m){* Order is load-bearing: write all batch state RELAXED, then RELEASE-store cur to* publish it, then wake parked workers. */static void pipe_dispatch(Model *m,int layer,const int *eids,int njobs){+#ifdef __linux__+ if(g_uring){+ uring_batch_reset(&g_ub_pipe);+ for(int q=0;q<njobs;q++){+ int li=uring_load_add(&g_ub_pipe,m,layer,eids[q],&m->ws[q],1);+ if(li!=q){ fprintf(stderr,"URING: expert batch overflow\n"); exit(1); }+ }+ if(uring_submit_batch(&g_ub_pipe)){ perror("URING: submit"); exit(1); }+ return;+ }+#endifg_pp.m=m;atomic_store_explicit(&g_pp.njobs,njobs,memory_order_relaxed);atomic_store_explicit(&g_pp.layer,layer,memory_order_relaxed);@@ -1675,6 +1880,12 @@ static void pipe_dispatch(Model *m,int layer,const int *eids,int njobs){pthread_mutex_lock(&g_pp.mx); pthread_cond_broadcast(&g_pp.cv); pthread_mutex_unlock(&g_pp.mx);}static inline void pipe_wait(int q){+#ifdef __linux__+ if(g_uring){+ if(uring_finalize_load(&g_ub_pipe,q,1)){ perror("URING: expert load"); exit(1); }+ return;+ }+#endifwhile(!atomic_load_explicit(&g_pp.ready[q],memory_order_acquire)) sched_yield();}@@ -2217,14 +2428,11 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int* l'eventuale load-pilota in volo sullo stesso layer (dopodiche' il* worker droppa ogni nuovo load <= layer -> ecache[layer] e' stabile* per tutto il resolve/matmul/promozione qui sotto). */- for(;;){- pthread_mutex_lock(&g_pilot_mx);- atomic_store_explicit(&g_cur_moe_layer,layer,memory_order_release);- int inf=atomic_load_explicit(&g_pilot_inflight,memory_order_acquire);- pthread_mutex_unlock(&g_pilot_mx);- if(inf!=layer) break;- sched_yield();- }+ pthread_mutex_lock(&g_pilot_mx);+ atomic_store_explicit(&g_cur_moe_layer,layer,memory_order_release);+ while(layer>=0 && layer<256 && g_pilot_inflight[layer]>0)+ pthread_cond_wait(&g_pilot_cv,&g_pilot_mx);+ pthread_mutex_unlock(&g_pilot_mx);}Cfg *c=&m->c; int D=c->hidden, E=c->n_experts, K=c->topk, I=c->moe_inter;float *choice=falloc(E);@@ -2800,7 +3008,7 @@ static void pilot_realload(Model *m, int layer, int eid){else { int lru=0; for(int z=1;z<nn;z++) if(Sl[z].used<Sl[lru].used) lru=z; slot=lru; isnew=0; }ESlot *dst=&Sl[slot];dst->eid=-1; /* nascondi dagli scan-hint mentre carica */- atomic_store_explicit(&g_pilot_inflight,layer,memory_order_release);+ g_pilot_inflight[layer]++;pthread_mutex_unlock(&g_pilot_mx);int rc=expert_load(m,layer,eid,dst,0); /* pread VERO β fuori dal lock, sovrapposto al compute; fatal=0: un errore su una speculazione NON deve uccidere il server */@@ -2813,18 +3021,96 @@ static void pilot_realload(Model *m, int layer, int eid){} else {atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed); /* load fallito: slot resta nascosto (eid=-1), mai pubblicato */}- atomic_store_explicit(&g_pilot_inflight,-1,memory_order_release);+ g_pilot_inflight[layer]--;+ pthread_cond_broadcast(&g_pilot_cv);pthread_mutex_unlock(&g_pilot_mx);if(rc!=0) /* mai swallow silenzioso: logga (una riga) e prosegui */fprintf(stderr,"[PILOT] load speculativo abbandonato: layer %d expert %d (I/O error/short read) β nessun impatto sull'output\n",layer,eid);}+#ifdef __linux__+typedef struct { int layer,eid,li; ESlot *dst; } PilotUringDone;+static void pilot_uring_batch(Model *m){+ PilotUringDone done[URING_LOAD_MAX]; int nd=0;+ uring_batch_reset(&g_ub_pilot);+ unsigned r=__atomic_load_n(&pilot_r,__ATOMIC_ACQUIRE);+ unsigned w=__atomic_load_n(&pilot_w,__ATOMIC_ACQUIRE);+ while(r!=w && nd<URING_LOAD_MAX){+ int layer=pilot_q[r&4095].l,eid=pilot_q[r&4095].e; r++;+ if(layer<0 || layer>=256){ atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed); continue; }+ pthread_mutex_lock(&g_pilot_mx);+ if(layer<=atomic_load_explicit(&g_cur_moe_layer,memory_order_acquire)){+ atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed);+ pthread_mutex_unlock(&g_pilot_mx); continue;+ }+ int found=0; ESlot *P=m->pin[layer];+ for(int z=0;z<m->npin[layer];z++) if(P[z].eid==eid){found=1;break;}+ ESlot *Sl=m->ecache[layer]; int nn=m->ecn[layer];+ for(int z=0;z<nn && !found;z++) if(Sl[z].eid==eid || Sl[z].eid==-(eid+2)) found=1;+ if(found){ pthread_mutex_unlock(&g_pilot_mx); continue; }+ int slot;+ if(nn<m->ecap){ slot=nn; m->ecn[layer]=nn+1; }+ else{+ slot=-1;+ for(int z=0;z<nn;z++){+ if(Sl[z].eid==-1){ slot=z; break; }+ if(Sl[z].eid< -1) continue; /* URING reservation in flight */+ if(slot<0 || Sl[z].used<Sl[slot].used) slot=z;+ }+ }+ if(slot<0){ atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed); pthread_mutex_unlock(&g_pilot_mx); continue; }+ ESlot *dst=&Sl[slot];+ dst->eid=-(eid+2); /* visible reservation; never considered resident/evictable */+ g_pilot_inflight[layer]++;+ pthread_mutex_unlock(&g_pilot_mx);++ int li=uring_load_add(&g_ub_pilot,m,layer,eid,dst,0);+ if(li<0){+ pthread_mutex_lock(&g_pilot_mx); dst->eid=-1; g_pilot_inflight[layer]--;+ pthread_cond_broadcast(&g_pilot_cv); pthread_mutex_unlock(&g_pilot_mx);+ atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed); continue;+ }+ done[nd++]=(PilotUringDone){layer,eid,li,dst};+ }+ __atomic_store_n(&pilot_r,r,__ATOMIC_RELEASE);+ if(!nd) return;+ if(uring_submit_batch(&g_ub_pilot)<0){+ int err=errno;+ for(int i=0;i<g_ub_pilot.nload;i++){+ g_ub_pilot.load[i].error=err; g_ub_pilot.load[i].done=1;+ }+ }+ for(int i=0;i<nd;i++){+ PilotUringDone *d=&done[i];+ int rc=uring_finalize_load(&g_ub_pilot,d->li,0);+ pthread_mutex_lock(&g_pilot_mx);+ if(rc==0){+ d->dst->eid=d->eid;+ d->dst->used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED);+ atomic_fetch_add_explicit(&g_pilot_loads,1,memory_order_relaxed);+ }else{+ d->dst->eid=-1;+ atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed);+ }+ g_pilot_inflight[d->layer]--;+ pthread_cond_broadcast(&g_pilot_cv);+ pthread_mutex_unlock(&g_pilot_mx);+ if(rc) fprintf(stderr,"[PILOT/URING] load speculativo abbandonato: layer %d expert %d: %s\n",+ d->layer,d->eid,strerror(g_ub_pilot.load[d->li].error));+ }+}+#endifstatic void *pilot_worker(void *arg){(void)arg;for(;;){unsigned r=__atomic_load_n(&pilot_r,__ATOMIC_ACQUIRE);unsigned w=__atomic_load_n(&pilot_w,__ATOMIC_ACQUIRE);if(r==w){ usleep(200); continue; }- if(g_pilot_real) pilot_realload(pilot_m, pilot_q[r&4095].l, pilot_q[r&4095].e);+ if(g_pilot_real){+#ifdef __linux__+ if(g_uring){ pilot_uring_batch(pilot_m); continue; }+#endif+ pilot_realload(pilot_m, pilot_q[r&4095].l, pilot_q[r&4095].e);+ }else expert_prefetch(pilot_m, pilot_q[r&4095].l, pilot_q[r&4095].e);__atomic_store_n(&pilot_r,r+1,__ATOMIC_RELEASE);}@@ -2888,7 +3174,8 @@ static void couple_prefetch(Model *m, int layer, const int *idx, int Ke){ESlot *P=m->pin[lt];for(int z=0;z<m->npin[lt] && !found;z++) if(P[z].eid==best) found=1;ESlot *Sl=m->ecache[lt];- for(int z=0;z<m->ecn[lt] && !found;z++) if(Sl[z].eid==best) found=1;+ for(int z=0;z<m->ecn[lt] && !found;z++)+ if(Sl[z].eid==best || Sl[z].eid==-(best+2)) found=1;pthread_mutex_unlock(&g_pilot_mx);if(!found){unsigned w=__atomic_load_n(&pilot_w,__ATOMIC_RELAXED);@@ -2949,7 +3236,8 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S){ESlot *P=m->pin[lnext];for(int z=0;z<m->npin[lnext] && !found;z++) if(P[z].eid==best) found=1;ESlot *Sl=m->ecache[lnext];- for(int z=0;z<m->ecn[lnext] && !found;z++) if(Sl[z].eid==best) found=1;+ for(int z=0;z<m->ecn[lnext] && !found;z++)+ if(Sl[z].eid==best || Sl[z].eid==-(best+2)) found=1;pthread_mutex_unlock(&g_pilot_mx);if(!found){unsigned w=__atomic_load_n(&pilot_w,__ATOMIC_RELAXED);@@ -4971,6 +5259,20 @@ int main(int argc, char **argv){g_pipe = getenv("PIPE")?atoi(getenv("PIPE")):0; /* default OFF: overlap expert load β matmul (byte-identical; reorders I/O). PIPE=1 opts in */g_pipe_nw = getenv("PIPE_WORKERS")?atoi(getenv("PIPE_WORKERS")):8; /* I/O worker threads */if(g_pipe_nw<1) g_pipe_nw=1;+ g_uring = getenv("URING")?atoi(getenv("URING")):0;+ if(g_uring){+#ifdef __linux__+ if(g_mmap){ fprintf(stderr,"URING=1 is incompatible with COLI_MMAP=1\n"); return 2; }+ g_pipe=1;+ if(uring_batch_init(&g_ub_pipe) || (g_pilot_real&&uring_batch_init(&g_ub_pilot))){+ fprintf(stderr,"URING=1: io_uring_setup failed: %s\n",strerror(errno)); return 2;+ }+ fprintf(stderr,"[URING] queued expert I/O active (depth=%d%s)\n",URING_REQ_MAX,+ g_pilot_real?", batched PILOT_REAL":"");+#else+ fprintf(stderr,"URING=1 is supported only on Linux\n"); return 2;+#endif+ }g_direct = getenv("DIRECT")?atoi(getenv("DIRECT")):0;g_idot = getenv("IDOT")?atoi(getenv("IDOT")):1; /* 0 = kernel f32 esatti (A/B) */if(getenv("ROUTE_TRACE")&&*getenv("ROUTE_TRACE")){diff --git a/c/tests/test_uring.c b/c/tests/test_uring.cnew file mode 100644index 00000000..9712f659--- /dev/null+++ b/c/tests/test_uring.c@@ -0,0 +1,114 @@+#define _GNU_SOURCE+#include <errno.h>+#include <fcntl.h>+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <unistd.h>+#define main coli_glm_main_unused+#include "../glm.c"+#undef main++static int fail(const char *s){ fprintf(stderr,"FAIL: %s\n",s); return 1; }++static int test_expert_layout(int fd){+ Model m={0}; ESlot slot={0}; UringBatch batch={0};+ m.c.hidden=4; m.c.moe_inter=3; m.ebits=8;+ m.S.n=6; m.S.cap=6; m.S.t=calloc(6,sizeof(st_tensor));+ if(!m.S.t) return fail("tensor metadata allocation");+ const char *proj[3]={"gate_proj","up_proj","down_proj"};+ int wbytes[3]={12,12,12}, sbytes[3]={12,12,16};+ unsigned char data[76];+ for(int i=0;i<36;i++) data[i]=(unsigned char)(i+1);+ float scales[10]; for(int i=0;i<10;i++) scales[i]=(float)i+0.5f;+ memcpy(data+36,scales,sizeof(scales));+ if(pwrite(fd,data,sizeof(data),0)!=(ssize_t)sizeof(data)){ free(m.S.t); return fail("expert fixture write"); }+ int64_t wo=0,so=36;+ for(int k=0;k<3;k++){+ char name[300];+ snprintf(name,sizeof(name),"model.layers.1.mlp.experts.7.%s.weight",proj[k]);+ m.S.t[k]=(st_tensor){strdup(name),fd,wo,wbytes[k],3,wbytes[k]}; wo+=wbytes[k];+ size_t n=strlen(name); memcpy(name+n,".qs",4);+ m.S.t[3+k]=(st_tensor){strdup(name),fd,so,sbytes[k],2,sbytes[k]/4}; so+=sbytes[k];+ }+ if(uring_batch_init(&batch)){ free(m.S.t); return fail("expert ring init"); }+ uring_batch_reset(&batch);+ int li=uring_load_add(&batch,&m,1,7,&slot,1);+ if(li!=0 || uring_submit_batch(&batch) || uring_finalize_load(&batch,li,1)){+ coli_uring_close(&batch.ring); free(m.S.t); return fail("expert batch load");+ }+ int bad=slot.eid!=7 || slot.g.fmt!=1 || slot.u.fmt!=1 || slot.d.fmt!=1+ || memcmp(slot.g.q8,data,12) || memcmp(slot.u.q8,data+12,12) || memcmp(slot.d.q8,data+24,12)+ || memcmp(slot.g.s,scales,12) || memcmp(slot.u.s,scales+3,12) || memcmp(slot.d.s,scales+6,16);+ coli_uring_close(&batch.ring);+ compat_aligned_free(slot.slab); free(slot.fslab);+ if(bad){ for(int i=0;i<m.S.n;i++) free(m.S.t[i].name); free(m.S.t); return fail("expert tensor views"); }++ m.c.n_experts=8; m.c.n_layers=2; m.ecap=2;+ m.pin=calloc(3,sizeof(ESlot*)); m.npin=calloc(3,sizeof(int));+ m.ecache=calloc(3,sizeof(ESlot*)); m.ecn=calloc(3,sizeof(int));+ m.ecache[1]=calloc(2,sizeof(ESlot));+ if(!m.pin||!m.npin||!m.ecache||!m.ecn||!m.ecache[1])+ return fail("pilot fixture allocation");+ if(uring_batch_init(&g_ub_pilot)) return fail("pilot ring init");+ memset(g_pilot_inflight,0,sizeof(g_pilot_inflight));+ atomic_store(&g_cur_moe_layer,-1); atomic_store(&g_pilot_loads,0); atomic_store(&g_pilot_drops,0);+ pilot_r=0; pilot_w=1; pilot_q[0].l=1; pilot_q[0].e=7;+ pilot_uring_batch(&m);+ bad=m.ecn[1]!=1 || m.ecache[1][0].eid!=7 || g_pilot_inflight[1]!=0+ || atomic_load(&g_pilot_loads)!=1 || atomic_load(&g_pilot_drops)!=0;+ coli_uring_close(&g_ub_pilot.ring);+ compat_aligned_free(m.ecache[1][0].slab); free(m.ecache[1][0].fslab);+ free(m.ecache[1]);+ free(m.pin); free(m.npin); free(m.ecache); free(m.ecn);+ for(int i=0;i<m.S.n;i++) free(m.S.t[i].name);+ free(m.S.t);+ return bad?fail("pilot uring publication"):0;+}++int main(void){+ char path[]="/tmp/coli-uring-XXXXXX";+ int fd=mkstemp(path); if(fd<0) return fail("mkstemp");+ unlink(path);+ enum { N=4, SZ=4096 };+ unsigned char src[N][SZ],dst[N][SZ];+ for(int i=0;i<N;i++){+ memset(src[i],0,sizeof(src[i])); memset(dst[i],0,sizeof(dst[i]));+ for(int j=0;j<SZ;j++) src[i][j]=(unsigned char)(i*37+j);+ if(pwrite(fd,src[i],SZ,(off_t)i*SZ)!=SZ){ close(fd); return fail("pwrite fixture"); }+ }+ ColiUring ring;+ if(coli_uring_init(&ring,8)){+ if(errno==EPERM || errno==ENOSYS || errno==EACCES){+ printf("test_uring: skipped (%s)\n",strerror(errno)); close(fd); return 0;+ }+ close(fd); return fail("io_uring_setup");+ }+ for(int i=0;i<N;i++) if(coli_uring_prep_read(&ring,fd,dst[i],SZ,(off_t)i*SZ,(uint64_t)i+1)){+ coli_uring_close(&ring); close(fd); return fail("prepare read");+ }+ if(coli_uring_enter(&ring,0)<0){ coli_uring_close(&ring); close(fd); return fail("submit"); }+ int seen[N]={0},complete=0;+ while(complete<N){+ struct io_uring_cqe cqe; int reaped=0;+ while(coli_uring_peek(&ring,&cqe)){+ reaped=1;+ if(cqe.user_data<1 || cqe.user_data>N || cqe.res!=SZ){+ coli_uring_close(&ring); close(fd); return fail("bad completion");+ }+ int i=(int)cqe.user_data-1;+ if(seen[i]++){ coli_uring_close(&ring); close(fd); return fail("duplicate completion"); }+ complete++;+ }+ if(!reaped && complete<N && coli_uring_enter(&ring,1)<0){+ coli_uring_close(&ring); close(fd); return fail("completion wait");+ }+ }+ for(int i=0;i<N;i++) if(memcmp(src[i],dst[i],SZ)){+ coli_uring_close(&ring); close(fd); return fail("read data mismatch");+ }+ coli_uring_close(&ring);+ if(ftruncate(fd,0) || test_expert_layout(fd)){ close(fd); return 1; }+ close(fd);+ puts("test_uring: ok"); return 0;+}diff --git a/c/uring.h b/c/uring.hnew file mode 100644index 00000000..bc0d8172--- /dev/null+++ b/c/uring.h@@ -0,0 +1,126 @@+#ifndef COLI_URING_H+#define COLI_URING_H++/* Minimal Linux io_uring reader. Colibri owns the ring from one thread, queues+ * a batch of positioned reads, and reaps CQEs without a userspace spin loop. */+#ifdef __linux__++#include <errno.h>+#include <linux/io_uring.h>+#include <stdint.h>+#include <string.h>+#include <sys/mman.h>+#include <sys/syscall.h>+#include <unistd.h>++#ifndef MAP_POPULATE+#define MAP_POPULATE 0+#endif++typedef struct {+ int fd;+ struct io_uring_params p;+ void *sq_map, *cq_map, *sqes_map;+ size_t sq_map_sz, cq_map_sz, sqes_map_sz;+ unsigned *sq_head, *sq_tail, *sq_mask, *sq_entries, *sq_array;+ unsigned *cq_head, *cq_tail, *cq_mask, *cq_entries;+ struct io_uring_sqe *sqes;+ struct io_uring_cqe *cqes;+} ColiUring;++static inline unsigned coli_uring_load_acquire(unsigned *p){+ return __atomic_load_n(p,__ATOMIC_ACQUIRE);+}+static inline void coli_uring_store_release(unsigned *p,unsigned v){+ __atomic_store_n(p,v,__ATOMIC_RELEASE);+}++static inline void coli_uring_close(ColiUring *r){+ if(!r) return;+ if(r->sqes_map && r->sqes_map!=MAP_FAILED) munmap(r->sqes_map,r->sqes_map_sz);+ if(r->cq_map && r->cq_map!=MAP_FAILED && r->cq_map!=r->sq_map) munmap(r->cq_map,r->cq_map_sz);+ if(r->sq_map && r->sq_map!=MAP_FAILED) munmap(r->sq_map,r->sq_map_sz);+ if(r->fd>=0) close(r->fd);+ memset(r,0,sizeof(*r)); r->fd=-1;+}++static inline int coli_uring_init(ColiUring *r,unsigned entries){+ memset(r,0,sizeof(*r)); r->fd=-1;+ r->fd=(int)syscall(SYS_io_uring_setup,entries,&r->p);+ if(r->fd<0) return -1;++ r->sq_map_sz=r->p.sq_off.array+r->p.sq_entries*sizeof(unsigned);+ r->cq_map_sz=r->p.cq_off.cqes+r->p.cq_entries*sizeof(struct io_uring_cqe);+ if(r->p.features&IORING_FEAT_SINGLE_MMAP){+ size_t n=r->sq_map_sz>r->cq_map_sz?r->sq_map_sz:r->cq_map_sz;+ r->sq_map=mmap(NULL,n,PROT_READ|PROT_WRITE,MAP_SHARED|MAP_POPULATE,r->fd,IORING_OFF_SQ_RING);+ if(r->sq_map==MAP_FAILED){ r->sq_map=NULL; coli_uring_close(r); return -1; }+ r->sq_map_sz=n; r->cq_map=r->sq_map; r->cq_map_sz=n;+ }else{+ r->sq_map=mmap(NULL,r->sq_map_sz,PROT_READ|PROT_WRITE,MAP_SHARED|MAP_POPULATE,r->fd,IORING_OFF_SQ_RING);+ if(r->sq_map==MAP_FAILED){ r->sq_map=NULL; coli_uring_close(r); return -1; }+ r->cq_map=mmap(NULL,r->cq_map_sz,PROT_READ|PROT_WRITE,MAP_SHARED|MAP_POPULATE,r->fd,IORING_OFF_CQ_RING);+ if(r->cq_map==MAP_FAILED){ r->cq_map=NULL; coli_uring_close(r); return -1; }+ }+ r->sqes_map_sz=r->p.sq_entries*sizeof(struct io_uring_sqe);+ r->sqes_map=mmap(NULL,r->sqes_map_sz,PROT_READ|PROT_WRITE,MAP_SHARED|MAP_POPULATE,r->fd,IORING_OFF_SQES);+ if(r->sqes_map==MAP_FAILED){ r->sqes_map=NULL; coli_uring_close(r); return -1; }++ char *sq=(char*)r->sq_map,*cq=(char*)r->cq_map;+ r->sq_head=(unsigned*)(sq+r->p.sq_off.head);+ r->sq_tail=(unsigned*)(sq+r->p.sq_off.tail);+ r->sq_mask=(unsigned*)(sq+r->p.sq_off.ring_mask);+ r->sq_entries=(unsigned*)(sq+r->p.sq_off.ring_entries);+ r->sq_array=(unsigned*)(sq+r->p.sq_off.array);+ r->cq_head=(unsigned*)(cq+r->p.cq_off.head);+ r->cq_tail=(unsigned*)(cq+r->p.cq_off.tail);+ r->cq_mask=(unsigned*)(cq+r->p.cq_off.ring_mask);+ r->cq_entries=(unsigned*)(cq+r->p.cq_off.ring_entries);+ r->sqes=(struct io_uring_sqe*)r->sqes_map;+ r->cqes=(struct io_uring_cqe*)(cq+r->p.cq_off.cqes);+ return 0;+}++static inline int coli_uring_prep_read(ColiUring *r,int fd,void *buf,size_t len,+ int64_t off,uint64_t user_data){+ if(!len || len>UINT32_MAX){ errno=EINVAL; return -1; }+ unsigned head=coli_uring_load_acquire(r->sq_head);+ unsigned tail=__atomic_load_n(r->sq_tail,__ATOMIC_RELAXED);+ if(tail-head>=*r->sq_entries){ errno=EAGAIN; return -1; }+ unsigned idx=tail&*r->sq_mask;+ struct io_uring_sqe *sqe=&r->sqes[idx];+ memset(sqe,0,sizeof(*sqe));+ sqe->opcode=IORING_OP_READ;+ sqe->fd=fd;+ sqe->off=(uint64_t)off;+ sqe->addr=(uint64_t)(uintptr_t)buf;+ sqe->len=(uint32_t)len;+ sqe->user_data=user_data;+ r->sq_array[idx]=idx;+ coli_uring_store_release(r->sq_tail,tail+1);+ return 0;+}++static inline int coli_uring_enter(ColiUring *r,unsigned min_complete){+ for(;;){+ unsigned head=coli_uring_load_acquire(r->sq_head);+ unsigned tail=coli_uring_load_acquire(r->sq_tail);+ unsigned submit=tail-head;+ unsigned flags=min_complete?IORING_ENTER_GETEVENTS:0;+ int n=(int)syscall(SYS_io_uring_enter,r->fd,submit,min_complete,flags,NULL,0);+ if(n>=0) return n;+ if(errno!=EINTR) return -1;+ }+}++static inline int coli_uring_peek(ColiUring *r,struct io_uring_cqe *out){+ unsigned head=__atomic_load_n(r->cq_head,__ATOMIC_RELAXED);+ unsigned tail=coli_uring_load_acquire(r->cq_tail);+ if(head==tail) return 0;+ *out=r->cqes[head&*r->cq_mask];+ coli_uring_store_release(r->cq_head,head+1);+ return 1;+}++#endif /* __linux__ */+#endif /* COLI_URING_H */diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex c1f42eca..d280a92f 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -40,6 +40,7 @@ Format: `VAR` β default β effect.| `COLI_METAL_SPIN` | off | Keep a GPU keep-alive spinner running (reduces dispatch latency; costs power). || `PIPE` | `0` (off) | Overlap expert disk-load with matmul via I/O worker threads. Byte-identical output; reorders I/O. `PIPE=1` opts in. || `PIPE_WORKERS` | `8` | Number of I/O worker threads when `PIPE=1`. Tune to your SSD (fewer avoids over-subscribing cores). |+| `URING` | `0` (off) | Linux-only queued expert I/O. `URING=1` implies `PIPE=1`, replaces blocking loader pthreads and spin waits with batched io_uring SQEs/CQEs, and batches `PILOT_REAL` loads on a separate ring. Fails clearly if the kernel denies io_uring; incompatible with `COLI_MMAP=1`. || `DIRECT` | `0` (off) | Use `O_DIRECT`/unbuffered reads for expert slabs. Helps sustained NVMe; keeps the zero-copy GPU path. || `COLI_NO_OMP_TUNE` | off | **Kill-switch** for the OpenMP hot-thread tuning (`OMP_WAIT_POLICY=active` spin + proc-bind). Set `=1` when the CPU is mostly waiting on the GPU (Metal) so spin doesn't steal the shared power budget. || `MLOCK` | `-1` (auto: on for macOS) | Wire the streamed expert cache into physical RAM (`mlock`) to dodge the memory compressor. `0` off, `1` force. |