Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

cuda: keep async expert slots alive through completion

ZacharyZcR0537e1adf151
diff --git a/c/Makefile b/c/Makefile
index d978bc49..c43922fb 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -941,6 +941,9 @@ tests/test_cap_precedence$(EXE): tests/test_cap_precedence.c colibri.c st.h urin
tests/test_cap_mixed_width$(EXE): tests/test_cap_mixed_width.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+tests/test_eslot_inflight$(EXE): tests/test_eslot_inflight.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
tests/test_ssd_probe$(EXE): tests/test_ssd_probe.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
diff --git a/c/colibri.c b/c/colibri.c
index 219beb3e..d0e6f515 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -360,12 +360,27 @@ typedef struct {
* expert non hanno tutti la stessa taglia (layer MTP int8 = 2x i layer int4). */
typedef struct { int eid; QT g,u,d; uint8_t *slab; float *fslab;
int64_t slab_cap, fslab_cap; uint64_t used;
+ unsigned in_flight; /* async GPU readers borrowing this slot */
/* pin-arena backing (#419): when set, slab/fslab are interior
* slices of a per-layer arena and must never be free()d β€”
* expert_host_release detaches them, expert_host_ensure
* re-attaches. NULL for every individually-allocated slot. */
uint8_t *aslab; float *afslab; } ESlot;
+static void eslot_acquire(ESlot *s){ __atomic_add_fetch(&s->in_flight,1,__ATOMIC_ACQ_REL); }
+static void eslot_release(ESlot *s){
+ unsigned old=__atomic_fetch_sub(&s->in_flight,1,__ATOMIC_ACQ_REL);
+ if(!old){ fprintf(stderr,"[CUDA] ESlot reference underflow\n"); abort(); }
+}
+static int eslot_busy(const ESlot *s){ return __atomic_load_n(&s->in_flight,__ATOMIC_ACQUIRE)!=0; }
+static void eslots_acquire(ESlot **slots,int n){ for(int i=0;i<n;i++) eslot_acquire(slots[i]); }
+static void eslots_release(ESlot **slots,int n){ for(int i=0;i<n;i++) eslot_release(slots[i]); }
+static int eslot_lru_victim(ESlot *slots,int n){
+ int lru=-1;
+ for(int i=0;i<n;i++) if(!eslot_busy(&slots[i])&&(lru<0||slots[i].used<slots[lru].used)) lru=i;
+ return lru;
+}
+
typedef struct {
float **Lc, **Rc, **Ic;
int *kv_start, max_t;
@@ -3058,6 +3073,8 @@ static int g_pres_home=-1; /* home device, -1 = path off *
static const float *g_pres_xsrc; /* nrm_d on the home device */
static float *g_pres_slots; /* [ndev][D] partial slots on home */
static int g_pres_used[COLI_CUDA_MAX_DEVICES], g_pres_nused;
+static ESlot *g_pres_refs[COLI_CUDA_MAX_DEVICES*64];
+static int g_pres_nrefs;
#endif /* ABSORB: -1 auto (decode S<=4), 0 mai, 1 sempre (test) */
static int g_dsa_force=0; /* DSA_FORCE=1: selezione sempre attiva (test: top-min(k,T)=denso) */
static int cmp_fdesc(const void *a,const void *b){
@@ -4503,9 +4520,13 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
}
double tg0=now_s();
int all=1, issued[COLI_CUDA_MAX_DEVICES]={0};
- for(int di=0;di<g_cuda_ndev && all;di++) if(pd_nc[di])
+ for(int di=0;di<g_cuda_ndev && all;di++) if(pd_nc[di]){
+ ESlot *refs[64]; for(int q=0;q<pd_nc[di];q++) refs[q]=pg_e[pd_which[di][q]];
+ eslots_acquire(refs,pd_nc[di]);
all=issued[di]=coli_cuda_expert_group_issue(pd_g[di],pd_u[di],pd_d[di],
pd_rows[di],pd_nc[di],group_x+(int64_t)pd_off[di]*D);
+ if(!issued[di]) eslots_release(refs,pd_nc[di]);
+ }
g_ovl_issue+=now_s()-tg0;
if(all){
static int announced2;
@@ -4523,8 +4544,10 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
m->gpu_expert_calls++;
}
} else {
- for(int di=0;di<g_cuda_ndev;di++)
- if(issued[di]) coli_cuda_expert_group_take(g_cuda_devices[di]);
+ for(int di=0;di<g_cuda_ndev;di++) if(issued[di]){
+ coli_cuda_expert_group_take(g_cuda_devices[di]);
+ for(int q=0;q<pd_nc[di];q++) eslot_release(pg_e[pd_which[di][q]]);
+ }
}
}
}
@@ -4793,6 +4816,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
g_ovl_cpu+=tg1-g_ovl_mark; /* CPU-row window between issue and take */
for(int di=0;di<g_cuda_ndev;di++) if(dev_nc0[di]){
const float *hy=coli_cuda_expert_group_take(g_cuda_devices[di]);
+ for(int q=0;q<dev_nc0[di];q++) eslot_release(eg_e[dev_which0[di][q]]);
int cur=0;
for(int q=0;q<dev_nc0[di];q++){
int gi=dev_which0[di][q], nr=eg_n[gi];
@@ -4848,22 +4872,29 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
if(g_group_async<0) g_group_async=getenv("COLI_GROUP_ASYNC")?atoi(getenv("COLI_GROUP_ASYNC")):0;
if(g_group_async && S<=4 && g_cuda_ndev>0){
int issued[COLI_CUDA_MAX_DEVICES]={0}, all=1;
- for(int di=0;di<g_cuda_ndev && all;di++) if(dev_nc[di])
+ for(int di=0;di<g_cuda_ndev && all;di++) if(dev_nc[di]){
+ ESlot *refs[64]; for(int q=0;q<dev_nc[di];q++) refs[q]=group_e[dev_which[di][q]];
+ eslots_acquire(refs,dev_nc[di]);
all=issued[di]=coli_cuda_expert_group_issue(dev_g[di],dev_u[di],dev_d[di],
dev_rows[di],dev_nc[di],group_x+(int64_t)dev_off[di]*D);
+ if(!issued[di]) eslots_release(refs,dev_nc[di]);
+ }
if(all){
static int announced;
if(!announced){ announced=1; fprintf(stderr,"[CUDA] expert group async path active\n"); }
async_done=1;
for(int di=0;di<g_cuda_ndev;di++) if(dev_nc[di]){
const float *hy=coli_cuda_expert_group_take(g_cuda_devices[di]);
+ for(int q=0;q<dev_nc[di];q++) eslot_release(group_e[dev_which[di][q]]);
if(hy){ dev_ok[di]=1;
memcpy(group_y+(int64_t)dev_off[di]*D,hy,(size_t)dev_total[di]*D*sizeof(float)); }
else dev_ok[di]=0; /* per-device sync failure: CPU fallback below */
}
if(g_prof) m->t_egpu+=now_s()-tg;
- } else for(int di=0;di<g_cuda_ndev;di++)
- if(issued[di]) coli_cuda_expert_group_take(g_cuda_devices[di]);
+ } else for(int di=0;di<g_cuda_ndev;di++) if(issued[di]){
+ coli_cuda_expert_group_take(g_cuda_devices[di]);
+ for(int q=0;q<dev_nc[di];q++) eslot_release(group_e[dev_which[di][q]]);
+ }
}
if(!async_done){
/* resident path (#431 PR-C0): issue the group on its device with the input
@@ -4875,11 +4906,14 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
for(int di=0;di<g_cuda_ndev;di++) if(dev_nc[di]){
float wbuf[64];
for(int q=0;q<dev_nc[di];q++) wbuf[q]=group_weight[(int64_t)dev_which[di][q]*S];
+ ESlot *refs[64]; for(int q=0;q<dev_nc[di];q++) refs[q]=group_e[dev_which[di][q]];
+ eslots_acquire(refs,dev_nc[di]);
if(coli_cuda_expert_group_resident_issue(dev_g[di],dev_u[di],dev_d[di],wbuf,dev_nc[di],
g_pres_home,g_pres_xsrc,g_pres_slots+(int64_t)g_pres_nused*D)){
g_pres_used[g_pres_nused++]=g_cuda_devices[di];
+ for(int q=0;q<dev_nc[di];q++) g_pres_refs[g_pres_nrefs++]=refs[q];
dev_ok[di]=2; /* handled on device: skip host collection */
- }
+ } else eslots_release(refs,dev_nc[di]);
}
}
}
@@ -4933,7 +4967,11 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
int promo = nmiss<m->ecap ? nmiss : m->ecap;
for(int a=0;a<promo;a++){ int q=nmiss-1-a; ESlot *dst;
if(*nn<m->ecap) dst=&Sl[(*nn)++];
- else { int lru=0; for(int z=1;z<*nn;z++) if(Sl[z].used<Sl[lru].used) lru=z; dst=&Sl[lru]; }
+ else { int lru=eslot_lru_victim(Sl,*nn);
+ if(lru<0){ static int warned;
+ if(!warned){ warned=1; fprintf(stderr,"[CUDA] all LRU expert slots are in flight; skipping cache promotion\n"); }
+ continue; }
+ dst=&Sl[lru]; }
ESlot tmp=*dst; *dst=m->ws[q]; m->ws[q]=tmp; dst->used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); }
}
}
@@ -5122,6 +5160,7 @@ static void pilot_realload(Model *m, int layer, int eid){
else {
slot=-1;
for(int z=0;z<nn;z++){
+ if(eslot_busy(&Sl[z])) continue; /* borrowed by an async GPU read */
if(Sl[z].eid==-1){ slot=z; break; } /* riusa uno slot libero/fallito */
if(Sl[z].eid< -1) continue; /* prenotazione di un ALTRO worker: mai vittima */
if(slot<0 || Sl[z].used<Sl[slot].used) slot=z;
@@ -5201,6 +5240,7 @@ static void pilot_uring_batch(Model *m){
else{
slot=-1;
for(int z=0;z<nn;z++){
+ if(eslot_busy(&Sl[z])) continue; /* borrowed by an async GPU read */
if(Sl[z].eid==-1){ slot=z; break; }
if(Sl[z].eid< -1) continue; /* URING reservation in flight */
if(slot<0 || Sl[z].used<Sl[slot].used) slot=z;
@@ -5575,7 +5615,7 @@ static int pipe_layer_sparse(Model *m, Layer *l, int li, float *x_dev, int S, in
if(g_cuda_resid && S==1){
g_pres_slots=coli_cuda_pipe_scratch(dev,25,(size_t)g_cuda_ndev*D*sizeof(float));
res_acc =coli_cuda_pipe_scratch(dev,26,(size_t)D*sizeof(float));
- if(g_pres_slots && res_acc){ g_pres_home=dev; g_pres_xsrc=nrm_d; g_pres_nused=0; }
+ if(g_pres_slots && res_acc){ g_pres_home=dev; g_pres_xsrc=nrm_d; g_pres_nused=0; g_pres_nrefs=0; }
else { g_pres_slots=NULL; res_acc=NULL; }
}
moe(m,l,li,nrm_host,S,out_host,0); /* self-times its own t_emm */
@@ -5587,8 +5627,9 @@ static int pipe_layer_sparse(Model *m, Layer *l, int li, float *x_dev, int S, in
* stream behind every issue event and reduces in issue order. A
* failure here would silently drop routed experts β€” that is a wrong
* answer, not a slow one, so it is fatal by design. */
- if(!coli_cuda_expert_group_resident_take(dev,g_pres_used,nused,g_pres_slots,res_acc,D)||
- !coli_cuda_pipe_add(dev,x_dev,res_acc,(size_t)D)){
+ int take_ok=coli_cuda_expert_group_resident_take(dev,g_pres_used,nused,g_pres_slots,res_acc,D);
+ eslots_release(g_pres_refs,g_pres_nrefs); g_pres_nrefs=0;
+ if(!take_ok || !coli_cuda_pipe_add(dev,x_dev,res_acc,(size_t)D)){
fprintf(stderr,"[CUDA] resident expert take failed β€” refusing to drop routed experts\n");
exit(1);
}
@@ -6997,7 +7038,7 @@ static void rss_guard(Model *m){
int nn=m->ecn[l], lru=-1;
for(int z=0;z<nn;z++){ /* solo slot pubblicati e con slab */
ESlot *cand=&m->ecache[l][z];
- if(cand->eid<0 || !cand->slab) continue;
+ if(cand->eid<0 || !cand->slab || eslot_busy(cand)) continue;
if(lru<0 || cand->used<m->ecache[l][lru].used) lru=z;
}
if(lru<0){ pthread_mutex_unlock(&g_pilot_mx); continue; }
diff --git a/c/tests/test_eslot_inflight.c b/c/tests/test_eslot_inflight.c
new file mode 100644
index 00000000..8ae9cfe6
--- /dev/null
+++ b/c/tests/test_eslot_inflight.c
@@ -0,0 +1,23 @@
+/* Async CUDA groups may borrow an LRU slot until stream completion. The
+ * oldest slot is therefore not necessarily an eligible eviction victim. */
+#define main coli_glm_main_unused
+#include "../colibri.c"
+#undef main
+
+#include <stdio.h>
+
+int main(void){
+ ESlot slots[3]={0};
+ slots[0].used=1; slots[1].used=2; slots[2].used=3;
+
+ if(eslot_lru_victim(slots,3)!=0) return 1;
+ eslot_acquire(&slots[0]);
+ if(eslot_lru_victim(slots,3)!=1) return 2;
+ eslot_acquire(&slots[1]); eslot_acquire(&slots[2]);
+ if(eslot_lru_victim(slots,3)!=-1) return 3;
+
+ eslot_release(&slots[0]); eslot_release(&slots[1]); eslot_release(&slots[2]);
+ if(eslot_lru_victim(slots,3)!=0) return 4;
+ puts("test_eslot_inflight: ok");
+ return 0;
+}