Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix: account real row widths in pin budgets

bheraldbc99ebfa2ab9
diff --git a/c/colibri.c b/c/colibri.c
index e39ce03d..31b28fc6 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -8128,6 +8128,33 @@ static int pin_rec_cmp(const void *a,const void *b){
const PinRec *x=a,*y=b; return x->c<y->c?1:x->c>y->c?-1:0;
}
+/* A pin budget buys a popularity-ranked PREFIX, but mixed-width containers do
+ * not have one honest bytes-per-expert divisor. GLM-5.2's routed rows are int4
+ * while its MTP row is int8: pricing every candidate at the widest row made a
+ * 51.1 GB budget buy the same 1,352 experts in both g64 and E8 containers even
+ * though their routed experts cost only 21.2 and 14.6 MB (#885).
+ *
+ * Keep the ranking semantics -- never skip a hot wide candidate to admit a
+ * colder narrow one -- and stop immediately before the first candidate that
+ * would cross the byte ceiling. `from` lets CUDA_RELEASE_HOST exclude the
+ * transient VRAM prefix before spending the disjoint RAM budget. */
+static double pin_range_bytes(Model *m, const PinRec *r, int from, int to){
+ double used=0.0;
+ for(int a=from;a<to;a++) used+=(double)expert_bytes_row(m,r[a].l,m->ebits);
+ return used;
+}
+static int pin_count_for_budget(Model *m, const PinRec *r, int from, int n,
+ double budget_b){
+ double used=0.0; int count=0;
+ if(budget_b<=0.0 || from>=n) return 0;
+ for(int a=from;a<n;a++){
+ double need=(double)expert_bytes_row(m,r[a].l,m->ebits);
+ if(need<=0.0 || used+need>budget_b) break;
+ used+=need; count++;
+ }
+ return count;
+}
+
#ifdef __linux__
/* #419: bind the pinned hot-store as ONE arena per layer instead of one mbind
* per slab. Per-slab policies cost ~2 unmergeable VMAs each; a PIN_GB=all load
@@ -8214,20 +8241,21 @@ static void pin_load(Model *m, const char *statspath, double gb, int trusted){
}
free(seen);
qsort(r,(size_t)n,sizeof(*r),pin_rec_cmp);
- int64_t eb=expert_bytes_probe(m,m->ebits);
/* PIN_GB=all (#80): NON "tutti" alla lettera. Pinnare l'intero set ignora il
* budget --ram e fa OOM-kill del kernel a meta' generazione (#229: host 92 GB
* ucciso con --ram 78, anon-rss 89 GB). Clampa a quanti expert entrano nel
* budget RAM, come AUTOPIN; il pin aggiorna resident_bytes, quindi cap_for_ram
* dopo restringe la LRU di conseguenza (nessun doppio conteggio). */
- int npin;
+ double pin_budget_b;
if(gb<0){
double ram_env=getenv("RAM_GB")?atof(getenv("RAM_GB")):0.0;
int est_ctx=getenv("CTX")?atoi(getenv("CTX")):4096; /* stesso default del call site */
double avail=expert_avail(m,ram_env,m->ebits,est_ctx);
- npin=avail>0?(int)(avail/eb):0;
- } else npin=(int)(gb*1e9/eb);
+ pin_budget_b=avail>0?avail:0.0;
+ } else pin_budget_b=gb*1e9;
+ int cpu_from=0;
#ifdef COLI_CUDA
+ int64_t eb=expert_bytes_probe(m,m->ebits); /* shared ws / CUDA staging ceiling */
/* The VRAM budget must be known BEFORE npin is finalized: with
* CUDA_RELEASE_HOST the VRAM-ranked prefix's host slabs are freed right
* after upload, so those slots must NOT consume the RAM pin budget.
@@ -8262,9 +8290,11 @@ static void pin_load(Model *m, const char *statspath, double gb, int trusted){
prefix_est=raw+(int)((budget-(double)raw*eb)/(0.80*eb))+g_cuda_ndev;
}
#endif
- npin+=prefix_est; /* additive: prefix RAM is returned after upload */
+ if(prefix_est>n) prefix_est=n;
+ cpu_from=prefix_est; /* prefix RAM is returned after upload */
}
#endif
+ int npin=cpu_from+pin_count_for_budget(m,r,cpu_from,n,pin_budget_b);
if(npin>n) npin=n;
if(npin<1){ free(r); return; }
int *cnt_l=calloc(c->n_layers+1,sizeof(int)); /* +1: riga MTP */
@@ -8273,7 +8303,7 @@ static void pin_load(Model *m, const char *statspath, double gb, int trusted){
int *slot_of=malloc((size_t)npin*sizeof(int)), *next=calloc(c->n_layers+1,sizeof(int));
for(int a=0;a<npin;a++) slot_of[a]=next[r[a].l]++;
for(int i=0;i<=c->n_layers;i++) m->npin[i]=cnt_l[i];
- double t0=now_s();
+ double t0=now_s(), pin_host_released=0.0;
#ifdef COLI_CUDA
if(prefix_est>0){ gpu_prefix=prefix_est; if(gpu_prefix>npin) gpu_prefix=npin; }
#else
@@ -8322,7 +8352,7 @@ static void pin_load(Model *m, const char *statspath, double gb, int trusted){
#pragma omp parallel for schedule(dynamic,1)
for(int a=base;a<hi;a++)
expert_load(m,r[a].l,r[a].e,&m->pin[r[a].l][slot_of[a]],1,0); /* startup pin load; demand=0, never classified */
- m->resident_bytes+=(int64_t)(hi-base)*eb;
+ m->resident_bytes+=(int64_t)pin_range_bytes(m,r,base,hi);
#ifdef COLI_CUDA
if(g_cuda_enabled && budget>0){
for(int a=base;a<hi && m->gpu_expert_bytes<budget;a++){
@@ -8366,7 +8396,7 @@ static void pin_load(Model *m, const char *statspath, double gb, int trusted){
m->gpu_expert_count++; m->gpu_expert_bytes+=actual;
remaining[best]-=actual; placed_b[best]+=actual; placed_n[best]++;
placed_w[best]+=(double)r[a].c;
- if(g_cuda_release_host) expert_host_release(m,s);
+ if(g_cuda_release_host){ expert_host_release(m,s); pin_host_released+=(double)need; }
placed=1;
} else {
qt_cuda_reset(&s->g); qt_cuda_reset(&s->u); qt_cuda_reset(&s->d);
@@ -8416,10 +8446,12 @@ static void pin_load(Model *m, const char *statspath, double gb, int trusted){
#pragma omp parallel for schedule(dynamic,1)
for(int a=gpu_prefix;a<npin;a++)
expert_load(m,r[a].l,r[a].e,&m->pin[r[a].l][slot_of[a]],1,0); /* startup pin load; demand=0, never classified */
- m->resident_bytes+=(int64_t)(npin-gpu_prefix)*eb;
+ m->resident_bytes+=(int64_t)pin_range_bytes(m,r,gpu_prefix,npin);
}
+ double warm_b=pin_range_bytes(m,r,0,npin)-pin_host_released;
+ if(warm_b<0.0) warm_b=0.0;
fprintf(stderr,"[PIN] placement: %d VRAM + %d RAM expert (%.1f GB warm) in %.0fs da %s\n",
- m->gpu_expert_count,npin-m->gpu_expert_count,(npin-m->gpu_expert_count)*eb/1e9,now_s()-t0,statspath);
+ m->gpu_expert_count,npin-m->gpu_expert_count,warm_b/1e9,now_s()-t0,statspath);
pin_wire(m); /* inchioda in RAM (no compressione) / wire in RAM (no compression) */
free(r); free(cnt_l); free(slot_of); free(next);
}
diff --git a/c/tests/test_cap_mixed_width.c b/c/tests/test_cap_mixed_width.c
index e1477fc0..e01bb4d1 100644
--- a/c/tests/test_cap_mixed_width.c
+++ b/c/tests/test_cap_mixed_width.c
@@ -197,6 +197,33 @@ int main(void){
CHECK(wide > narrow); /* int8 head is wider than int4 rows */
CHECK(probe == wide); /* the probe is the container maximum */
+ /* ---- B. PIN_GB buys the ranked prefix at each row's real cost (#885) -- */
+ {
+ PinRec ranked[5]={
+ {FIRST_DENSE,0,500}, {FIRST_DENSE+1,0,400},
+ {FIRST_DENSE+2,0,300}, {FIRST_DENSE+3,0,200},
+ {N_LAYERS,0,100}
+ };
+ double four_narrow=4.0*(double)narrow;
+ int exact=pin_count_for_budget(&m,ranked,0,5,four_narrow);
+ int old=(int)(four_narrow/(double)probe);
+ printf("B. PIN budget %.0f B: row-aware %d experts, widest-divisor %d\n",
+ four_narrow,exact,old);
+ CHECK(exact==4); /* all four affordable routed experts */
+ CHECK(old<exact); /* #885's under-pinning is reproduced */
+ CHECK(pin_range_bytes(&m,ranked,0,exact)<=four_narrow);
+
+ /* Prefix semantics are intentional: do not skip a hotter wide expert to
+ * admit a colder narrow one, and support a disjoint post-VRAM suffix. */
+ PinRec mixed[3]={
+ {FIRST_DENSE,0,30}, {N_LAYERS,0,20}, {FIRST_DENSE+1,0,10}
+ };
+ CHECK(pin_count_for_budget(&m,mixed,0,3,(double)narrow+(double)wide-1.0)==1);
+ CHECK(pin_count_for_budget(&m,mixed,0,3,(double)narrow+(double)wide)==2);
+ CHECK(pin_count_for_budget(&m,mixed,1,3,(double)wide+(double)narrow)==2);
+ CHECK(pin_count_for_budget(&m,mixed,3,3,1e9)==0);
+ }
+
/* ---- (1) the divisor is the sum of the REAL widths --------------------- */
int nsp=0; for(int i=0;i<c->n_layers;i++) if(m.L[i].sparse) nsp++;
double row_b = expert_cache_row_bytes(&m,m.ebits);