Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
plan: print every placement term, and a switch to A/B the slab shrink (#885)
JustVugg591eacedbb2d
diff --git a/c/colibri.c b/c/colibri.cindex 7b0af931..e39ce03d 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -89,6 +89,21 @@ static inline void omp_set_num_threads(int n){ (void)n; }* active" without needing COLI_METAL (and the backend_metal.h / Metal framework link it* would drag in) on every platform's test build. */static int g_metal_enabled;+/* COLI_SLAB_SHRINK=0 disables the #856 slab shrink at runtime.+ *+ * The shrink stops a wide slab migrating into a narrow row through the+ * ws[]<->LRU swap, which is what lets the cap be priced per row. It is also the+ * only new allocator work in that change: a slot alternating between the int8+ * MTP row and the int4 main rows reallocates instead of staying wide, and at+ * ~19-38 MB glibc serves that with mmap/munmap plus page faults on first touch.+ * #869 stated that cost was unmeasured; #885 then measured five runs on an+ * int4-g64 container at 1.12/0.98/1.11/0.97/1.05 -- spread 0.15, bimodal --+ * against <=0.02 on every other arm of the same box.+ *+ * This exists so that hypothesis can be A/B'd without a rebuild. Setting it to 0+ * restores grow-only and re-introduces the migration, so the per-row cap becomes+ * optimistic again: a diagnostic, not a supported configuration. */+static int g_slab_shrink = 1;#ifdef COLI_METAL#include "backend_metal.h"/* No <omp.h> here: the guarded include above already provides it under _OPENMP@@ -97,7 +112,9 @@ static int g_metal_enabled;* stock macOS -- exactly the platform this backend targets -- even though the* Makefile advertises the single-threaded path ("libomp not found: building* single-threaded"). */-static int g_metal_enabled;+/* g_metal_enabled is declared unconditionally above (#587) so portable code can+ * read it; the duplicate that used to sit here was a tentative definition the+ * linker merged, hence no warning and no symptom. */static int g_metal_gemm_min=16; /* COLI_METAL_GEMM_MIN: min rows to send a matmul_qt GEMM to GPU */static int g_moe_exact=0;/* output dello shared expert gia' calcolato su GPU (solo Metal layer-CB) */@@ -2337,7 +2354,7 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal, i* have nothing to shrink. */int64_t want=wtot+8192;int64_t hyst=want/4; if(hyst<(1<<16)) hyst=1<<16; /* 64 KB clears the 16 KB METAL rounding */- if(!s->slab || want > s->slab_cap || (!s->aslab && s->slab_cap > want+hyst)){+ if(!s->slab || want > s->slab_cap || (g_slab_shrink && !s->aslab && s->slab_cap > want+hyst)){#ifdef COLI_METAL/* page-align + zero-copy wrap: the GPU reads this slab in place (unified memory) */if(s->slab && g_metal_enabled) coli_metal_unregister(s->slab);@@ -2356,7 +2373,7 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal, i/* The scales migrate with their weights, so they shrink on the same rule (#856).* fslab_cap counts FLOATS, so the hysteresis is in floats too. */int64_t fhyst=ftot/4; if(fhyst<(1<<14)) fhyst=1<<14; /* floats, so 64 KB again */- if(!s->fslab || ftot > s->fslab_cap || (!s->afslab && s->fslab_cap > ftot+fhyst)){+ if(!s->fslab || ftot > s->fslab_cap || (g_slab_shrink && !s->afslab && s->fslab_cap > ftot+fhyst)){#ifdef COLI_METAL/* page-align + register: the GPU reads the scales in place (unified memory).* Honours `fatal` exactly like the CPU arm below — a speculative pilot load@@ -2567,7 +2584,7 @@ static int uring_load_add(UringBatch *b,Model *m,int layer,int eid,ESlot *s,int* cap_for_ram() computes stops being true. Same hysteresis, same arena guard. */int64_t want=wtot+8192, hyst=want/4; if(hyst<(1<<16)) hyst=1<<16;int64_t fhyst=ftot/4; if(fhyst<(1<<14)) fhyst=1<<14; /* floats, so 64 KB again */- if(!s->slab || want>s->slab_cap || (!s->aslab && s->slab_cap > want+hyst)){+ if(!s->slab || want>s->slab_cap || (g_slab_shrink && !s->aslab && s->slab_cap > want+hyst)){#ifdef COLI_METALif(s->slab&&g_metal_enabled) coli_metal_unregister(s->slab);compat_aligned_free(s->slab);@@ -2582,7 +2599,7 @@ static int uring_load_add(UringBatch *b,Model *m,int layer,int eid,ESlot *s,ints->slab_cap=wtot+8192;#endif}- if(!s->fslab || ftot>s->fslab_cap || (!s->afslab && s->fslab_cap > ftot+fhyst)){+ if(!s->fslab || ftot>s->fslab_cap || (g_slab_shrink && !s->afslab && s->fslab_cap > ftot+fhyst)){#ifdef COLI_METALif(s->fslab&&g_metal_enabled) coli_metal_unregister(s->fslab);free(s->fslab); size_t fb=(((size_t)ftot*sizeof(float))+16383)&~(size_t)16383;@@ -9215,6 +9232,11 @@ int main(int argc, char **argv){if(g_pipe_nw<1) g_pipe_nw=1;g_pipe_block = getenv("COLI_PIPE_BLOCK")?atoi(getenv("COLI_PIPE_BLOCK")):0; /* blocking pipe_wait (default: spin) */g_direct = getenv("DIRECT")?atoi(getenv("DIRECT")):0;+ g_slab_shrink = getenv("COLI_SLAB_SHRINK") ? atoi(getenv("COLI_SLAB_SHRINK")) : 1;+ if(!g_slab_shrink)+ fprintf(stderr,"[SLAB] shrink disabled (COLI_SLAB_SHRINK=0): reused slots keep the widest "+ "width they have held, so the per-row cap is optimistic again. Diagnostic "+ "for #885 -- not a supported configuration.\n");{ const char *dh=getenv("COLI_DISKCLASS_WINDOW"); /* DISK-CLASS recency window, see its declaration */if(dh){ g_direct_heat_ticks=(uint32_t)strtoul(dh,NULL,10); g_direct_heat_explicit=1; } }g_uring = getenv("URING")?atoi(getenv("URING")):0;@@ -9548,10 +9570,30 @@ int main(int argc, char **argv){m.ecap,&preserved_cap);double pin_bytes=autopin_preserve_lru(planned_pin,expert_available,lru_reserve);- if(pin_bytes+1.0<planned_pin)- fprintf(stderr,"[PIN] auto: %.1f GB plan capped to %.1f GB to preserve "- "the no-pin LRU cap %d/layer\n",- planned_pin/1e9,pin_bytes/1e9,preserved_cap);+ /* Print every term, every time, capped or not.+ *+ * This line used to appear ONLY when the clamp bit (pin_bytes <+ * planned_pin), which makes "the reserve was zero" and "the reserve+ * did not bind" indistinguishable from the outside -- and those are+ * the two hypotheses anyone debugging placement is choosing between.+ * #885 took fourteen instrumented runs on a 128 GB host to establish+ * numbers this line already had in registers. @mohamedmastouri2000-boop+ * asked for exactly this, twice, before anyone acted on it.+ *+ * budget = what expert_avail() left after resident + slack+ * plan = 0.5 * budget * confidence(history), the autopin quota+ * reserve = cap * per-row width, held back for the adaptive LRU (#815)+ * max_pin = budget - reserve, the ceiling the plan is clamped to+ *+ * A reserve near the budget means the pin is being starved by a+ * reservation the LRU may never claim, since the LRU is demand-filled+ * and only allocates on a miss. That is visible here at a glance. */+ fprintf(stderr,"[PIN] auto: budget %.1f GB | plan %.1f GB (conf %.2f, %lld selections) | "+ "LRU reserve %.1f GB (cap %d/layer x %.1f MB/row-set) | max_pin %.1f GB -> pinning %.1f GB%s\n",+ expert_available/1e9, planned_pin/1e9, conf, (long long)hist,+ lru_reserve/1e9, preserved_cap, expert_cache_bytes_per_slot(&m,ebits)/1e6,+ (expert_available-lru_reserve)/1e9, pin_bytes/1e9,+ pin_bytes+1.0<planned_pin ? " [CAPPED by the LRU reserve]" : "");double pin_gb=pin_bytes/1e9;if(pin_gb>=0.5) pin_load(&m, g_usage_path, pin_gb, 0); /* auto-discovered: not trusted */}diff --git a/c/tests/test_cap_mixed_width.c b/c/tests/test_cap_mixed_width.cindex 236d79bf..e1477fc0 100644--- a/c/tests/test_cap_mixed_width.c+++ b/c/tests/test_cap_mixed_width.c@@ -279,6 +279,26 @@ int main(void){CHECK(main_cache.slab_cap >= narrow); /* and still holds what it holds */}+ /* ---- E. the diagnostic switch actually gates the shrink ---------------- */+ /* g_slab_shrink is set from COLI_SLAB_SHRINK in main(), which this test does+ * not run -- so it is set directly here. Verifying the switch by exporting+ * the variable would have proved nothing, and very nearly did. */+ {+ ESlot s2; memset(&s2,0,sizeof s2);+ g_slab_shrink = 0;+ CHECK(expert_load(&m,N_LAYERS,0,&s2,0,0)==0); /* wide */+ int64_t wide_cap = s2.slab_cap;+ CHECK(expert_load(&m,FIRST_DENSE,0,&s2,0,0)==0); /* then narrow */+ printf("E. shrink OFF: %lld B -> %lld B (must not move)\n",+ (long long)wide_cap,(long long)s2.slab_cap);+ CHECK(s2.slab_cap == wide_cap); /* grow-only: the wide slab stays */+ g_slab_shrink = 1;+ CHECK(expert_load(&m,FIRST_DENSE,1,&s2,0,0)==0); /* narrow again, shrink on */+ printf("E. shrink ON : %lld B -> %lld B (must come down)\n",+ (long long)wide_cap,(long long)s2.slab_cap);+ CHECK(s2.slab_cap < wide_cap);+ }+/* ---- C. arena slices must never be shrunk ------------------------------ *//* pin_arena_bind hands slots interior pointers into ONE per-layer allocation* (#419). Freeing one would corrupt the heap, and they are already per-layer