Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fp8: fmt=6 (E8/IQ3) branch in qt_scale_bytes()
monotophic9fdc6bb86145
diff --git a/c/colibri.c b/c/colibri.cindex d722c5a8..8bc53cd2 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -208,21 +208,17 @@ static int64_t qt_bytes(const QT *t){ /* byte residenti del tensore */* its own to split qt_bytes(t) into a weight half and a scale half. Hardcoding* scale_b=O*4 at those two call sites -- i.e. assuming PER-ROW scale for every* format -- is right for fmt 0/1/2/3 but wrong for fmt=4 (grouped,- * O*ceil(I/gs) scales), fmt=5 (int3-g64, O*ceil(I/64) scales), and fmt=7- * (per-128x128-block, ceil(O/128)*ceil(I/128) scales): any tensor in one of- * those three formats reaching qt_wire_mmap/qt_unwire_mmap would mlock/munlock- * the wrong byte ranges on BOTH halves (weight_b = qt_bytes(t)-scale_b shifts- * too). fmt=6 (E8) is deliberately NOT covered here: its .qs is a fixed 4-byte- * tag with no wire-mmap-relevant weight/scale split of its own (matches- * qt_bytes' `+4` literal above), and t->fmt==6 tensors never reach- * qt_wire_mmap/qt_unwire_mmap's mlock path in this build (int3-g64/E8 stay- * CPU-side, see qt_cuda_upload -- MMAP wiring is a CUDA/Metal-adjacent memory- * concern this build doesn't extend to them). This is not purely a dormant- * fmt=7-only fix: fmt=4 is the routed-expert "int4-g64" format (qt_resolve_fmt- * assigns it to ESlot g/u/d the same as any other tensor, see- * expert_load_impl), so any COLI_MMAP + mem_should_wire() session pinning- * fmt=4 experts mlocks the wrong ranges without this. UNVERIFIED at runtime- * (no model run performed -- static/arithmetic fix only, see the report).+ * O*ceil(I/gs) scales), fmt=5 (int3-g64, O*ceil(I/64) scales), fmt=6 (E8/IQ3,+ * a FIXED 4-byte tag, see below), and fmt=7 (per-128x128-block,+ * ceil(O/128)*ceil(I/128) scales): any tensor in one of those four formats+ * reaching qt_wire_mmap/qt_unwire_mmap would mlock/munlock the wrong byte+ * ranges on BOTH halves (weight_b = qt_bytes(t)-scale_b shifts too). This is+ * not purely a dormant fmt=7-only fix: fmt=4 is the routed-expert+ * "int4-g64" format (qt_resolve_fmt assigns it to ESlot g/u/d the same as+ * any other tensor, see expert_load_impl), so any COLI_MMAP ++ * mem_should_wire() session pinning fmt=4 experts mlocks the wrong ranges+ * without this. UNVERIFIED at runtime (no model run performed -- static/+ * arithmetic fix only, see the report).** REVIEW FINDING (maintainer, #528): this function existed correctly (the* fmt=7 branch below is right) but neither qt_wire_mmap nor qt_unwire_mmap@@ -231,14 +227,40 @@ static int64_t qt_bytes(const QT *t){ /* byte residenti del tensore */* call site, and the resulting -Wunused-function warning on this then-dead* function was suppressed by -Wno-unused-function in CFLAGS rather than* caught. qt_wire_split() below is now the ONE place both call sites get- * weight_b/scale_b from -- see it and its call sites for the actual fix. */+ * weight_b/scale_b from -- see it and its call sites for the actual fix.+ *+ * FIX ROUND (audit finding, SHOULD-FIX): this comment used to ALSO claim+ * fmt=6 (E8/IQ3) was "deliberately NOT covered" because "t->fmt==6 tensors+ * never reach qt_wire_mmap/qt_unwire_mmap's mlock path in this build+ * (int3-g64/E8 stay CPU-side, see qt_cuda_upload -- MMAP wiring is a+ * CUDA/Metal-adjacent memory concern this build doesn't extend to them)" --+ * that conflated two UNRELATED mechanisms: qt_cuda_upload decides GPU-VRAM+ * upload eligibility (fmt=5/6 genuinely excluded there, no CUDA kernel for+ * either -- see qt_cuda_upload's own `fmt==5||fmt==6` guard), while+ * qt_wire_mmap/qt_unwire_mmap mlock HOST RAM pages under COLI_MMAP, an+ * entirely CPU-side concern. Staying CPU-side (no CUDA kernel) is exactly+ * what makes a tensor a CANDIDATE for RAM wiring, not exempt from it.+ * expert_load_impl assigns fmt=6 to ESlot g/u/d exactly like fmt=4/5 (same+ * qt_resolve_fmt call site, three lines above the fmt=4 comment), and+ * pin_wire calls qt_wire_mmap/qt_unwire_mmap on every pinned ESlot's g/u/d+ * unconditionally, with no format filter -- so a pinned E8/IQ3 expert under+ * COLI_MMAP + mem_should_wire() DOES reach here. Before the fmt=6 branch+ * below, the O*4 fallback would have returned a scale_b wildly larger than+ * the tensor's real scale allocation (a FIXED 4 bytes, qsalloc(1) in+ * qt_from_disk's fmt==6 branch -- confirmed against qt_bytes' own `+4`+ * literal and qt_from_disk's st_read_f32_cap cardinality switch, both of+ * which already treat fmt=6's scale as exactly 1 float, not O floats), then+ * mlock/munlock'd that many bytes starting at t->s -- past the end of a+ * 4-byte allocation. UNVERIFIED at runtime (same static/arithmetic-only+ * caveat as the fmt=4/5/7 fix above -- no model run performed). */static int64_t qt_scale_bytes(const QT *t){if(t->fmt==4){ int ng=(t->I+t->gs-1)/t->gs; return (int64_t)t->O*ng*4; }if(t->fmt==5){ int64_t ng=((int64_t)t->I+63)/64; return (int64_t)t->O*ng*4; }+ if(t->fmt==6) return 4; /* E8/IQ3: FIXED 4-byte tag (qsalloc(1)), not O*4 -- see this+ * function's own header comment for why this is reachable and+ * load-bearing, not dead code. */if(t->fmt==7){ int64_t nblkO=((int64_t)t->O+127)/128, nblkI=((int64_t)t->I+127)/128; return nblkO*nblkI*4; }- return (int64_t)t->O*4; /* fmt 0 (t->s NULL, guarded by callers)/1/2/3/6: per-row (or, for- * fmt=6, the 4-byte tag -- callers of this function never see fmt=6,- * see the comment above; the fallback is harmless dead code for it). */+ return (int64_t)t->O*4; /* fmt 0 (t->s NULL, guarded by callers)/1/2/3: per-row. */}/* qt_wire_mmap/qt_unwire_mmap's shared weight/scale byte-range split -- the ONE* place that computes it, so the two sites can never independently drift backdiff --git a/c/tests/test_fp8_load.c b/c/tests/test_fp8_load.cindex 9a22a69a..9c01b24a 100644--- a/c/tests/test_fp8_load.c+++ b/c/tests/test_fp8_load.c@@ -481,8 +481,10 @@ static void check_fp8_bytes(int O, int I, const char *tag){* -Wno-unused-function suppressed the warning that should have caught it) --* fails here. Covers every format qt_wire_split's callers can see in practice* (fmt=1 per-row unaffected; fmt=4/5 grouped-scale formats qt_scale_bytes'- * own comment names as previously-broken too; fmt=7 nblk>O, the shape this- * review round is about). */+ * own comment names as previously-broken too; fmt=6 (E8/IQ3, FIX ROUND,+ * audit finding: a FIXED 4-byte tag, not O*4 -- see qt_scale_bytes' own+ * comment for why this is reachable, not dead code); fmt=7 nblk>O, the+ * shape this review round is about). */static void check_wire_split(int fmt, int O, int I, int gs, const char *tag){QT t; memset(&t,0,sizeof t); t.fmt=fmt; t.O=O; t.I=I; t.gs=gs;int64_t want_scale = qt_scale_bytes(&t);@@ -497,12 +499,17 @@ static void check_wire_split(int fmt, int O, int I, int gs, const char *tag){CHECK(got_weight == want_weight);/* the regression this test exists to catch: a per-row-only scale_b==O*4* must NOT be what qt_wire_split returns for a format whose real scale- * cardinality differs from O (fmt=4/5/7 here) -- confirm the value has+ * cardinality differs from O (fmt=4/5/6/7 here) -- confirm the value has* actually moved off that old constant, not just matched qt_scale_bytes()* by coincidence at a degenerate shape. */int64_t old_wrong_scale = (int64_t)O*4;- if((fmt==4 || fmt==5 || fmt==7) && want_scale != old_wrong_scale)+ if((fmt==4 || fmt==5 || fmt==6 || fmt==7) && want_scale != old_wrong_scale)CHECK(got_scale != old_wrong_scale);+ /* fmt=6's scale is a FIXED 4 bytes regardless of [O,I] -- assert the+ * literal value directly too, not just "moved off O*4", since a future+ * regression that made it O-dependent in some OTHER wrong way would+ * still pass the generic check above. */+ if(fmt==6) CHECK(want_scale == 4);}/* ---- Site-level wire regression (FIX ROUND, validator finding: mutation-@@ -600,6 +607,8 @@ int main(void){check_wire_split(1, 4096,4096, 0, "qt_wire_split fmt=1 plain int8 (per-row scale, unaffected by the fix)");check_wire_split(4, 2048,6144, 64, "qt_wire_split fmt=4 grouped int4 (O*ceil(I/gs) scale, not O*4)");check_wire_split(5, 2048,6144, 0, "qt_wire_split fmt=5 int3-g64 (O*ceil(I/64) scale, not O*4)");+ check_wire_split(6, 2048,6144, 0, "qt_wire_split fmt=6 E8/IQ3 (FIXED 4-byte tag, not O*4=8192B)");+ check_wire_split(6, 1,1, 0, "qt_wire_split fmt=6 E8/IQ3 degenerate O=1 (O*4 would coincidentally also be 4 -- exercises the literal-4 assert, not just the moved-off-O*4 one)");check_wire_split(7, 2,16384, 0, "qt_wire_split fmt=7 nblk(128) >> O(2): scale=512B, NOT O*4=8B");check_wire_split(7, 2048,6144, 0, "qt_wire_split fmt=7 spec example: scale=3072B, NOT O*4=8192B");test_wire_site_regression();