Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(colibri): add fmt=8 arms to the CPU absorb path
monotophic0ad3eef77066
diff --git a/c/colibri.c b/c/colibri.cindex 64bf3680..8ffd06bd 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -2266,6 +2266,43 @@ static void qt_cuda_colocate(QT *dst,const QT *src){}static void layer_cuda_shard_kvb(Layer *l,int H,int Q,int V){if(!g_cuda_enabled||!g_cuda_dense||g_cuda_ndev<2||l->kv_b.fmt==0)return;+ /* SHARD FORMAT ALLOWLIST (explicit refusal; this was an ACCIDENTAL fail-safe): the+ * rb/weights/scale arithmetic below is written for exactly fmt=1 (int8, per-row+ * scale), fmt=2 (int4 per-row), fmt=3 (int2 per-row) and fmt=4 (int4 grouped).+ * Any other fmt reaching it computes a wrong row-byte stride, takes l->kv_b.q4 as+ * the weight pointer (NULL for fmt=8, whose raw e4m3 bytes live in q8 -- see the+ * QT struct comment), and slices l->kv_b.s with per-row/per-group geometry that+ * fmt=8's per-128x128-BLOCK scales (and fmt=6's single 4-byte tag) simply do not+ * have. fmt=8 only ever "worked" here by accident: q4==NULL made+ * coli_cuda_tensor_upload_g's !weights check reject the upload before anything+ * dereferenced it -- silent, unnamed, and one refactor away from a misread.+ * Refuse BY NAME instead, BEFORE any pointer/stride use, and say what happens+ * instead: the un-sharded kv_b stays whole on its layer home device, where fmt=8+ * kv_b decode runs the CPU absorb path (qt_addrow/qt_matvec_rows' fmt=8+ * branches below; the CUDA absorb kernels refuse fmt=8 via absorb_fmt_ok's+ * fmt 0..4 allowlist) -- COLI_CUDA_ATTN_SHARD is a no-op for it. Same+ * "refuse rather than misread" discipline as qt_addrow/+ * qt_matvec_rows' guards; notice only (no exit): sharding is an opt-in+ * optimization and skipping it is the correct, working behavior. Bounded once+ * per process per fmt, never per layer (metal_fmt_gate_notice, the precedent+ * for bounded notices, is coarser still: one line per tensor KIND, naming only+ * the first offending fmt). */+ if(l->kv_b.fmt!=1&&l->kv_b.fmt!=2&&l->kv_b.fmt!=3&&l->kv_b.fmt!=4){+ static int refused_fmt[32];+ if(!refused_fmt[l->kv_b.fmt&31]){ refused_fmt[l->kv_b.fmt&31]=1;+ if(l->kv_b.fmt==8)+ fprintf(stderr,"layer_cuda_shard_kvb: kv_b fmt=8 (fp8-e4m3, per-128x128-block "+ "scales) has no head-shard layout here -- refusing the shard; fmt=8 kv_b "+ "runs the absorb path on the layer home device instead, so "+ "COLI_CUDA_ATTN_SHARD is a no-op for it (applies to every layer)\n");+ else+ fprintf(stderr,"layer_cuda_shard_kvb: unsupported kv_b fmt=%d for the head-shard "+ "upload (only fmt 1/2/3/4 match the per-row byte/scale strides computed "+ "here) -- refusing the shard; kv_b stays whole on its layer home device "+ "(applies to every layer)\n",l->kv_b.fmt);+ }+ return;+ }int rb=l->kv_b.fmt==1?l->kv_b.I:(l->kv_b.fmt==2||l->kv_b.fmt==4)?(l->kv_b.I+1)/2:(l->kv_b.I+3)/4;const uint8_t *weights=l->kv_b.fmt==1?(const uint8_t*)l->kv_b.q8:l->kv_b.q4;@@ -3858,6 +3895,32 @@ static void expert_prefetch(Model *m, int layer, int eid){/* ---- helper per l'ABSORPTION: accesso per-riga ai QT quantizzati ---- *//* acc[0..I) += coef * W[row,:] (dequant al volo) */+/* One fmt=8 block scale, checked before it multiplies anything.+ *+ * A NaN scale has no safe interpretation: it poisons the whole block and every+ * accumulator downstream of it, and this function cannot repair it, so it is+ * refused by name with the block that carried it -- the same "refuse rather+ * than misread" discipline the format guards below apply.+ *+ * A ZERO scale is NOT refused: it is valid data. Block scales are amax/448, so+ * a genuinely all-zero block (padding, an unused slice) legitimately produces+ * zero, and decoding it as zeros is the correct answer. It cannot be confused+ * with a decode against an unwritten table the way it can on the GPU, because+ * there is no table here to be unwritten -- the CPU decoder reads its e4m3+ * values from a compile-time constant table, which is why the LUT-ready gate+ * exists only on the CUDA side. Do not re-add a zero refusal: it would reject+ * valid checkpoints.+ *+ * The check is per BLOCK, not per element: one branch per FP8_BLOCK columns. */+static float fp8_block_scale(float sc, int64_t blkO, int64_t bi, const char *who){+ if(isnan(sc)){+ fprintf(stderr,"%s: fmt=8 scale block [%lld,%lld] is NaN -- refusing rather than "+ "propagate it through the absorb accumulator\n",who,(long long)blkO,(long long)bi);+ exit(1);+ }+ return sc;+}+static void qt_addrow(const QT *t, int row, float coef, float *acc){int I=t->I;if(t->fmt==0){ const float *w=t->qf+(int64_t)row*I; for(int i=0;i<I;i++) acc[i]+=coef*w[i]; return; }@@ -3881,8 +3944,23 @@ static void qt_addrow(const QT *t, int row, float coef, float *acc){for(int k=0;k<n;k++){ unsigned u=((lo[k>>2]>>((k&3)*2))&3)|(((hi[k>>3]>>(k&7))&1)<<2);acc[base+k]+=cg*(float)((int)u-4); } }return; }+ /* fmt=8 (fp8-e4m3-b128, absorb-path support added here): t->s holds ONE f32 scale+ * per 128x128 BLOCK (ceil(O/128)*ceil(I/128) entries, block-row-major), not O, and+ * t->q4 is NULL for this format -- raw e4m3 bytes live in t->q8 instead, same+ * convention as fmt=1 (see the QT struct comment). Mirrors matmul_fp8's (quant.h)+ * block-scale indexing exactly: blkO=row/FP8_BLOCK selects the scale row, then one+ * scale per FP8_BLOCK-wide slice of I. This is the branch that used to be missing+ * -- see the guard below's history note. */+ if(t->fmt==8){ const uint8_t *w=(const uint8_t*)t->q8+(int64_t)row*I;+ int64_t nblkI=fp8_nblk(I), blkO=(int64_t)row/FP8_BLOCK;+ const float *scl=t->s+blkO*nblkI;+ for(int64_t bi=0; bi*FP8_BLOCK<I; bi++){+ int base=(int)(bi*FP8_BLOCK); int blen=FP8_BLOCK; if(base+blen>I) blen=I-base;+ float sc=coef*fp8_block_scale(scl[bi],blkO,bi,"qt_addrow");+ for(int i=base;i<base+blen;i++) acc[i]+=sc*e4m3_decode(w[i]); }+ return; }/* GUARD (fix round 2, engine defect -- clean-room conformance trial found a real- * SIGSEGV, reproduced): fmt 0/4/5 already returned above; everything below this+ * SIGSEGV, reproduced): fmt 0/4/5/8 already returned above; everything below this* point assumes a PER-ROW scale (t->s[row]) followed by fmt=1 (int8, explicit* branch), fmt=2 (int4 packed, explicit branch), or the tail's own IMPLICIT fmt=3* (int2 packed, the final unconditional block) -- there was no guard stopping any@@ -3891,20 +3969,16 @@ static void qt_addrow(const QT *t, int row, float coef, float *acc){* a heap OVERREAD, and the untouched fall-through then misreads t->q4's real E8* lattice bytes as int2-packed data (same bug SHAPE as #298's CUDA absorb-kernel* fix, and the same one this file's own fmt=4/5 branches above were added to- * dodge -- fmt=6 was simply missed). fmt=8 (fp8-e4m3-b128): t->s holds- * ceil(O/128)*ceil(I/128) per-block floats, not O -- t->s[row] overreads for- * row>=nblk (e.g. a [130,130] tensor has nblk=4, so every row past 3 already reads- * out of bounds), AND t->q4 is NULL for fmt=8 (raw bytes live in t->q8 instead,- * same convention as fmt=1 -- see the QT struct comment), so the fall-through's- * `t->q4+(int64_t)row*((I+3)/4)` dereferences NULL-plus-offset: SIGSEGV,- * reproduced (see the report's proof-of-bite transcript). Refuse loudly instead --- * this function has no byte-count context of its own to validate against (it only- * ever sees an already-resolved QT), so "unsupported fmt" is the only check- * available, same "refuse rather than misread" discipline qt_resolve_fmt applies- * at load time. */+ * dodge -- fmt=6 was simply missed). fmt=8 previously landed here too (t->s[row]+ * overreads past row>=nblk, t->q4 is NULL -> SIGSEGV via the int2 fall-through);+ * it now returns above via its own branch and never reaches this guard. Refuse+ * loudly instead for anything else -- this function has no byte-count context of+ * its own to validate against (it only ever sees an already-resolved QT), so+ * "unsupported fmt" is the only check available, same "refuse rather than+ * misread" discipline qt_resolve_fmt applies at load time. */if(t->fmt!=1 && t->fmt!=2 && t->fmt!=3){fprintf(stderr,"qt_addrow: unsupported fmt=%d for the per-row-scale absorb path "- "(only fmt 1/2/3 reach this point; fmt 0/4/5 are handled above and return "+ "(only fmt 1/2/3 reach this point; fmt 0/4/5/8 are handled above and return ""before it) -- refusing rather than misread t->s[row]/t->q4\n", t->fmt);exit(1);}@@ -3954,18 +4028,35 @@ static void qt_matvec_rows(const QT *t, int r0, int n, const float *x, float *y)for(int k=0;k<n;k++){ unsigned u=((lo[k>>2]>>((k&3)*2))&3)|(((hi[k>>3]>>(k&7))&1)<<2);acc+=(float)((int)u-4)*x[base+k]; }a+=(double)(acc*sr[g]); } }+ /* fmt=8 (fp8-e4m3-b128, absorb-path support added here): per-128x128-BLOCK f32+ * scale, block-row-major (ceil(O/128)*ceil(I/128) entries), raw bytes in t->q8+ * (t->q4 is NULL for this format). Same block-scale indexing as matmul_fp8+ * (quant.h) and qt_addrow's fmt=8 branch above: blkO=row/FP8_BLOCK picks the+ * scale row, one scale per FP8_BLOCK-wide slice of I, double-accumulated+ * across blocks like matmul_fp8 to avoid unfairly penalizing cross-block+ * cancellation (widen-then-multiply, a+=(double)acc*sc -- the same rounding+ * as matmul_fp8 and this function's grouped fmt=4 arm; fmt=5's arm rounds+ * differently, multiplying in float before widening). */+ else if(t->fmt==8){ const uint8_t *w=(const uint8_t*)t->q8+(int64_t)row*I;+ int64_t nblkI=fp8_nblk(I), blkO=(int64_t)row/FP8_BLOCK;+ const float *scl=t->s+blkO*nblkI;+ for(int64_t bi=0; bi*FP8_BLOCK<I; bi++){+ int base=(int)(bi*FP8_BLOCK); int blen=FP8_BLOCK; if(base+blen>I) blen=I-base;+ float sc=fp8_block_scale(scl[bi],blkO,bi,"qt_matvec_rows"); float acc=0;+ for(int i=base;i<base+blen;i++) acc+=e4m3_decode(w[i])*x[i];+ a+=(double)acc*sc; } }/* fmt=3 (int2 packed, per-row scale) is the only fmt this final arm legitimately- * handles -- fmt 0/4/5 matched above, fmt 1/2 have their own explicit branches+ * handles -- fmt 0/4/5/8 matched above, fmt 1/2 have their own explicit branches* above too. Same GUARD and same reasoning as qt_addrow's (fix round 2, engine- * defect): fmt=6's t->s is a fixed 4-byte tag (t->s[row] overreads for row>0),- * fmt=8's t->s holds per-128x128-block floats (t->s[row] overreads for- * row>=nblk) and t->q4 is NULL for fmt=8 -- both would have silently misread or- * crashed here exactly like qt_addrow did before its own fix; refuse instead. */+ * defect): fmt=6's t->s is a fixed 4-byte tag (t->s[row] overreads for row>0) --+ * it would silently misread or crash here exactly like qt_addrow did before its+ * fix; refuse instead. fmt=8 previously fell into this same trap and now has its+ * own branch above instead. */else if(t->fmt==3){ const uint8_t *w=t->q4+(int64_t)row*((I+3)/4); float s=t->s[row]; float acc=0;for(int i=0;i<I;i++){ uint8_t b=w[i>>2]; acc+=((int)((b>>((i&3)*2))&3)-2)*x[i]; } a=acc*s; }else {fprintf(stderr,"qt_matvec_rows: unsupported fmt=%d for the per-row-scale absorb "- "path (only fmt 0/1/2/3/4/5 are handled) -- refusing rather than misread "+ "path (only fmt 0/1/2/3/4/5/8 are handled) -- refusing rather than misread ""t->s[row]/t->q4\n", t->fmt);exit(1);}diff --git a/c/fp8_format.h b/c/fp8_format.hindex 4c465c2b..04610efe 100644--- a/c/fp8_format.h+++ b/c/fp8_format.h@@ -9,10 +9,10 @@* restating 128 / >>7 as literals.** The in-repo FP8 tests are NOT independent of this constant: the reference- * decoders in test_fp8_passthrough.c, test_fp8_load.c, test_fp8_e2e_loader.c- * and test_qwen38_native_weights.c all index with FP8_BLOCK/fp8_nblk via- * quant.h, so they move in lockstep with an edit here (only- * test_backend_metal.mm's ref_fp8_nblk keeps its own literal, on purpose).+ * decoders in test_fp8_passthrough.c, test_fp8_load.c, test_fp8_e2e_loader.c,+ * test_qwen38_native_weights.c and test_qt_addrow.c all index with+ * FP8_BLOCK/fp8_nblk via quant.h, so they move in lockstep with an edit here+ * (only test_backend_metal.mm's ref_fp8_nblk keeps its own literal, on purpose).* An edit to FP8_BLOCK is a format change, not a tunable -- those tests* cannot catch a wrong edit on their own. */#ifndef COLI_FP8_FORMAT_Hdiff --git a/c/tests/test_qt_addrow.c b/c/tests/test_qt_addrow.cindex 46c3673c..254c9e77 100644--- a/c/tests/test_qt_addrow.c+++ b/c/tests/test_qt_addrow.c@@ -4,31 +4,35 @@* fmt 0/4/5 explicitly, then fall through assuming a PER-ROW scale (t->s[row]) followed by* fmt=1/2/3 (qt_addrow) or fmt=0/1/2/3/4/5 (qt_matvec_rows, via an if/else-if chain ending* in a bare `else`) -- nothing stopped fmt=6 (E8/IQ3, t->s is a FIXED 4-byte tag, not O- * floats) or fmt=8 (fp8-e4m3-b128, t->s holds per-128x128-block floats, not O; t->q4 is- * NULL) from reaching that fall-through. For fmt=8 specifically this SIGSEGVs: t->s[row]- * overreads (silently, usually not fatal on its own), then the untouched tail computes- * `t->q4+(int64_t)row*((I+3)/4)` on a NULL t->q4 and dereferences it. For fmt=6 it silently- * misreads the real E8/IQ3 lattice bytes as int2-packed data (same bug SHAPE as #298's CUDA- * absorb-kernel fix, which is why this file's own fmt=4/5 branches exist -- fmt=6 was simply- * missed). Both functions now refuse loudly (exit(1), naming the function and the fmt) for- * any fmt they don't explicitly handle, matching qt_resolve_fmt's own "refuse rather than- * misread" discipline.+ * floats) from reaching that fall-through, where it silently misreads the real E8/IQ3+ * lattice bytes as int2-packed data (same bug SHAPE as #298's CUDA absorb-kernel fix, which+ * is why this file's own fmt=4/5 branches exist -- fmt=6 was simply missed). fmt=6 has no+ * decoder in either function and still refuses loudly (exit(1), naming the function and the+ * fmt), matching qt_resolve_fmt's own "refuse rather than misread" discipline.*- * This file: (1) proves the refusal fires for fmt=6 and fmt=8 through BOTH functions- * (fork+pipe+waitpid, this suite's established house pattern for exit(1)-terminated paths --- * see tests/test_fp8_load.c's expect_refuse/expect_stamp_refuse); (2) proves every format- * BOTH functions still legitimately handle (0/1/2/3/4/5) produces byte-identical results- * against an independently-written reference dequantizer (qt_dequant_row_ref below -- NOT- * copy-pasted from qt_addrow/qt_matvec_rows, restructured as a single per-element loop per- * format, so a real regression in either the guard's placement or the untouched per-fmt math- * would show up here, not just tautologically re-run the same code). Reachability note (not- * a scope excuse, just context): both functions serve ONLY the kv_b absorb path- * (attention_rows/decode call sites), and tools/repack_fp8_passthrough.py deliberately- * excludes kv_b_proj from fmt=8 repacking -- so this fires only via a hand-slotted or- * ambiguous-collision container, not this repo's own tooling's own output. Crash-instead-of-- * refuse is still a real defect (spec I6: loud failure, every refusal names its condition),- * and the fmt=8 QT surface these functions can now be handed is one this same PR pair- * created. */+ * fmt=8 (fp8-e4m3-b128) USED to hit this same fall-through and SIGSEGV (t->s[row] overread,+ * then a NULL t->q4 dereference -- see git history for the pre-fix account) but now has its+ * own explicit branch in both functions (the absorb-decode commit this file accompanies):+ * t->s holds per-128x128-block scales, t->q8 holds raw e4m3 bytes (t->q4 stays NULL), decoded+ * through the same e4m3 LUT / block-scale geometry (quant.h's e4m3_decode/fp8_nblk/FP8_BLOCK)+ * that matmul_fp8 already exercises on the dense/expert path. This file now (1) proves fmt=8+ * produces correct, tolerance-bounded output through BOTH functions -- an exact-dequant check+ * against an independent per-element reference (qt_dequant_row_ref's new fmt==8 branch,+ * disjoint code shape from qt_addrow/qt_matvec_rows' block-batched loops) plus a direct parity+ * check against matmul_fp8 (the proven non-absorb fp8 reference) on a kv_b-shaped tensor; (2)+ * proves the refusal still fires for fmt=6 through both functions (fork+pipe+waitpid, this+ * suite's established house pattern for exit(1)-terminated paths -- see+ * tests/test_fp8_load.c's expect_refuse/expect_stamp_refuse); (3) proves every other format+ * both functions handle (0/1/2/3/4/5) produces byte-identical results against the same+ * independent reference dequantizer, so a real regression in either the guard's placement or+ * the untouched per-fmt math would show up here, not just tautologically re-run the same code.+ * Reachability note (not a scope excuse, just context): both functions serve ONLY the kv_b+ * absorb path (attention_rows/decode call sites), and the repo's own repack tool mints+ * exactly the container these arms decode -- tools/repack_fp8_passthrough.py emits kv_b_proj+ * (kind "kvb", in its RESIDENT_KINDS/STAMPABLE_KINDS sets) as byte-preserved fmt=8 with a+ * stamped scale sidecar, and attention_rows' `int absorb = kvs || ...` makes absorb+ * unbypassable on the batched serving path. These arms are the decode support that+ * container needs. */#define main coli_glm_main_unused#include "../colibri.c"#undef main@@ -48,13 +52,22 @@ static uint64_t rng = 0xA11CE5EEDF00Dull;static uint8_t rndbyte(void){ rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17; return (uint8_t)(rng & 0xFF); }static float rndsmallf(void){ rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17;return ((int64_t)(rng & 0xFFF) - 0x800) / (float)0x800; } /* [-1,1)-ish, small magnitude */+/* fmt=8 fixtures must avoid the two NaN byte patterns (0x7F/0xFF, see quant.h's E4M3_LUT+ * comment) -- a NaN weight is a real, policy-accepted outcome in production, but it would+ * make this file's relative-error comparisons meaningless (NaN != NaN), same reasoning+ * tests/test_fp8_load.c's rndbyte_nonan documents. */+static uint8_t rndbyte_nonan(void){+ for(;;){ uint8_t b=rndbyte(); if(b!=0x7F && b!=0xFF) return b; }+}/* ---- independent reference dequantizer: W[row,:] as floats, one element per loop* iteration -- deliberately NOT the same code shape as qt_addrow/qt_matvec_rows (those* unroll pairs for fmt=2/3, split low/high planes for fmt=5) so this genuinely* cross-checks the production math, not just the production code running twice. ---- */+static void qt_dequant_row_ref_fmt8(const QT *t, int row, float *out);static void qt_dequant_row_ref(const QT *t, int row, float *out){int I=t->I;+ if(t->fmt==8){ qt_dequant_row_ref_fmt8(t,row,out); return; }if(t->fmt==0){ const float *w=t->qf+(int64_t)row*I; for(int i=0;i<I;i++) out[i]=w[i]; return; }if(t->fmt==4){const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); int gs=t->gs, ng=(I+gs-1)/gs;@@ -80,6 +93,22 @@ static void qt_dequant_row_ref(const QT *t, int row, float *out){{ const uint8_t *w=t->q4+(int64_t)row*((I+3)/4);for(int i=0;i<I;i++){ uint8_t b=w[i>>2]; int v=(b>>((i&3)*2))&3; out[i]=((int)v-2)*s; } }}+/* fmt=8 (fp8-e4m3-b128) independent reference: a flat per-ELEMENT loop, deliberately NOT+ * the block-batched-accumulation shape qt_addrow/qt_matvec_rows and matmul_fp8 all share --+ * this recomputes each element's block index and looks up its scale independently, on every+ * iteration, rather than walking block-by-block. Reuses quant.h's e4m3_decode/fp8_nblk/+ * FP8_BLOCK constants deliberately: those ARE the declared format geometry under test here+ * (the same LUT/geometry every fmt=8 consumer in the tree shares), not implementation+ * detail this reference should reinvent. */+static void qt_dequant_row_ref_fmt8(const QT *t, int row, float *out){+ int I=t->I;+ int64_t nblkI=fp8_nblk(I), blkO=(int64_t)row/FP8_BLOCK;+ const float *scl=t->s+blkO*nblkI;+ for(int i=0;i<I;i++){+ int64_t bi=(int64_t)i/FP8_BLOCK;+ out[i]=e4m3_decode((uint8_t)t->q8[(int64_t)row*I+i])*scl[bi];+ }+}/* ---- fixture builders: one per format, deterministic pseudo-random payload ---- */static void fill_fmt0(QT *t, int O, int I){@@ -126,6 +155,17 @@ static void fill_fmt5(QT *t, int O, int I){for(int i=0;i<O*rb;i++) t->q4[i]=rndbyte();for(int i=0;i<O*ng;i++) t->s[i]=0.025f+0.0003f*(float)i;}+/* kv_b-shaped fmt=8 fixture: I is left caller-chosen so callers can pick both the+ * real contraction dimension (kv_lora_rank=512, GLM-5.2's MLA latent width) and+ * small tail-covering shapes. */+static void fill_fmt8(QT *t, int O, int I){+ t->fmt=8; t->O=O; t->I=I; t->gs=0;+ int64_t nblkO=fp8_nblk(O), nblkI=fp8_nblk(I), nblk=nblkO*nblkI;+ t->q8=(int8_t*)malloc((size_t)O*I);+ t->s=(float*)malloc((size_t)nblk*sizeof(float));+ for(int64_t i=0;i<(int64_t)O*I;i++) t->q8[i]=(int8_t)rndbyte_nonan();+ for(int64_t i=0;i<nblk;i++) t->s[i]=0.01f+0.001f*(float)i;+}static void free_qt(QT *t){ free(t->qf); free(t->q8); free(t->q4); free(t->s); memset(t,0,sizeof *t); }/* ---- byte-identity: qt_addrow / qt_matvec_rows vs the independent reference, every@@ -185,22 +225,113 @@ static void test_byte_identity_all_formats(void){memset(&t,0,sizeof t); fill_fmt3(&t,4,17); check_addrow_identity(&t,"fmt=3"); check_matvec_identity(&t,"fmt=3"); free_qt(&t);memset(&t,0,sizeof t); fill_fmt4(&t,4,40,16);check_addrow_identity(&t,"fmt=4"); check_matvec_identity(&t,"fmt=4"); free_qt(&t);memset(&t,0,sizeof t); fill_fmt5(&t,4,130); check_addrow_identity(&t,"fmt=5"); check_matvec_identity(&t,"fmt=5"); free_qt(&t);+ /* fmt=8: small single-block-both-dims shape (mirrors the other formats' O=4-ish+ * defaults) plus a kv_b-shaped multi-block shape (O=192 -> nblkO=2 with a partial+ * tail block; I=512=kv_lora_rank -> nblkI=4, exact blocks) so both the partial-row-+ * block and the multi-column-block paths through the new branches are exercised. */+ memset(&t,0,sizeof t); fill_fmt8(&t,6,17); check_addrow_identity(&t,"fmt=8 (single block)"); check_matvec_identity(&t,"fmt=8 (single block)"); free_qt(&t);+ memset(&t,0,sizeof t); fill_fmt8(&t,192,512); check_addrow_identity(&t,"fmt=8 (kv_b-shaped, multi-block)"); check_matvec_identity(&t,"fmt=8 (kv_b-shaped, multi-block)"); free_qt(&t);+ /* nblkI>=2 WITH a partial COLUMN tail: I=200 -> nblkI=2 with a 72-wide tail block,+ * O=130 -> nblkO=2 with a 2-row tail. The kv_b-shaped case above has I=512 (exact+ * column blocks), so the per-row scale STRIDE (nblkI) and the bi block-scale index+ * were only ever exercised at shapes where flooring/misdeviating them is invisible.+ * Mutation-checked: corrupting the block-scale index math (nblkI=I/FP8_BLOCK, the+ * floor) passes every fmt=8 case above and fails exactly this one. */+ memset(&t,0,sizeof t); fill_fmt8(&t,130,200); check_addrow_identity(&t,"fmt=8 (partial column tail)"); check_matvec_identity(&t,"fmt=8 (partial column tail)"); free_qt(&t);+}++/* ---- fmt=8 absorb-path vs the proven non-absorb matmul_fp8 reference (quant.h) ----+ * qt_matvec_rows(t,row,1,x,&y) and matmul_fp8's per-output-row computation are the SAME+ * mathematical quantity (y[row] = sum_i x[i]*dequant(W[row,i])) computed with the exact+ * same block-scale geometry and the exact same accumulation order (float-accumulate within+ * a 128-wide block, double-accumulate the per-block partials) -- both were written to+ * mirror that convention deliberately (see this PR's qt_matvec_rows fmt=8 branch comment+ * in colibri.c). Compiled in the same translation unit with the same flags, so the two are+ * expected to agree tightly; the tolerance below is a guard against incidental FP-contraction+ * differences between the two call sites, not evidence of a real algorithmic mismatch. */+static void test_fmt8_matmul_fp8_parity(void){+ enum { O=192, I=512 }; /* kv_b-shaped: I=kv_lora_rank=512, O spans a partial row-block */+ QT t; memset(&t,0,sizeof t); fill_fmt8(&t,O,I);+ float *x=(float*)malloc((size_t)I*sizeof(float));+ for(int i=0;i<I;i++) x[i]=rndsmallf();+ float *yref=(float*)malloc((size_t)O*sizeof(float));+ matmul_fp8(yref,x,(const uint8_t*)t.q8,t.s,1,I,O);+ for(int row=0; row<O; row++){+ float y=0.f;+ qt_matvec_rows(&t,row,1,x,&y);+ float want=yref[row];+ float ae=fabsf(y-want);+ float rel = fabsf(want)>1e-6f ? ae/fabsf(want) : ae;+ if(rel > 1e-5f){+ printf("FAIL fmt=8 matmul_fp8 parity: row=%d got=%.9g want=%.9g rel=%.3g\n",+ row,(double)y,(double)want,(double)rel);+ fails++;+ }+ }+ free(x); free(yref); free_qt(&t);+}++/* ---- fmt=8 NaN propagation (policy pin -- see quant.h's "NaN POLICY" note): a NaN+ * weight byte (0x7F/0xFF) decodes to a real IEEE NaN and is left to PROPAGATE, relying+ * on the tested downstream sampler net (tests/test_logit_nan.c), never scrubbed at the+ * weight level. The identity checks above deliberately exclude NaN bytes+ * (rndbyte_nonan) -- and would pass NaN lanes silently anyway (NaN > eps compares+ * false) -- so this pins the absorb-path behavior explicitly against the independent+ * reference: qt_addrow poisons exactly the accumulator lanes whose reference dequant is+ * NaN (per-element accumulate, both NaN byte codes), qt_matvec_rows poisons the whole+ * dot product of any row containing one, and clean rows/lanes of the same tensor stay+ * NaN-free and tolerance-identical to the reference. ---- */+static void test_fmt8_nan_propagation(void){+ enum { O=130, I=200 }; /* same tail-covering shape as the identity case above */+ QT t; memset(&t,0,sizeof t); fill_fmt8(&t,O,I);+ /* one NaN of EACH byte code, in different row-blocks and different column blocks;+ * (129,199) lands in the tail row-block x tail column-block corner */+ const int nan_row[2]={3,129}, nan_col[2]={5,199};+ t.q8[(int64_t)nan_row[0]*I+nan_col[0]]=(int8_t)0x7F;+ t.q8[(int64_t)nan_row[1]*I+nan_col[1]]=(int8_t)0xFF;+ float *ref=(float*)malloc((size_t)I*sizeof(float));+ float *acc=(float*)malloc((size_t)I*sizeof(float));+ float *x=(float*)malloc((size_t)I*sizeof(float));+ for(int i=0;i<I;i++) x[i]=rndsmallf();+ float coef=1.7f; /* != 1, same reasoning as check_addrow_identity */+ for(int r=0;r<2;r++){+ int row=nan_row[r];+ qt_dequant_row_ref(&t,row,ref);+ CHECK(isnan(ref[nan_col[r]])); /* the reference itself says: this lane is NaN */+ memset(acc,0,(size_t)I*sizeof(float));+ qt_addrow(&t,row,coef,acc);+ for(int i=0;i<I;i++){+ if(isnan(ref[i])){+ if(!isnan(acc[i])){ printf("FAIL fmt=8 NaN: qt_addrow row=%d i=%d expected NaN, got %.9g\n",row,i,(double)acc[i]); fails++; }+ } else if(isnan(acc[i])){+ printf("FAIL fmt=8 NaN: qt_addrow row=%d i=%d poisoned a clean lane\n",row,i); fails++;+ } else {+ float want=coef*ref[i];+ float ae=fabsf(acc[i]-want);+ float rel=fabsf(want)>1e-6f?ae/fabsf(want):ae;+ if(rel>1e-5f){ printf("FAIL fmt=8 NaN: qt_addrow row=%d i=%d clean lane got=%.9g want=%.9g\n",row,i,(double)acc[i],(double)want); fails++; }+ }+ }+ float y=0.f; qt_matvec_rows(&t,row,1,x,&y);+ CHECK(isnan(y)); /* dot product over a NaN-bearing row is NaN, like the reference sum */+ }+ /* a clean row of the SAME tensor stays NaN-free through both functions */+ { int row=4; float y=0.f;+ qt_dequant_row_ref(&t,row,ref);+ for(int i=0;i<I;i++) CHECK(!isnan(ref[i]));+ memset(acc,0,(size_t)I*sizeof(float)); qt_addrow(&t,row,coef,acc);+ for(int i=0;i<I;i++) CHECK(!isnan(acc[i]));+ qt_matvec_rows(&t,row,1,x,&y); CHECK(!isnan(y)); }+ free(ref); free(acc); free(x); free_qt(&t);}/* ---- refusal: fork+pipe+waitpid, this suite's house pattern (see test_fp8_load.c's* expect_refuse/expect_stamp_refuse) -- must exit(1) with a "refus"-containing message,* never reach the caller's continuation, never crash with a signal. ---- */+/* fmt=8 no longer belongs in this refusal suite -- it has its own decoder now (see the+ * byte-identity and matmul_fp8-parity tests above). fmt=6 (E8/IQ3) has no absorb decoder+ * and must keep refusing loudly through both functions. */typedef void (*absorb_fn)(void);-static void call_addrow_fmt8(void){- QT t; memset(&t,0,sizeof t);- enum { O=130, I=130 }; /* the coordinator's own repro shape: nblkO=nblkI=2, nblk=4 */- static uint8_t q8[O*I]; static float s[4];- for(int i=0;i<O*I;i++) q8[i]=rndbyte();- for(int i=0;i<4;i++) s[i]=0.01f;- t.fmt=8; t.O=O; t.I=I; t.gs=0; t.q8=(int8_t*)q8; t.s=s;- float acc[I]; memset(acc,0,sizeof acc);- qt_addrow(&t,0,1.f,acc); /* must exit(1) inside; must NOT return */-}static void call_addrow_fmt6(void){QT t; memset(&t,0,sizeof t);enum { O=4, I=98 };@@ -211,17 +342,6 @@ static void call_addrow_fmt6(void){float acc[I]; memset(acc,0,sizeof acc);qt_addrow(&t,0,1.f,acc);}-static void call_matvec_fmt8(void){- QT t; memset(&t,0,sizeof t);- enum { O=130, I=130 };- static uint8_t q8[O*I]; static float s[4];- for(int i=0;i<O*I;i++) q8[i]=rndbyte();- for(int i=0;i<4;i++) s[i]=0.01f;- t.fmt=8; t.O=O; t.I=I; t.gs=0; t.q8=(int8_t*)q8; t.s=s;- static float x[I]; for(int i=0;i<I;i++) x[i]=rndsmallf();- float y=0.f;- qt_matvec_rows(&t,0,1,x,&y);-}static void call_matvec_fmt6(void){QT t; memset(&t,0,sizeof t);enum { O=4, I=98 };@@ -270,15 +390,60 @@ static int expect_refuse_call(absorb_fn fn, const char *tag){#endif}+/* fmt=8 block scales that cannot be multiplied through. A NaN scale poisons+ * every accumulator downstream of it; a ZERO scale emits an all-zero block,+ * which is the same output a decode against an unwritten table produces -- so+ * both are refused by name rather than propagated. One probe per value per+ * function; the scale is poisoned AFTER fill_fmt8 has built a valid tensor, so+ * the only thing under test is the new guard. */+static void poison_scale(QT *t, float v){ t->s[0]=v; }++static void call_addrow_fmt8_nan(void){+ QT t; memset(&t,0,sizeof t); fill_fmt8(&t,6,17); poison_scale(&t,(float)NAN);+ float acc[17]; memset(acc,0,sizeof acc);+ qt_addrow(&t,0,1.f,acc);+}+static void call_matvec_fmt8_nan(void){+ QT t; memset(&t,0,sizeof t); fill_fmt8(&t,6,17); poison_scale(&t,(float)NAN);+ static float x[17]; for(int i=0;i<17;i++) x[i]=rndsmallf();+ float y=0.f; qt_matvec_rows(&t,0,1,x,&y);+}+/* A ZERO block scale is valid data, not corruption: block scales are amax/448,+ * so an all-zero block legitimately carries one. Both arms must decode it to+ * zeros in place -- not refuse it, and not leave the accumulator untouched.+ * These two assertions exist to keep a zero refusal from being re-added. */+static void test_fmt8_zero_scale_decodes_to_zeros(void){+ enum { O=6, I=17 };+ { QT t; memset(&t,0,sizeof t); fill_fmt8(&t,O,I); poison_scale(&t,0.f);+ float acc[I]; for(int i=0;i<I;i++) acc[i]=1.5f; /* pre-loaded: axpy must ADD zero */+ qt_addrow(&t,0,1.f,acc);+ int ok=1; for(int i=0;i<I;i++) if(acc[i]!=1.5f) ok=0;+ if(!ok) printf("FAIL qt_addrow: zero fmt=8 block scale did not add zeros\n");+ CHECK(ok);+ free_qt(&t);+ }+ { QT t; memset(&t,0,sizeof t); fill_fmt8(&t,O,I); poison_scale(&t,0.f);+ static float x[I]; for(int i=0;i<I;i++) x[i]=rndsmallf();+ float y=42.f; qt_matvec_rows(&t,0,1,x,&y);+ int ok = (y==0.f);+ if(!ok) printf("FAIL qt_matvec_rows: zero fmt=8 block scale gave %g, expected 0\n",(double)y);+ CHECK(ok);+ free_qt(&t);+ }+}+static void test_refusals(void){- CHECK(expect_refuse_call(call_addrow_fmt8, "qt_addrow refuses fmt=8 (was SIGSEGV)"));CHECK(expect_refuse_call(call_addrow_fmt6, "qt_addrow refuses fmt=6"));- CHECK(expect_refuse_call(call_matvec_fmt8, "qt_matvec_rows refuses fmt=8"));CHECK(expect_refuse_call(call_matvec_fmt6, "qt_matvec_rows refuses fmt=6"));+ CHECK(expect_refuse_call(call_addrow_fmt8_nan, "qt_addrow refuses a NaN fmt=8 block scale"));+ CHECK(expect_refuse_call(call_matvec_fmt8_nan, "qt_matvec_rows refuses a NaN fmt=8 block scale"));}int main(void){test_byte_identity_all_formats();+ test_fmt8_matmul_fp8_parity();+ test_fmt8_nan_propagation();+ test_fmt8_zero_scale_decodes_to_zeros();test_refusals();if(fails){ printf("qt_addrow/qt_matvec_rows tests: %d FAILED\n", fails); return 1; }printf("qt_addrow/qt_matvec_rows tests: ok\n");diff --git a/c/tools/repack_fp8_passthrough.py b/c/tools/repack_fp8_passthrough.pyindex 97c06156..3f0ff07c 100644--- a/c/tools/repack_fp8_passthrough.py+++ b/c/tools/repack_fp8_passthrough.py@@ -113,27 +113,22 @@ read side. `main()` also copies `config.json` (mandatory, colibri.c'sload_cfg) and `generation_config.json` (best-effort) from --indir into--outdir. Together with kv_b_proj emission below, this makes the tool'soutput a standalone-loadable model directory on its own, with one caveat: the-engine's fmt=8 absorb support (branch `f8/absorb-fmt8`) must land before a-kv_b_proj-carrying container is safe to decode through the batched serving-path -- see below.+CUDA absorb kernels do not yet decode fmt=8, so a kv_b_proj-carrying+container decodes through the CPU absorb path only -- see below.kv_b_proj (kind "kvb") is now emitted the same way as o_proj/attn: byte-preserved fp8 weight + renamed `.qs` scale sidecar, stamped (it clears thecollision shape the same way o_proj does -- M1 audit, GLM-5.2's kv_b_proj is-never at the ambiguous [O,I] this tool's `_check_geometry` guards). THIS TOOL-CHANGE IS INDEPENDENT of, and does not wait on, the engine-side absorb work:-colibri.c's MLA-absorption CPU path (qt_addrow/qt_matvec_rows, called only on-l->kv_b) and the CUDA absorb kernels have no fmt==8 case as of this tool's own-HEAD -- that support is being built in parallel on `f8/absorb-fmt8` and is-NOT part of this diff. A container minted with this tool BEFORE that engine-branch lands will load clean (qt_from_disk resolves the stamp exactly like-o_proj's) but crash -- loudly, not silently -- on any decode that reaches the-batched (`kvs`-nonNULL) serving path, because `ABSORB=0` cannot bypass absorb-there. Sequencing that correctly (engine work before this container is used-for decode) is the gate's job, not this tool's: repacking kv_b_proj here is-the tool-side half of a two-sided integration, done now because engine and-tool work can (and per the gate's assembly manifest, should) proceed in-parallel.+never at the ambiguous [O,I] this tool's `_check_geometry` guards). The+engine-side half of this two-sided integration now exists on the CPU:+colibri.c's MLA-absorption path (qt_addrow/qt_matvec_rows, called only on+l->kv_b) carries explicit fmt==8 decode arms, so a container minted here+loads clean (qt_from_disk resolves the stamp exactly like o_proj's) and+decodes correctly through the batched (`kvs`-nonNULL) serving path, where+absorb cannot be bypassed. The CUDA absorb kernels still have no fmt==8+case (absorb_fmt_ok delegates to coli_cuda_weight_at_supported's fmt 0..4+allowlist), so fmt=8 kv_b decode stays on the CPU absorb path until that+lands.METADATA STAMP (reference implementation of the FORMATS-registry FR -- seedocs/FORMATS.md): every output shard's safetensors `__metadata__` carries a