Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(colibri): add fmt=8 arms to the CPU absorb path

monotophic0ad3eef77066
diff --git a/c/colibri.c b/c/colibri.c
index 64bf3680..8ffd06bd 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -2266,6 +2266,43 @@ static void qt_cuda_colocate(QT *dst,const QT *src){
}
static void layer_cuda_shard_kvb(Layer *l,int H,int Q,int V){
if(!g_cuda_enabled||!g_cuda_dense||g_cuda_ndev<2||l->kv_b.fmt==0)return;
+ /* SHARD FORMAT ALLOWLIST (explicit refusal; this was an ACCIDENTAL fail-safe): the
+ * rb/weights/scale arithmetic below is written for exactly fmt=1 (int8, per-row
+ * scale), fmt=2 (int4 per-row), fmt=3 (int2 per-row) and fmt=4 (int4 grouped).
+ * Any other fmt reaching it computes a wrong row-byte stride, takes l->kv_b.q4 as
+ * the weight pointer (NULL for fmt=8, whose raw e4m3 bytes live in q8 -- see the
+ * QT struct comment), and slices l->kv_b.s with per-row/per-group geometry that
+ * fmt=8's per-128x128-BLOCK scales (and fmt=6's single 4-byte tag) simply do not
+ * have. fmt=8 only ever "worked" here by accident: q4==NULL made
+ * coli_cuda_tensor_upload_g's !weights check reject the upload before anything
+ * dereferenced it -- silent, unnamed, and one refactor away from a misread.
+ * Refuse BY NAME instead, BEFORE any pointer/stride use, and say what happens
+ * instead: the un-sharded kv_b stays whole on its layer home device, where fmt=8
+ * kv_b decode runs the CPU absorb path (qt_addrow/qt_matvec_rows' fmt=8
+ * branches below; the CUDA absorb kernels refuse fmt=8 via absorb_fmt_ok's
+ * fmt 0..4 allowlist) -- COLI_CUDA_ATTN_SHARD is a no-op for it. Same
+ * "refuse rather than misread" discipline as qt_addrow/
+ * qt_matvec_rows' guards; notice only (no exit): sharding is an opt-in
+ * optimization and skipping it is the correct, working behavior. Bounded once
+ * per process per fmt, never per layer (metal_fmt_gate_notice, the precedent
+ * for bounded notices, is coarser still: one line per tensor KIND, naming only
+ * the first offending fmt). */
+ if(l->kv_b.fmt!=1&&l->kv_b.fmt!=2&&l->kv_b.fmt!=3&&l->kv_b.fmt!=4){
+ static int refused_fmt[32];
+ if(!refused_fmt[l->kv_b.fmt&31]){ refused_fmt[l->kv_b.fmt&31]=1;
+ if(l->kv_b.fmt==8)
+ fprintf(stderr,"layer_cuda_shard_kvb: kv_b fmt=8 (fp8-e4m3, per-128x128-block "
+ "scales) has no head-shard layout here -- refusing the shard; fmt=8 kv_b "
+ "runs the absorb path on the layer home device instead, so "
+ "COLI_CUDA_ATTN_SHARD is a no-op for it (applies to every layer)\n");
+ else
+ fprintf(stderr,"layer_cuda_shard_kvb: unsupported kv_b fmt=%d for the head-shard "
+ "upload (only fmt 1/2/3/4 match the per-row byte/scale strides computed "
+ "here) -- refusing the shard; kv_b stays whole on its layer home device "
+ "(applies to every layer)\n",l->kv_b.fmt);
+ }
+ return;
+ }
int rb=l->kv_b.fmt==1?l->kv_b.I:
(l->kv_b.fmt==2||l->kv_b.fmt==4)?(l->kv_b.I+1)/2:(l->kv_b.I+3)/4;
const uint8_t *weights=l->kv_b.fmt==1?(const uint8_t*)l->kv_b.q8:l->kv_b.q4;
@@ -3858,6 +3895,32 @@ static void expert_prefetch(Model *m, int layer, int eid){
/* ---- helper per l'ABSORPTION: accesso per-riga ai QT quantizzati ---- */
/* acc[0..I) += coef * W[row,:] (dequant al volo) */
+/* One fmt=8 block scale, checked before it multiplies anything.
+ *
+ * A NaN scale has no safe interpretation: it poisons the whole block and every
+ * accumulator downstream of it, and this function cannot repair it, so it is
+ * refused by name with the block that carried it -- the same "refuse rather
+ * than misread" discipline the format guards below apply.
+ *
+ * A ZERO scale is NOT refused: it is valid data. Block scales are amax/448, so
+ * a genuinely all-zero block (padding, an unused slice) legitimately produces
+ * zero, and decoding it as zeros is the correct answer. It cannot be confused
+ * with a decode against an unwritten table the way it can on the GPU, because
+ * there is no table here to be unwritten -- the CPU decoder reads its e4m3
+ * values from a compile-time constant table, which is why the LUT-ready gate
+ * exists only on the CUDA side. Do not re-add a zero refusal: it would reject
+ * valid checkpoints.
+ *
+ * The check is per BLOCK, not per element: one branch per FP8_BLOCK columns. */
+static float fp8_block_scale(float sc, int64_t blkO, int64_t bi, const char *who){
+ if(isnan(sc)){
+ fprintf(stderr,"%s: fmt=8 scale block [%lld,%lld] is NaN -- refusing rather than "
+ "propagate it through the absorb accumulator\n",who,(long long)blkO,(long long)bi);
+ exit(1);
+ }
+ return sc;
+}
+
static void qt_addrow(const QT *t, int row, float coef, float *acc){
int I=t->I;
if(t->fmt==0){ const float *w=t->qf+(int64_t)row*I; for(int i=0;i<I;i++) acc[i]+=coef*w[i]; return; }
@@ -3881,8 +3944,23 @@ static void qt_addrow(const QT *t, int row, float coef, float *acc){
for(int k=0;k<n;k++){ unsigned u=((lo[k>>2]>>((k&3)*2))&3)|(((hi[k>>3]>>(k&7))&1)<<2);
acc[base+k]+=cg*(float)((int)u-4); } }
return; }
+ /* fmt=8 (fp8-e4m3-b128, absorb-path support added here): t->s holds ONE f32 scale
+ * per 128x128 BLOCK (ceil(O/128)*ceil(I/128) entries, block-row-major), not O, and
+ * t->q4 is NULL for this format -- raw e4m3 bytes live in t->q8 instead, same
+ * convention as fmt=1 (see the QT struct comment). Mirrors matmul_fp8's (quant.h)
+ * block-scale indexing exactly: blkO=row/FP8_BLOCK selects the scale row, then one
+ * scale per FP8_BLOCK-wide slice of I. This is the branch that used to be missing
+ * -- see the guard below's history note. */
+ if(t->fmt==8){ const uint8_t *w=(const uint8_t*)t->q8+(int64_t)row*I;
+ int64_t nblkI=fp8_nblk(I), blkO=(int64_t)row/FP8_BLOCK;
+ const float *scl=t->s+blkO*nblkI;
+ for(int64_t bi=0; bi*FP8_BLOCK<I; bi++){
+ int base=(int)(bi*FP8_BLOCK); int blen=FP8_BLOCK; if(base+blen>I) blen=I-base;
+ float sc=coef*fp8_block_scale(scl[bi],blkO,bi,"qt_addrow");
+ for(int i=base;i<base+blen;i++) acc[i]+=sc*e4m3_decode(w[i]); }
+ return; }
/* GUARD (fix round 2, engine defect -- clean-room conformance trial found a real
- * SIGSEGV, reproduced): fmt 0/4/5 already returned above; everything below this
+ * SIGSEGV, reproduced): fmt 0/4/5/8 already returned above; everything below this
* point assumes a PER-ROW scale (t->s[row]) followed by fmt=1 (int8, explicit
* branch), fmt=2 (int4 packed, explicit branch), or the tail's own IMPLICIT fmt=3
* (int2 packed, the final unconditional block) -- there was no guard stopping any
@@ -3891,20 +3969,16 @@ static void qt_addrow(const QT *t, int row, float coef, float *acc){
* a heap OVERREAD, and the untouched fall-through then misreads t->q4's real E8
* lattice bytes as int2-packed data (same bug SHAPE as #298's CUDA absorb-kernel
* fix, and the same one this file's own fmt=4/5 branches above were added to
- * dodge -- fmt=6 was simply missed). fmt=8 (fp8-e4m3-b128): t->s holds
- * ceil(O/128)*ceil(I/128) per-block floats, not O -- t->s[row] overreads for
- * row>=nblk (e.g. a [130,130] tensor has nblk=4, so every row past 3 already reads
- * out of bounds), AND t->q4 is NULL for fmt=8 (raw bytes live in t->q8 instead,
- * same convention as fmt=1 -- see the QT struct comment), so the fall-through's
- * `t->q4+(int64_t)row*((I+3)/4)` dereferences NULL-plus-offset: SIGSEGV,
- * reproduced (see the report's proof-of-bite transcript). Refuse loudly instead --
- * this function has no byte-count context of its own to validate against (it only
- * ever sees an already-resolved QT), so "unsupported fmt" is the only check
- * available, same "refuse rather than misread" discipline qt_resolve_fmt applies
- * at load time. */
+ * dodge -- fmt=6 was simply missed). fmt=8 previously landed here too (t->s[row]
+ * overreads past row>=nblk, t->q4 is NULL -> SIGSEGV via the int2 fall-through);
+ * it now returns above via its own branch and never reaches this guard. Refuse
+ * loudly instead for anything else -- this function has no byte-count context of
+ * its own to validate against (it only ever sees an already-resolved QT), so
+ * "unsupported fmt" is the only check available, same "refuse rather than
+ * misread" discipline qt_resolve_fmt applies at load time. */
if(t->fmt!=1 && t->fmt!=2 && t->fmt!=3){
fprintf(stderr,"qt_addrow: unsupported fmt=%d for the per-row-scale absorb path "
- "(only fmt 1/2/3 reach this point; fmt 0/4/5 are handled above and return "
+ "(only fmt 1/2/3 reach this point; fmt 0/4/5/8 are handled above and return "
"before it) -- refusing rather than misread t->s[row]/t->q4\n", t->fmt);
exit(1);
}
@@ -3954,18 +4028,35 @@ static void qt_matvec_rows(const QT *t, int r0, int n, const float *x, float *y)
for(int k=0;k<n;k++){ unsigned u=((lo[k>>2]>>((k&3)*2))&3)|(((hi[k>>3]>>(k&7))&1)<<2);
acc+=(float)((int)u-4)*x[base+k]; }
a+=(double)(acc*sr[g]); } }
+ /* fmt=8 (fp8-e4m3-b128, absorb-path support added here): per-128x128-BLOCK f32
+ * scale, block-row-major (ceil(O/128)*ceil(I/128) entries), raw bytes in t->q8
+ * (t->q4 is NULL for this format). Same block-scale indexing as matmul_fp8
+ * (quant.h) and qt_addrow's fmt=8 branch above: blkO=row/FP8_BLOCK picks the
+ * scale row, one scale per FP8_BLOCK-wide slice of I, double-accumulated
+ * across blocks like matmul_fp8 to avoid unfairly penalizing cross-block
+ * cancellation (widen-then-multiply, a+=(double)acc*sc -- the same rounding
+ * as matmul_fp8 and this function's grouped fmt=4 arm; fmt=5's arm rounds
+ * differently, multiplying in float before widening). */
+ else if(t->fmt==8){ const uint8_t *w=(const uint8_t*)t->q8+(int64_t)row*I;
+ int64_t nblkI=fp8_nblk(I), blkO=(int64_t)row/FP8_BLOCK;
+ const float *scl=t->s+blkO*nblkI;
+ for(int64_t bi=0; bi*FP8_BLOCK<I; bi++){
+ int base=(int)(bi*FP8_BLOCK); int blen=FP8_BLOCK; if(base+blen>I) blen=I-base;
+ float sc=fp8_block_scale(scl[bi],blkO,bi,"qt_matvec_rows"); float acc=0;
+ for(int i=base;i<base+blen;i++) acc+=e4m3_decode(w[i])*x[i];
+ a+=(double)acc*sc; } }
/* fmt=3 (int2 packed, per-row scale) is the only fmt this final arm legitimately
- * handles -- fmt 0/4/5 matched above, fmt 1/2 have their own explicit branches
+ * handles -- fmt 0/4/5/8 matched above, fmt 1/2 have their own explicit branches
* above too. Same GUARD and same reasoning as qt_addrow's (fix round 2, engine
- * defect): fmt=6's t->s is a fixed 4-byte tag (t->s[row] overreads for row>0),
- * fmt=8's t->s holds per-128x128-block floats (t->s[row] overreads for
- * row>=nblk) and t->q4 is NULL for fmt=8 -- both would have silently misread or
- * crashed here exactly like qt_addrow did before its own fix; refuse instead. */
+ * defect): fmt=6's t->s is a fixed 4-byte tag (t->s[row] overreads for row>0) --
+ * it would silently misread or crash here exactly like qt_addrow did before its
+ * fix; refuse instead. fmt=8 previously fell into this same trap and now has its
+ * own branch above instead. */
else if(t->fmt==3){ const uint8_t *w=t->q4+(int64_t)row*((I+3)/4); float s=t->s[row]; float acc=0;
for(int i=0;i<I;i++){ uint8_t b=w[i>>2]; acc+=((int)((b>>((i&3)*2))&3)-2)*x[i]; } a=acc*s; }
else {
fprintf(stderr,"qt_matvec_rows: unsupported fmt=%d for the per-row-scale absorb "
- "path (only fmt 0/1/2/3/4/5 are handled) -- refusing rather than misread "
+ "path (only fmt 0/1/2/3/4/5/8 are handled) -- refusing rather than misread "
"t->s[row]/t->q4\n", t->fmt);
exit(1);
}
diff --git a/c/fp8_format.h b/c/fp8_format.h
index 4c465c2b..04610efe 100644
--- a/c/fp8_format.h
+++ b/c/fp8_format.h
@@ -9,10 +9,10 @@
* restating 128 / >>7 as literals.
*
* The in-repo FP8 tests are NOT independent of this constant: the reference
- * decoders in test_fp8_passthrough.c, test_fp8_load.c, test_fp8_e2e_loader.c
- * and test_qwen38_native_weights.c all index with FP8_BLOCK/fp8_nblk via
- * quant.h, so they move in lockstep with an edit here (only
- * test_backend_metal.mm's ref_fp8_nblk keeps its own literal, on purpose).
+ * decoders in test_fp8_passthrough.c, test_fp8_load.c, test_fp8_e2e_loader.c,
+ * test_qwen38_native_weights.c and test_qt_addrow.c all index with
+ * FP8_BLOCK/fp8_nblk via quant.h, so they move in lockstep with an edit here
+ * (only test_backend_metal.mm's ref_fp8_nblk keeps its own literal, on purpose).
* An edit to FP8_BLOCK is a format change, not a tunable -- those tests
* cannot catch a wrong edit on their own. */
#ifndef COLI_FP8_FORMAT_H
diff --git a/c/tests/test_qt_addrow.c b/c/tests/test_qt_addrow.c
index 46c3673c..254c9e77 100644
--- a/c/tests/test_qt_addrow.c
+++ b/c/tests/test_qt_addrow.c
@@ -4,31 +4,35 @@
* fmt 0/4/5 explicitly, then fall through assuming a PER-ROW scale (t->s[row]) followed by
* fmt=1/2/3 (qt_addrow) or fmt=0/1/2/3/4/5 (qt_matvec_rows, via an if/else-if chain ending
* in a bare `else`) -- nothing stopped fmt=6 (E8/IQ3, t->s is a FIXED 4-byte tag, not O
- * floats) or fmt=8 (fp8-e4m3-b128, t->s holds per-128x128-block floats, not O; t->q4 is
- * NULL) from reaching that fall-through. For fmt=8 specifically this SIGSEGVs: t->s[row]
- * overreads (silently, usually not fatal on its own), then the untouched tail computes
- * `t->q4+(int64_t)row*((I+3)/4)` on a NULL t->q4 and dereferences it. For fmt=6 it silently
- * misreads the real E8/IQ3 lattice bytes as int2-packed data (same bug SHAPE as #298's CUDA
- * absorb-kernel fix, which is why this file's own fmt=4/5 branches exist -- fmt=6 was simply
- * missed). Both functions now refuse loudly (exit(1), naming the function and the fmt) for
- * any fmt they don't explicitly handle, matching qt_resolve_fmt's own "refuse rather than
- * misread" discipline.
+ * floats) from reaching that fall-through, where it silently misreads the real E8/IQ3
+ * lattice bytes as int2-packed data (same bug SHAPE as #298's CUDA absorb-kernel fix, which
+ * is why this file's own fmt=4/5 branches exist -- fmt=6 was simply missed). fmt=6 has no
+ * decoder in either function and still refuses loudly (exit(1), naming the function and the
+ * fmt), matching qt_resolve_fmt's own "refuse rather than misread" discipline.
*
- * This file: (1) proves the refusal fires for fmt=6 and fmt=8 through BOTH functions
- * (fork+pipe+waitpid, this suite's established house pattern for exit(1)-terminated paths --
- * see tests/test_fp8_load.c's expect_refuse/expect_stamp_refuse); (2) proves every format
- * BOTH functions still legitimately handle (0/1/2/3/4/5) produces byte-identical results
- * against an independently-written reference dequantizer (qt_dequant_row_ref below -- NOT
- * copy-pasted from qt_addrow/qt_matvec_rows, restructured as a single per-element loop per
- * format, so a real regression in either the guard's placement or the untouched per-fmt math
- * would show up here, not just tautologically re-run the same code). Reachability note (not
- * a scope excuse, just context): both functions serve ONLY the kv_b absorb path
- * (attention_rows/decode call sites), and tools/repack_fp8_passthrough.py deliberately
- * excludes kv_b_proj from fmt=8 repacking -- so this fires only via a hand-slotted or
- * ambiguous-collision container, not this repo's own tooling's own output. Crash-instead-of-
- * refuse is still a real defect (spec I6: loud failure, every refusal names its condition),
- * and the fmt=8 QT surface these functions can now be handed is one this same PR pair
- * created. */
+ * fmt=8 (fp8-e4m3-b128) USED to hit this same fall-through and SIGSEGV (t->s[row] overread,
+ * then a NULL t->q4 dereference -- see git history for the pre-fix account) but now has its
+ * own explicit branch in both functions (the absorb-decode commit this file accompanies):
+ * t->s holds per-128x128-block scales, t->q8 holds raw e4m3 bytes (t->q4 stays NULL), decoded
+ * through the same e4m3 LUT / block-scale geometry (quant.h's e4m3_decode/fp8_nblk/FP8_BLOCK)
+ * that matmul_fp8 already exercises on the dense/expert path. This file now (1) proves fmt=8
+ * produces correct, tolerance-bounded output through BOTH functions -- an exact-dequant check
+ * against an independent per-element reference (qt_dequant_row_ref's new fmt==8 branch,
+ * disjoint code shape from qt_addrow/qt_matvec_rows' block-batched loops) plus a direct parity
+ * check against matmul_fp8 (the proven non-absorb fp8 reference) on a kv_b-shaped tensor; (2)
+ * proves the refusal still fires for fmt=6 through both functions (fork+pipe+waitpid, this
+ * suite's established house pattern for exit(1)-terminated paths -- see
+ * tests/test_fp8_load.c's expect_refuse/expect_stamp_refuse); (3) proves every other format
+ * both functions handle (0/1/2/3/4/5) produces byte-identical results against the same
+ * independent reference dequantizer, so a real regression in either the guard's placement or
+ * the untouched per-fmt math would show up here, not just tautologically re-run the same code.
+ * Reachability note (not a scope excuse, just context): both functions serve ONLY the kv_b
+ * absorb path (attention_rows/decode call sites), and the repo's own repack tool mints
+ * exactly the container these arms decode -- tools/repack_fp8_passthrough.py emits kv_b_proj
+ * (kind "kvb", in its RESIDENT_KINDS/STAMPABLE_KINDS sets) as byte-preserved fmt=8 with a
+ * stamped scale sidecar, and attention_rows' `int absorb = kvs || ...` makes absorb
+ * unbypassable on the batched serving path. These arms are the decode support that
+ * container needs. */
#define main coli_glm_main_unused
#include "../colibri.c"
#undef main
@@ -48,13 +52,22 @@ static uint64_t rng = 0xA11CE5EEDF00Dull;
static uint8_t rndbyte(void){ rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17; return (uint8_t)(rng & 0xFF); }
static float rndsmallf(void){ rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17;
return ((int64_t)(rng & 0xFFF) - 0x800) / (float)0x800; } /* [-1,1)-ish, small magnitude */
+/* fmt=8 fixtures must avoid the two NaN byte patterns (0x7F/0xFF, see quant.h's E4M3_LUT
+ * comment) -- a NaN weight is a real, policy-accepted outcome in production, but it would
+ * make this file's relative-error comparisons meaningless (NaN != NaN), same reasoning
+ * tests/test_fp8_load.c's rndbyte_nonan documents. */
+static uint8_t rndbyte_nonan(void){
+ for(;;){ uint8_t b=rndbyte(); if(b!=0x7F && b!=0xFF) return b; }
+}
/* ---- independent reference dequantizer: W[row,:] as floats, one element per loop
* iteration -- deliberately NOT the same code shape as qt_addrow/qt_matvec_rows (those
* unroll pairs for fmt=2/3, split low/high planes for fmt=5) so this genuinely
* cross-checks the production math, not just the production code running twice. ---- */
+static void qt_dequant_row_ref_fmt8(const QT *t, int row, float *out);
static void qt_dequant_row_ref(const QT *t, int row, float *out){
int I=t->I;
+ if(t->fmt==8){ qt_dequant_row_ref_fmt8(t,row,out); return; }
if(t->fmt==0){ const float *w=t->qf+(int64_t)row*I; for(int i=0;i<I;i++) out[i]=w[i]; return; }
if(t->fmt==4){
const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); int gs=t->gs, ng=(I+gs-1)/gs;
@@ -80,6 +93,22 @@ static void qt_dequant_row_ref(const QT *t, int row, float *out){
{ const uint8_t *w=t->q4+(int64_t)row*((I+3)/4);
for(int i=0;i<I;i++){ uint8_t b=w[i>>2]; int v=(b>>((i&3)*2))&3; out[i]=((int)v-2)*s; } }
}
+/* fmt=8 (fp8-e4m3-b128) independent reference: a flat per-ELEMENT loop, deliberately NOT
+ * the block-batched-accumulation shape qt_addrow/qt_matvec_rows and matmul_fp8 all share --
+ * this recomputes each element's block index and looks up its scale independently, on every
+ * iteration, rather than walking block-by-block. Reuses quant.h's e4m3_decode/fp8_nblk/
+ * FP8_BLOCK constants deliberately: those ARE the declared format geometry under test here
+ * (the same LUT/geometry every fmt=8 consumer in the tree shares), not implementation
+ * detail this reference should reinvent. */
+static void qt_dequant_row_ref_fmt8(const QT *t, int row, float *out){
+ int I=t->I;
+ int64_t nblkI=fp8_nblk(I), blkO=(int64_t)row/FP8_BLOCK;
+ const float *scl=t->s+blkO*nblkI;
+ for(int i=0;i<I;i++){
+ int64_t bi=(int64_t)i/FP8_BLOCK;
+ out[i]=e4m3_decode((uint8_t)t->q8[(int64_t)row*I+i])*scl[bi];
+ }
+}
/* ---- fixture builders: one per format, deterministic pseudo-random payload ---- */
static void fill_fmt0(QT *t, int O, int I){
@@ -126,6 +155,17 @@ static void fill_fmt5(QT *t, int O, int I){
for(int i=0;i<O*rb;i++) t->q4[i]=rndbyte();
for(int i=0;i<O*ng;i++) t->s[i]=0.025f+0.0003f*(float)i;
}
+/* kv_b-shaped fmt=8 fixture: I is left caller-chosen so callers can pick both the
+ * real contraction dimension (kv_lora_rank=512, GLM-5.2's MLA latent width) and
+ * small tail-covering shapes. */
+static void fill_fmt8(QT *t, int O, int I){
+ t->fmt=8; t->O=O; t->I=I; t->gs=0;
+ int64_t nblkO=fp8_nblk(O), nblkI=fp8_nblk(I), nblk=nblkO*nblkI;
+ t->q8=(int8_t*)malloc((size_t)O*I);
+ t->s=(float*)malloc((size_t)nblk*sizeof(float));
+ for(int64_t i=0;i<(int64_t)O*I;i++) t->q8[i]=(int8_t)rndbyte_nonan();
+ for(int64_t i=0;i<nblk;i++) t->s[i]=0.01f+0.001f*(float)i;
+}
static void free_qt(QT *t){ free(t->qf); free(t->q8); free(t->q4); free(t->s); memset(t,0,sizeof *t); }
/* ---- byte-identity: qt_addrow / qt_matvec_rows vs the independent reference, every
@@ -185,22 +225,113 @@ static void test_byte_identity_all_formats(void){
memset(&t,0,sizeof t); fill_fmt3(&t,4,17); check_addrow_identity(&t,"fmt=3"); check_matvec_identity(&t,"fmt=3"); free_qt(&t);
memset(&t,0,sizeof t); fill_fmt4(&t,4,40,16);check_addrow_identity(&t,"fmt=4"); check_matvec_identity(&t,"fmt=4"); free_qt(&t);
memset(&t,0,sizeof t); fill_fmt5(&t,4,130); check_addrow_identity(&t,"fmt=5"); check_matvec_identity(&t,"fmt=5"); free_qt(&t);
+ /* fmt=8: small single-block-both-dims shape (mirrors the other formats' O=4-ish
+ * defaults) plus a kv_b-shaped multi-block shape (O=192 -> nblkO=2 with a partial
+ * tail block; I=512=kv_lora_rank -> nblkI=4, exact blocks) so both the partial-row-
+ * block and the multi-column-block paths through the new branches are exercised. */
+ memset(&t,0,sizeof t); fill_fmt8(&t,6,17); check_addrow_identity(&t,"fmt=8 (single block)"); check_matvec_identity(&t,"fmt=8 (single block)"); free_qt(&t);
+ memset(&t,0,sizeof t); fill_fmt8(&t,192,512); check_addrow_identity(&t,"fmt=8 (kv_b-shaped, multi-block)"); check_matvec_identity(&t,"fmt=8 (kv_b-shaped, multi-block)"); free_qt(&t);
+ /* nblkI>=2 WITH a partial COLUMN tail: I=200 -> nblkI=2 with a 72-wide tail block,
+ * O=130 -> nblkO=2 with a 2-row tail. The kv_b-shaped case above has I=512 (exact
+ * column blocks), so the per-row scale STRIDE (nblkI) and the bi block-scale index
+ * were only ever exercised at shapes where flooring/misdeviating them is invisible.
+ * Mutation-checked: corrupting the block-scale index math (nblkI=I/FP8_BLOCK, the
+ * floor) passes every fmt=8 case above and fails exactly this one. */
+ memset(&t,0,sizeof t); fill_fmt8(&t,130,200); check_addrow_identity(&t,"fmt=8 (partial column tail)"); check_matvec_identity(&t,"fmt=8 (partial column tail)"); free_qt(&t);
+}
+
+/* ---- fmt=8 absorb-path vs the proven non-absorb matmul_fp8 reference (quant.h) ----
+ * qt_matvec_rows(t,row,1,x,&y) and matmul_fp8's per-output-row computation are the SAME
+ * mathematical quantity (y[row] = sum_i x[i]*dequant(W[row,i])) computed with the exact
+ * same block-scale geometry and the exact same accumulation order (float-accumulate within
+ * a 128-wide block, double-accumulate the per-block partials) -- both were written to
+ * mirror that convention deliberately (see this PR's qt_matvec_rows fmt=8 branch comment
+ * in colibri.c). Compiled in the same translation unit with the same flags, so the two are
+ * expected to agree tightly; the tolerance below is a guard against incidental FP-contraction
+ * differences between the two call sites, not evidence of a real algorithmic mismatch. */
+static void test_fmt8_matmul_fp8_parity(void){
+ enum { O=192, I=512 }; /* kv_b-shaped: I=kv_lora_rank=512, O spans a partial row-block */
+ QT t; memset(&t,0,sizeof t); fill_fmt8(&t,O,I);
+ float *x=(float*)malloc((size_t)I*sizeof(float));
+ for(int i=0;i<I;i++) x[i]=rndsmallf();
+ float *yref=(float*)malloc((size_t)O*sizeof(float));
+ matmul_fp8(yref,x,(const uint8_t*)t.q8,t.s,1,I,O);
+ for(int row=0; row<O; row++){
+ float y=0.f;
+ qt_matvec_rows(&t,row,1,x,&y);
+ float want=yref[row];
+ float ae=fabsf(y-want);
+ float rel = fabsf(want)>1e-6f ? ae/fabsf(want) : ae;
+ if(rel > 1e-5f){
+ printf("FAIL fmt=8 matmul_fp8 parity: row=%d got=%.9g want=%.9g rel=%.3g\n",
+ row,(double)y,(double)want,(double)rel);
+ fails++;
+ }
+ }
+ free(x); free(yref); free_qt(&t);
+}
+
+/* ---- fmt=8 NaN propagation (policy pin -- see quant.h's "NaN POLICY" note): a NaN
+ * weight byte (0x7F/0xFF) decodes to a real IEEE NaN and is left to PROPAGATE, relying
+ * on the tested downstream sampler net (tests/test_logit_nan.c), never scrubbed at the
+ * weight level. The identity checks above deliberately exclude NaN bytes
+ * (rndbyte_nonan) -- and would pass NaN lanes silently anyway (NaN > eps compares
+ * false) -- so this pins the absorb-path behavior explicitly against the independent
+ * reference: qt_addrow poisons exactly the accumulator lanes whose reference dequant is
+ * NaN (per-element accumulate, both NaN byte codes), qt_matvec_rows poisons the whole
+ * dot product of any row containing one, and clean rows/lanes of the same tensor stay
+ * NaN-free and tolerance-identical to the reference. ---- */
+static void test_fmt8_nan_propagation(void){
+ enum { O=130, I=200 }; /* same tail-covering shape as the identity case above */
+ QT t; memset(&t,0,sizeof t); fill_fmt8(&t,O,I);
+ /* one NaN of EACH byte code, in different row-blocks and different column blocks;
+ * (129,199) lands in the tail row-block x tail column-block corner */
+ const int nan_row[2]={3,129}, nan_col[2]={5,199};
+ t.q8[(int64_t)nan_row[0]*I+nan_col[0]]=(int8_t)0x7F;
+ t.q8[(int64_t)nan_row[1]*I+nan_col[1]]=(int8_t)0xFF;
+ float *ref=(float*)malloc((size_t)I*sizeof(float));
+ float *acc=(float*)malloc((size_t)I*sizeof(float));
+ float *x=(float*)malloc((size_t)I*sizeof(float));
+ for(int i=0;i<I;i++) x[i]=rndsmallf();
+ float coef=1.7f; /* != 1, same reasoning as check_addrow_identity */
+ for(int r=0;r<2;r++){
+ int row=nan_row[r];
+ qt_dequant_row_ref(&t,row,ref);
+ CHECK(isnan(ref[nan_col[r]])); /* the reference itself says: this lane is NaN */
+ memset(acc,0,(size_t)I*sizeof(float));
+ qt_addrow(&t,row,coef,acc);
+ for(int i=0;i<I;i++){
+ if(isnan(ref[i])){
+ if(!isnan(acc[i])){ printf("FAIL fmt=8 NaN: qt_addrow row=%d i=%d expected NaN, got %.9g\n",row,i,(double)acc[i]); fails++; }
+ } else if(isnan(acc[i])){
+ printf("FAIL fmt=8 NaN: qt_addrow row=%d i=%d poisoned a clean lane\n",row,i); fails++;
+ } else {
+ float want=coef*ref[i];
+ float ae=fabsf(acc[i]-want);
+ float rel=fabsf(want)>1e-6f?ae/fabsf(want):ae;
+ if(rel>1e-5f){ printf("FAIL fmt=8 NaN: qt_addrow row=%d i=%d clean lane got=%.9g want=%.9g\n",row,i,(double)acc[i],(double)want); fails++; }
+ }
+ }
+ float y=0.f; qt_matvec_rows(&t,row,1,x,&y);
+ CHECK(isnan(y)); /* dot product over a NaN-bearing row is NaN, like the reference sum */
+ }
+ /* a clean row of the SAME tensor stays NaN-free through both functions */
+ { int row=4; float y=0.f;
+ qt_dequant_row_ref(&t,row,ref);
+ for(int i=0;i<I;i++) CHECK(!isnan(ref[i]));
+ memset(acc,0,(size_t)I*sizeof(float)); qt_addrow(&t,row,coef,acc);
+ for(int i=0;i<I;i++) CHECK(!isnan(acc[i]));
+ qt_matvec_rows(&t,row,1,x,&y); CHECK(!isnan(y)); }
+ free(ref); free(acc); free(x); free_qt(&t);
}
/* ---- refusal: fork+pipe+waitpid, this suite's house pattern (see test_fp8_load.c's
* expect_refuse/expect_stamp_refuse) -- must exit(1) with a "refus"-containing message,
* never reach the caller's continuation, never crash with a signal. ---- */
+/* fmt=8 no longer belongs in this refusal suite -- it has its own decoder now (see the
+ * byte-identity and matmul_fp8-parity tests above). fmt=6 (E8/IQ3) has no absorb decoder
+ * and must keep refusing loudly through both functions. */
typedef void (*absorb_fn)(void);
-static void call_addrow_fmt8(void){
- QT t; memset(&t,0,sizeof t);
- enum { O=130, I=130 }; /* the coordinator's own repro shape: nblkO=nblkI=2, nblk=4 */
- static uint8_t q8[O*I]; static float s[4];
- for(int i=0;i<O*I;i++) q8[i]=rndbyte();
- for(int i=0;i<4;i++) s[i]=0.01f;
- t.fmt=8; t.O=O; t.I=I; t.gs=0; t.q8=(int8_t*)q8; t.s=s;
- float acc[I]; memset(acc,0,sizeof acc);
- qt_addrow(&t,0,1.f,acc); /* must exit(1) inside; must NOT return */
-}
static void call_addrow_fmt6(void){
QT t; memset(&t,0,sizeof t);
enum { O=4, I=98 };
@@ -211,17 +342,6 @@ static void call_addrow_fmt6(void){
float acc[I]; memset(acc,0,sizeof acc);
qt_addrow(&t,0,1.f,acc);
}
-static void call_matvec_fmt8(void){
- QT t; memset(&t,0,sizeof t);
- enum { O=130, I=130 };
- static uint8_t q8[O*I]; static float s[4];
- for(int i=0;i<O*I;i++) q8[i]=rndbyte();
- for(int i=0;i<4;i++) s[i]=0.01f;
- t.fmt=8; t.O=O; t.I=I; t.gs=0; t.q8=(int8_t*)q8; t.s=s;
- static float x[I]; for(int i=0;i<I;i++) x[i]=rndsmallf();
- float y=0.f;
- qt_matvec_rows(&t,0,1,x,&y);
-}
static void call_matvec_fmt6(void){
QT t; memset(&t,0,sizeof t);
enum { O=4, I=98 };
@@ -270,15 +390,60 @@ static int expect_refuse_call(absorb_fn fn, const char *tag){
#endif
}
+/* fmt=8 block scales that cannot be multiplied through. A NaN scale poisons
+ * every accumulator downstream of it; a ZERO scale emits an all-zero block,
+ * which is the same output a decode against an unwritten table produces -- so
+ * both are refused by name rather than propagated. One probe per value per
+ * function; the scale is poisoned AFTER fill_fmt8 has built a valid tensor, so
+ * the only thing under test is the new guard. */
+static void poison_scale(QT *t, float v){ t->s[0]=v; }
+
+static void call_addrow_fmt8_nan(void){
+ QT t; memset(&t,0,sizeof t); fill_fmt8(&t,6,17); poison_scale(&t,(float)NAN);
+ float acc[17]; memset(acc,0,sizeof acc);
+ qt_addrow(&t,0,1.f,acc);
+}
+static void call_matvec_fmt8_nan(void){
+ QT t; memset(&t,0,sizeof t); fill_fmt8(&t,6,17); poison_scale(&t,(float)NAN);
+ static float x[17]; for(int i=0;i<17;i++) x[i]=rndsmallf();
+ float y=0.f; qt_matvec_rows(&t,0,1,x,&y);
+}
+/* A ZERO block scale is valid data, not corruption: block scales are amax/448,
+ * so an all-zero block legitimately carries one. Both arms must decode it to
+ * zeros in place -- not refuse it, and not leave the accumulator untouched.
+ * These two assertions exist to keep a zero refusal from being re-added. */
+static void test_fmt8_zero_scale_decodes_to_zeros(void){
+ enum { O=6, I=17 };
+ { QT t; memset(&t,0,sizeof t); fill_fmt8(&t,O,I); poison_scale(&t,0.f);
+ float acc[I]; for(int i=0;i<I;i++) acc[i]=1.5f; /* pre-loaded: axpy must ADD zero */
+ qt_addrow(&t,0,1.f,acc);
+ int ok=1; for(int i=0;i<I;i++) if(acc[i]!=1.5f) ok=0;
+ if(!ok) printf("FAIL qt_addrow: zero fmt=8 block scale did not add zeros\n");
+ CHECK(ok);
+ free_qt(&t);
+ }
+ { QT t; memset(&t,0,sizeof t); fill_fmt8(&t,O,I); poison_scale(&t,0.f);
+ static float x[I]; for(int i=0;i<I;i++) x[i]=rndsmallf();
+ float y=42.f; qt_matvec_rows(&t,0,1,x,&y);
+ int ok = (y==0.f);
+ if(!ok) printf("FAIL qt_matvec_rows: zero fmt=8 block scale gave %g, expected 0\n",(double)y);
+ CHECK(ok);
+ free_qt(&t);
+ }
+}
+
static void test_refusals(void){
- CHECK(expect_refuse_call(call_addrow_fmt8, "qt_addrow refuses fmt=8 (was SIGSEGV)"));
CHECK(expect_refuse_call(call_addrow_fmt6, "qt_addrow refuses fmt=6"));
- CHECK(expect_refuse_call(call_matvec_fmt8, "qt_matvec_rows refuses fmt=8"));
CHECK(expect_refuse_call(call_matvec_fmt6, "qt_matvec_rows refuses fmt=6"));
+ CHECK(expect_refuse_call(call_addrow_fmt8_nan, "qt_addrow refuses a NaN fmt=8 block scale"));
+ CHECK(expect_refuse_call(call_matvec_fmt8_nan, "qt_matvec_rows refuses a NaN fmt=8 block scale"));
}
int main(void){
test_byte_identity_all_formats();
+ test_fmt8_matmul_fp8_parity();
+ test_fmt8_nan_propagation();
+ test_fmt8_zero_scale_decodes_to_zeros();
test_refusals();
if(fails){ printf("qt_addrow/qt_matvec_rows tests: %d FAILED\n", fails); return 1; }
printf("qt_addrow/qt_matvec_rows tests: ok\n");
diff --git a/c/tools/repack_fp8_passthrough.py b/c/tools/repack_fp8_passthrough.py
index 97c06156..3f0ff07c 100644
--- a/c/tools/repack_fp8_passthrough.py
+++ b/c/tools/repack_fp8_passthrough.py
@@ -113,27 +113,22 @@ read side. `main()` also copies `config.json` (mandatory, colibri.c's
load_cfg) and `generation_config.json` (best-effort) from --indir into
--outdir. Together with kv_b_proj emission below, this makes the tool's
output a standalone-loadable model directory on its own, with one caveat: the
-engine's fmt=8 absorb support (branch `f8/absorb-fmt8`) must land before a
-kv_b_proj-carrying container is safe to decode through the batched serving
-path -- see below.
+CUDA absorb kernels do not yet decode fmt=8, so a kv_b_proj-carrying
+container decodes through the CPU absorb path only -- see below.
kv_b_proj (kind "kvb") is now emitted the same way as o_proj/attn: byte-
preserved fp8 weight + renamed `.qs` scale sidecar, stamped (it clears the
collision shape the same way o_proj does -- M1 audit, GLM-5.2's kv_b_proj is
-never at the ambiguous [O,I] this tool's `_check_geometry` guards). THIS TOOL
-CHANGE IS INDEPENDENT of, and does not wait on, the engine-side absorb work:
-colibri.c's MLA-absorption CPU path (qt_addrow/qt_matvec_rows, called only on
-l->kv_b) and the CUDA absorb kernels have no fmt==8 case as of this tool's own
-HEAD -- that support is being built in parallel on `f8/absorb-fmt8` and is
-NOT part of this diff. A container minted with this tool BEFORE that engine
-branch lands will load clean (qt_from_disk resolves the stamp exactly like
-o_proj's) but crash -- loudly, not silently -- on any decode that reaches the
-batched (`kvs`-nonNULL) serving path, because `ABSORB=0` cannot bypass absorb
-there. Sequencing that correctly (engine work before this container is used
-for decode) is the gate's job, not this tool's: repacking kv_b_proj here is
-the tool-side half of a two-sided integration, done now because engine and
-tool work can (and per the gate's assembly manifest, should) proceed in
-parallel.
+never at the ambiguous [O,I] this tool's `_check_geometry` guards). The
+engine-side half of this two-sided integration now exists on the CPU:
+colibri.c's MLA-absorption path (qt_addrow/qt_matvec_rows, called only on
+l->kv_b) carries explicit fmt==8 decode arms, so a container minted here
+loads clean (qt_from_disk resolves the stamp exactly like o_proj's) and
+decodes correctly through the batched (`kvs`-nonNULL) serving path, where
+absorb cannot be bypassed. The CUDA absorb kernels still have no fmt==8
+case (absorb_fmt_ok delegates to coli_cuda_weight_at_supported's fmt 0..4
+allowlist), so fmt=8 kv_b decode stays on the CPU absorb path until that
+lands.
METADATA STAMP (reference implementation of the FORMATS-registry FR -- see
docs/FORMATS.md): every output shard's safetensors `__metadata__` carries a