Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fp8: CPU read path + matmul_fp8, as fmt=7 (public ordinal, #524)

monotophic41fe5a7a7655
diff --git a/c/Makefile b/c/Makefile
index 10b1fa46..872b030a 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -214,7 +214,7 @@ else
PYTHON ?= python3
endif
CUDA_OBJ =
-TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_mirror$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_temp_env$(EXE) tests/test_tok_o200k$(EXE) tests/test_kv_alloc$(EXE) tests/test_int3$(EXE) tests/test_int3_load$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) tests/test_router_nan$(EXE) tests/test_pipe_block$(EXE) tests/test_e8_kernel$(EXE) tests/test_rans$(EXE) tests/test_corpus_draft$(EXE) tests/test_pilot_ring$(EXE)
+TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_mirror$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_temp_env$(EXE) tests/test_tok_o200k$(EXE) tests/test_kv_alloc$(EXE) tests/test_int3$(EXE) tests/test_int3_load$(EXE) tests/test_fp8_passthrough$(EXE) tests/test_fp8_load$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) tests/test_router_nan$(EXE) tests/test_pipe_block$(EXE) tests/test_e8_kernel$(EXE) tests/test_rans$(EXE) tests/test_corpus_draft$(EXE) tests/test_pilot_ring$(EXE)
ifneq (,$(LINUX))
TEST_BINS += tests/test_uring$(EXE)
endif
@@ -542,6 +542,12 @@ tests/test_int3$(EXE): tests/test_int3.c colibri.c st.h uring.h json.h tok.h tok
tests/test_int3_load$(EXE): tests/test_int3_load.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+tests/test_fp8_passthrough$(EXE): tests/test_fp8_passthrough.c quant.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
+tests/test_fp8_load$(EXE): tests/test_fp8_load.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
tests/test_logit_nan$(EXE): tests/test_logit_nan.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
diff --git a/c/colibri.c b/c/colibri.c
index 65acbf30..10005197 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -103,7 +103,11 @@ typedef struct {
* fmt=1 INT8 -> q8 (1 byte/param) + scala per riga
* fmt=2 INT4 -> q4 (2 valori per byte, impacchettati) + scala per riga
* INT4 e' cio' che fa stare la densa residente nei 15 GB (0.5 byte/param). */
-/* fmt: 0 F32, 1 INT8, 2 INT4 (2/byte), 3 INT2 (4/byte), 4 INT4-GROUPED, 5 INT3-G64.
+/* fmt: 0 F32, 1 INT8, 2 INT4 (2/byte), 3 INT2 (4/byte), 4 INT4-GROUPED, 5 INT3-G64,
+ * 6 E8/IQ3 lattice, 7 FP8-E4M3 (native, passthrough -- see quant.h). fmt=7 is a
+ * PUBLIC ordinal, assigned by the maintainer on #524; it developed under the
+ * PRIVATE ORDINAL BLOCK convention below as fmt=100 and graduated out of that
+ * block into this list at the same time this comment was written.
* q4 ospita int4/int2/int3 packed. fmt=4 (grouped int4, #242): per-row nibbles + one f32
* scale per group of `gs` inputs (s has O*ceil(I/gs) entries).
* fmt=6 (E8/IQ3 lattice, #452): 98B per 256 weights = 3.0625 bits/weight, grid
@@ -112,7 +116,53 @@ typedef struct {
* fmt=5 (int3, per-GROUP scales, group=64, see quant.h I3_*): values in [-4,3] stored per
* 64-input group as 24 bytes = 16B low plane (2 bits/val, int2 layout) + 8B high plane
* (1 bit/val), plus ONE f32 scale PER GROUP (s has O*ceil(I/64) entries, not O). 3.5
- * bits/weight effective β€” the quality/size sweet spot measured in the #132 ablation. */
+ * bits/weight effective β€” the quality/size sweet spot measured in the #132 ablation.
+ * fmt=7 (native FP8-e4m3 passthrough, resident/quality-core tier -- see quant.h's
+ * FP8_BLOCK/e4m3_decode/matmul_fp8): q8 holds O*I raw e4m3 bytes, ONE byte per weight,
+ * byte-identical layout to fmt=1's weight bytes (q4 is unused/NULL, same as fmt=1) --
+ * disambiguated from fmt=1 (and, at small [O,I], from fmt=6 -- see below) purely by
+ * scale geometry, see qt_resolve_fmt's "THE DESIGN LANDMINE" comment further down.
+ * s holds ONE f32 scale PER 128x128 BLOCK, ceil(O/128)*ceil(I/128) entries total
+ * (row-major: block-row-major then block-col), NOT O and NOT O*ceil(I/gs) --
+ * qt_bytes()/qt_scale_bytes() below are the authoritative byte-count formulas. The
+ * scale ENCODING is itself a declared PROPERTY of this format, not a hardcoded
+ * constant -- f32 (4 bytes/block, what's above) is the value THIS build
+ * implements; qt_resolve_fmt's "SCALE ENCODING IS A DECLARED PROPERTY" comment
+ * documents why (a DeepSeek-V4 checkpoint ships this identical weight geometry
+ * with a UE8M0 scale encoding instead) and how an unimplemented encoding is
+ * recognized and refused by name rather than silently misread.
+ * gs is unused (0) for fmt=7, same as fmt 1/2/3. */
+/* ---- PRIVATE ORDINAL BLOCK CONVENTION ------------------------------------
+ * fmt values 0-7 are upstream-assigned, public, stable ordinals -- do not
+ * reuse or renumber them (fmt=7 is the newest member: assigned by the
+ * maintainer on #524, after developing under this block as fmt=100 -- see
+ * "graduated" above). fmt values 100+ remain this repo's PRIVATE/EXPERIMENTAL
+ * block for any OTHER in-flight format proposal: ordinals a branch mints for
+ * itself during development so it can't collide with a number upstream claims
+ * out from under it -- exactly what happened to fmt=7's OWN earlier private
+ * number, fmt=6, when #465's E8/IQ3 proposal claimed that ordinal upstream
+ * while this branch was still developing against it, forcing a re-mint to
+ * fmt=100 (see upstream_contribution/FORMATS_registry_draft.md for the
+ * incident that prompted the rule).
+ *
+ * A PRIVATE-BLOCK ordinal is an internal enum value only -- qt_resolve_fmt
+ * (below) infers format purely from byte arithmetic; the container on disk
+ * carries no format ordinal at all. (A self-describing container stamp that
+ * would persist a format's NAME, not its ordinal, is a follow-up proposal --
+ * see qt_resolve_fmt's own note on where that plumbing would attach -- not
+ * present in this build.) Nothing outside this binary's own compiled code
+ * ever observes a 100+ number, so renumbering one later (e.g. when a format
+ * is upstreamed and assigned a real public ordinal at merge, as just happened
+ * for fmt=7) is a pure find-and-replace with zero on-disk or cross-version
+ * compatibility impact.
+ *
+ * Rule for adding a new format to this branch or a future one: claim the next
+ * unused 100+ integer, never a number already claimed upstream (check dev
+ * before picking one -- this is exactly the mistake that forced fmt=7's own
+ * proposal to renumber away from fmt=6) or by another in-flight private
+ * format. Never ship a 100+ ordinal as a public default/committed-upstream
+ * value -- the real ordinal is assigned by the maintainer at merge time,
+ * exactly as fmt=7's was. */
typedef struct {
int fmt; float *qf; int8_t *q8; uint8_t *q4; float *s; int O, I, gs; /* gs=group size (0=per-row, 128=grouped) */
#ifdef COLI_CUDA
@@ -137,8 +187,52 @@ static int64_t qt_bytes(const QT *t){ /* byte residenti del tensore */
return (int64_t)t->O*ng*24 + (int64_t)t->O*ng*4; }
if(t->fmt==6) /* E8/IQ3: 98B per 256 weights, scales in-block, .qs is a 4-byte tag */
return (int64_t)t->O*(((int64_t)t->I+255)/256)*98 + 4;
+ if(t->fmt==7){ /* fp8-e4m3 passthrough: O*I raw e4m3 bytes (n, byte-identical layout
+ * to fmt=1's weight bytes) + one f32 scale per 128x128 block
+ * (FP8_BLOCK in quant.h, included below qt_bytes -- keep the
+ * arithmetic literal here, same discipline as fmt=5's comment
+ * above). Missing this branch would fall through to the fmt=2
+ * default below (packed-nibble formula, ~half the real weight
+ * bytes) and undercount a resident fp8 tensor's byte footprint --
+ * feeds AUTOPIN/RAM-budget math, so this branch is load-bearing
+ * from day one, not a later fix (contrast fmt=6 above, upstream's
+ * own #452 review round 1 caught this exact bug shape for E8). */
+ int64_t nblkO=((int64_t)t->O+127)/128, nblkI=((int64_t)t->I+127)/128;
+ return n + nblkO*nblkI*4; }
return (int64_t)t->O*((t->I+1)/2) + (int64_t)t->O*4; /* fmt=2 int4 per-row */
}
+/* scale-array byte count only, format-aware -- split out of qt_bytes() because
+ * qt_wire_mmap/qt_unwire_mmap mlock the weight and scale ranges as TWO SEPARATE
+ * regions (separate allocations, not one contiguous buffer: qt_from_disk always
+ * qalloc's t->q8/t->q4 and t->s independently) and need the scale byte count on
+ * its own to split qt_bytes(t) into a weight half and a scale half. The two call
+ * sites used to hardcode scale_b=O*4 -- i.e. assume PER-ROW scale for every
+ * format -- which is right for fmt 0/1/2/3 but wrong for fmt=4 (grouped,
+ * O*ceil(I/gs) scales), fmt=5 (int3-g64, O*ceil(I/64) scales), and fmt=7
+ * (per-128x128-block, ceil(O/128)*ceil(I/128) scales): any tensor in one of
+ * those three formats reaching qt_wire_mmap/qt_unwire_mmap would mlock/munlock
+ * the wrong byte ranges on BOTH halves (weight_b = qt_bytes(t)-scale_b shifts
+ * too). fmt=6 (E8) is deliberately NOT covered here: its .qs is a fixed 4-byte
+ * tag with no wire-mmap-relevant weight/scale split of its own (matches
+ * qt_bytes' `+4` literal above), and t->fmt==6 tensors never reach
+ * qt_wire_mmap/qt_unwire_mmap's mlock path in this build (int3-g64/E8 stay
+ * CPU-side, see qt_cuda_upload -- MMAP wiring is a CUDA/Metal-adjacent memory
+ * concern this build doesn't extend to them). fixed generically since the same
+ * bug shape applies to fmt=4/5, not just the format that surfaced it -- fmt=4
+ * is the routed-expert "int4-g64" format (qt_resolve_fmt assigns it to ESlot
+ * g/u/d the same as any other tensor, see expert_load_impl), so this is not
+ * purely a dormant fmt=7-only fix: any COLI_MMAP + mem_should_wire() session
+ * pinning fmt=4 experts was already mlocking the wrong ranges before this
+ * branch existed. UNVERIFIED at runtime (no model run performed -- static/
+ * arithmetic fix only, see the report). */
+static int64_t qt_scale_bytes(const QT *t){
+ if(t->fmt==4){ int ng=(t->I+t->gs-1)/t->gs; return (int64_t)t->O*ng*4; }
+ if(t->fmt==5){ int64_t ng=((int64_t)t->I+63)/64; return (int64_t)t->O*ng*4; }
+ if(t->fmt==7){ int64_t nblkO=((int64_t)t->O+127)/128, nblkI=((int64_t)t->I+127)/128; return nblkO*nblkI*4; }
+ return (int64_t)t->O*4; /* fmt 0 (t->s NULL, guarded by callers)/1/2/3/6: per-row (or, for
+ * fmt=6, the 4-byte tag -- callers of this function never see fmt=6,
+ * see the comment above; the fallback is harmless dead code for it). */
+}
typedef struct {
float *in_ln, *post_ln;
@@ -407,6 +501,14 @@ static int vk_matmul_pair_qt(QT *a, float *ya, QT *b, float *yb, const float *x,
static int qt_cuda_upload(QT *t){
if(t->fmt==5) return 0; /* int3-g64: no CUDA kernel yet β€” tensor stays CPU-side */
if(t->fmt==6 && !g_cuda_e8_ready) return 0; /* E8 without its codebook would decode garbage */
+ if(t->fmt==7) return 0; /* fp8-e4m3 passthrough: no CUDA kernel yet either (matches the
+ * fmt=5/6 idiom above; matmul_qt_ex's own caller-side fmt exclusion
+ * already keeps this from being reached with cuda_eligible tensors
+ * in the exact-kernel path, but row_bytes()/weight_at() in
+ * backend_cuda.cu have no fmt=7 case either -- explicit early-
+ * return here so a future caller doesn't have to rediscover that by
+ * tracing through to the CUDA source. UNVERIFIED at runtime: no CUDA
+ * toolchain on this macOS build host to compile-check. */
const void *weights = t->fmt==0 ? (const void*)t->qf
: t->fmt==1 ? (const void*)t->q8 : (const void*)t->q4;
if(t->fmt==4) /* grouped int4 (#334): scales are [O, ceil(I/gs)] β€” the plain
@@ -684,13 +786,26 @@ static void matmul_i4_grouped_pair(float *yg, float *yu, const float *x,
* (~+12% perplexity), measured. Every other prefill matmul keeps IDOT as before. */
static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot){
#ifdef COLI_METAL
+ /* fmt=7 (fp8 passthrough) deliberately absent from this allowlist, same as fmt=5/6:
+ * the S>=g_metal_gemm_min batched prefill GEMM path (coli_metal_gemm) only knows
+ * fmt 1/2/4 in this build, so it fails CLOSED to the CPU branch below (matmul_fp8)
+ * rather than being silently misread. coli_metal_gemm() also carries its own
+ * internal fmt!=1&&fmt!=2&&fmt!=4 guard, so this is belt-and-braces. */
if(g_metal_enabled && S>=g_metal_gemm_min && !spec_pinned() && (w->fmt==1||w->fmt==2||w->fmt==4) && !omp_in_parallel()){
const void *wp = w->fmt==1 ? (const void*)w->q8 : (const void*)w->q4;
if(coli_metal_gemm(y,x,wp,w->s,w->fmt,S,w->I,w->O,w->gs)) return;
}
#endif
#ifdef COLI_CUDA
- if(g_cuda_enabled && w->cuda_eligible && !w->cuda_failed && w->fmt!=5 && !omp_in_parallel()){
+ /* fmt=7 excluded exactly like fmt=5/6: the CUDA backend's row_bytes()/weight_at()
+ * (backend_cuda.cu) have no case for it and fall through to `return 0` / undefined
+ * weight_at() behavior. row_bytes()==0 already makes coli_cuda_tensor_upload_g
+ * refuse (defensive fail-closed), but that path is reached only after paying an
+ * upload attempt and printing a "disabled after an error" message every load;
+ * excluding it here up front is the same fmt=5/6 idiom, silent and immediate.
+ * UNVERIFIED on this build (no CUDA toolchain on macOS to compile-check; traced
+ * from backend_cuda.cu source only). */
+ if(g_cuda_enabled && w->cuda_eligible && !w->cuda_failed && w->fmt!=5 && w->fmt!=7 && !omp_in_parallel()){
const void *weights = w->fmt==0 ? (const void*)w->qf
: w->fmt==1 ? (const void*)w->q8 : (const void*)w->q4;
if(coli_cuda_matmul(&w->cuda,y,x,weights,w->s,w->fmt,S,w->I,w->O,w->cuda_device,w->gs)) return;
@@ -716,6 +831,7 @@ static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot)
if(w->fmt==1) matmul_q(y,x,w->q8,w->s,S,w->I,w->O);
else if(w->fmt==3) matmul_i2(y,x,w->q4,w->s,S,w->I,w->O);
else if(w->fmt==5) matmul_i3(y,x,w->q4,w->s,S,w->I,w->O);
+ else if(w->fmt==7) matmul_fp8(y,x,(const uint8_t*)w->q8,w->s,S,w->I,w->O);
else matmul_i4(y,x,w->q4,w->s,S,w->I,w->O);
}
@@ -1138,26 +1254,135 @@ static int detect_group_size(int O, int I, int64_t ns){
* diventava un int2 valido e il matmul leggeva oltre il buffer (O*I nibble a
* 4/byte). Qui i byte del peso devono corrispondere a un layout noto e i byte
* della scala alla cardinalita' attesa (O per-row, O*ng per-gruppo) β€” altrimenti
- * si termina invece di sforare. Ritorna fmt (1/2/3/4/5) e scrive *gs. */
+ * si termina invece di sforare. Ritorna fmt (1/2/3/4/5/6/7) e scrive *gs.
+ * No container-level format stamp is consulted here: a self-describing
+ * __metadata__ stamp that could resolve a genuine byte-collision below
+ * (instead of refusing it unconditionally) is a follow-up proposal, not
+ * present in this build -- every ambiguous shape this function can see
+ * refuses, full stop. */
static int qt_resolve_fmt(const char *name, int O, int I, int64_t nb, int64_t ns, int *gs){
int64_t exp_i8=(int64_t)O*I, exp_i4=(int64_t)O*((I+1)/2), exp_i2=(int64_t)O*((I+3)/4);
int64_t exp_i3=(int64_t)O*i3_rowbytes(I); /* int3-g64 (fmt=5): 24B per 64-input group */
/* fmt=6 (E8/IQ3, #452): scales live inside the 98B super-blocks, so the .qs
* convention is kept with a single-float tag β€” ns==4 is the discriminator
- * (every other format carries at least O floats of real scales). */
- if(ns==4 && nb==(int64_t)O*e8_rowbytes(I)){ *gs=0; return 6; }
+ * (every other format carries at least O floats of real scales).
+ *
+ * SECOND DESIGN LANDMINE: e8_rowbytes(I) = ceil(I/256)*98 is the constant 98
+ * for every I in (0,256], so this check's nb==O*e8_rowbytes(I) collapses to
+ * nb==O*98 -- which is ALSO fp8-e4m3-b128's raw-byte weight count (O*I) at
+ * the ONE value of I where I itself equals 98 (solving 98k==I for I in
+ * ((k-1)*256,k*256] only admits k=1, I=98). At that same I=98 a handful of
+ * fp8 scale-array shapes ALSO carry exactly ns==4 bytes, same as this
+ * check's own tag, and each is a genuine collision candidate this block
+ * must catch before the unconditional `return 6` below:
+ * - a SINGLE-BLOCK fp8 tensor (O<=128, fp8_nblk(O)*fp8_nblk(I)==1) with
+ * f32 block scales: ns==1*4==4.
+ * - a FOUR-BLOCK fp8 tensor (fp8_nblk(O)*fp8_nblk(I)==4, e.g. O in
+ * (384,512] at this I) with UE8M0 (1 byte/block) scales: ns==4*1==4 --
+ * the SAME arithmetic collision the fmt=1-vs-fmt=7 landmine below has
+ * with UE8M0, just against fmt=6's tag instead of fmt=1's per-row
+ * count; see that landmine's own comment for the general shape.
+ * - at O==1 specifically, fmt=1's own per-row tag (O*4) is also 4.
+ * Refuse unconditionally when any of these coincide with the E8/IQ3 tag --
+ * this build has no stamp to break the tie (see the function's own header
+ * comment) -- rather than let dev's own unconditional `return 6` silently
+ * misread an fp8-e4m3-b128 (or, at O==1, plain int8) tensor as E8/IQ3-
+ * lattice-decoded garbage with no error. No real GLM tensor has these
+ * shapes; the discipline exists for untrusted containers. */
+ if(ns==4 && nb==(int64_t)O*e8_rowbytes(I)){
+ int fp8_blk_f32_also = (nb==(int64_t)O*I) && (fp8_nblk(O)*fp8_nblk(I)==1);
+ int fp8_blk_ue8m0_also = (nb==(int64_t)O*I) && (fp8_nblk(O)*fp8_nblk(I)==4);
+ int i8_row_also = (nb==(int64_t)O*I) && (O==1); /* ns==O*4==4 iff O==1 */
+ if(fp8_blk_f32_also || fp8_blk_ue8m0_also || i8_row_also){
+ fprintf(stderr,"%s: [%d,%d] byte layout (nb=%lld ns=%lld) matches E8/IQ3 "
+ "(fmt=6, 4-byte tag)%s%s%s; refusing rather than guessing (untrusted "
+ "container, fmt=6 collision at I=98)\n",
+ name,O,I,(long long)nb,(long long)ns,
+ fp8_blk_f32_also ? " AND per-128x128-block FP8 f32 scales (fmt=7, single block)" : "",
+ fp8_blk_ue8m0_also ? " AND per-128x128-block FP8 ue8m0 scales (fmt=7, 4 blocks, recognized-not-implemented)" : "",
+ i8_row_also ? " AND plain int8 per-row (fmt=1, O=1)" : "");
+ exit(1);
+ }
+ *gs=0; return 6;
+ }
/* Row formats take precedence: for tiny I the int3-g64 byte count can coincide with
* a row layout (e.g. [O,48]: ceil(48/2)=24=1*24). For real tensor shapes the counts
* are distinct, and the weight bytes β€” not the scale size β€” are the int3 tag, because
* int3-g64 and grouped-int4-at-gs=64 carry the SAME scale cardinality O*ceil(I/64). */
int fmt = (nb==exp_i8)?1 : (nb==exp_i4)?2 : (nb==exp_i2)?3 : (nb==exp_i3)?5 : 0;
if(!fmt){
- fprintf(stderr,"%s: quantized weight is %lld bytes β€” no int8/int4/int2/int3-g64 layout for [%d,%d], refusing (untrusted container)\n",
+ fprintf(stderr,"%s: quantized weight is %lld bytes β€” no int8/int4/int2/int3-g64/fp8 layout for [%d,%d], refusing (untrusted container)\n",
name,(long long)nb,O,I); exit(1); }
*gs=0;
if(fmt==2){ int g=detect_group_size(O,I,ns); if(g>0){ fmt=4; *gs=g; } }
+ /* fmt=1 vs fmt=7 (native FP8-e4m3 passthrough): THE DESIGN LANDMINE. Weight
+ * bytes are IDENTICAL (O*I raw bytes, both matched exp_i8 above) -- fmt=1
+ * and fmt=7 can only be told apart by the SCALE array's geometry: fmt=1 is
+ * per-row (ns==O*4 bytes); fmt=7 is per-128x128-BLOCK, ns==ceil(O/128)*
+ * ceil(I/128)*4 bytes for THIS build's implemented f32 scale encoding. For
+ * most real shapes these two byte counts are distinct and the match is
+ * unambiguous. But for small O (<=128) and/or small I, ceil(O/128)*
+ * ceil(I/128) can equal O exactly (e.g. O=1,I<=128 -> 1*1=1=O; O=2,I in
+ * [129,256] -> 1*2=2=O; O=256,I in (16256,16384] -> 2*128=256=O) -- a
+ * tensor whose scale array satisfies BOTH conventions at once. Guessing
+ * either way risks silently reading a genuine per-row-int8 tensor as
+ * block-scaled FP8 (or vice versa), which would corrupt every weight
+ * without any error. Refuse instead (same "untrusted container"
+ * discipline as the rest of this function).
+ *
+ * SCALE ENCODING IS A DECLARED PROPERTY, not a hardcoded constant: f32 (4
+ * bytes/block, above) is what THIS build implements, but it is not the
+ * only encoding real fp8-e4m3-b128 weight geometry ships with. DeepSeek-V4
+ * ships the SAME weight layout (FP8 E4M3, 128x128 blocks) with UE8M0
+ * scales instead -- one byte per block, a power-of-two exponent, dtype
+ * F8_E8M0 (maintainer finding, colibri #524) -- so a fp8-e4m3-b128
+ * container's scale sidecar can legitimately be ceil(O/128)*ceil(I/128)*1
+ * bytes, not *4. That byte count is a REAL, distinct signature (never
+ * equal to the f32 block-scale count above for any O,I>=1, since one is
+ * exactly 4x the other and 4n==n only at n==0) -- recognized here rather
+ * than silently misread as a truncated or corrupt f32 scale array. This
+ * build does not implement UE8M0 decode: recognizing the signature and
+ * refusing BY NAME (rather than falling through to the generic
+ * "wrong byte count" refusal below, or worse, matching it against the
+ * wrong candidate) is the whole point -- a future decoder lands into this
+ * seam rather than a near-duplicate format. Checked for collision against
+ * every OTHER format's ns arithmetic reachable from this nb==O*I branch
+ * (fmt=1 and fmt=7-f32, both above): the byte counts are realistically
+ * distinct (nblkO*nblkI is orders of magnitude smaller than O*4 for any
+ * real GLM-sized matrix), but NOT categorically distinct -- the same
+ * small-O regime that makes the fmt=1-vs-fmt=7-f32 collision above
+ * possible also makes a fmt=1-vs-fmt=7-ue8m0 collision possible (e.g.
+ * O=1, I in (384,512]: nblkO=1, nblkI=4, ue8m0 ns=4=O*4=fmt=1's own
+ * per-row count) -- handled below by refusing whenever the ue8m0
+ * signature matches, noting the row collision too when it also applies.
+ * No real GLM tensor has these shapes; see also the SECOND DESIGN
+ * LANDMINE above for the analogous ue8m0-vs-fmt=6 corner this same
+ * signature can hit. */
+ if(fmt==1){
+ int64_t nblkO=fp8_nblk(O), nblkI=fp8_nblk(I);
+ int64_t ns_row=(int64_t)O*4, ns_blk=nblkO*nblkI*4, ns_blk_ue8m0=nblkO*nblkI;
+ int is_row=(ns==ns_row), is_blk=(ns==ns_blk), is_blk_ue8m0=(ns==ns_blk_ue8m0);
+ if(is_row && is_blk){
+ fprintf(stderr,"%s: [%d,%d] scale array is %lld bytes β€” matches BOTH per-row "
+ "int8 (fmt=1) and per-128x128-block FP8 (fmt=7) scale geometry; refusing "
+ "rather than guessing (untrusted container, THE DESIGN LANDMINE)\n",
+ name,O,I,(long long)ns); exit(1);
+ }
+ if(is_blk_ue8m0){
+ fprintf(stderr,"%s: [%d,%d] fp8-e4m3-b128 with ue8m0 scales recognized but not "
+ "implemented; only f32 block scales are supported in this build (nb=%lld "
+ "bytes matches raw e4m3 weight bytes, ns=%lld bytes matches %lld blocks x "
+ "1 byte/block)%s\n",
+ name,O,I,(long long)nb,(long long)ns,(long long)(nblkO*nblkI),
+ is_row ? " -- scale array ALSO matches per-row int8 (fmt=1); refusing either way (untrusted container)" : "");
+ exit(1);
+ }
+ if(is_blk && !is_row) fmt=7;
+ }
int64_t exp_scale = (fmt==4)? (int64_t)O*((I+*gs-1)/(*gs))
- : (fmt==5)? (int64_t)O*i3_groups(I) : (int64_t)O; /* in FLOAT */
+ : (fmt==5)? (int64_t)O*i3_groups(I)
+ : (fmt==7)? fp8_nblk(O)*fp8_nblk(I)
+ : (int64_t)O; /* in FLOAT */
if(ns != exp_scale*4){
fprintf(stderr,"%s: scale array is %lld bytes β€” expected %lld for [%d,%d] fmt=%d, refusing (untrusted container)\n",
name,(long long)ns,(long long)(exp_scale*4),O,I,fmt); exit(1); }
@@ -1200,16 +1425,25 @@ static void qt_from_disk(Model *m, const char *name, int O, int I, int bits, int
else if(fmt==6){ /* E8/IQ3: everything in-block, .qs is the 4-byte tag */
if(t->fmt!=6||!t->q4){ t->fmt=6; t->O=O; t->I=I; t->gs=0; t->q4=qalloc(nb); t->s=qsalloc(1); }
st_read_raw(&m->S,name,t->q4,drop); }
+ else if(fmt==7){ int64_t nblk=fp8_nblk(O)*fp8_nblk(I); /* fp8-e4m3: raw bytes (q8, like fmt=1)
+ * + one f32 scale per 128x128 block. BOTH weights and scale via qalloc, not falloc,
+ * from day one -- the GPU-visibility lesson fmt=4 and fmt=5 each had to learn
+ * separately (see review round 1 audit history in the report). */
+ if(t->fmt!=7||!t->q8){ t->fmt=7; t->O=O; t->I=I; t->gs=0; t->q8=qalloc(nb); t->s=(float*)qalloc((size_t)nblk*sizeof(float)); }
+ st_read_raw(&m->S,name,t->q8,drop); }
else { if(t->fmt!=fmt||!t->q4){ t->fmt=fmt; t->O=O; t->I=I; t->gs=0; t->q4=qalloc(nb); t->s=qsalloc(O); } st_read_raw(&m->S,name,t->q4,drop); }
/* cap MUST match the scale cardinality qt_resolve_fmt already validated and
* the falloc above actually reserved, per format: grouped-int4 (fmt=4) keeps
- * O*ceil(I/gs) scales and int3-g64 (fmt=5) keeps O*i3_groups(I); everything
- * else is per-row O. Using the per-row bound for a grouped format would
- * reject a legitimate container (fmt=5 regressed exactly that way). */
+ * O*ceil(I/gs) scales, int3-g64 (fmt=5) keeps O*i3_groups(I), E8/IQ3 (fmt=6)
+ * keeps the 1-float tag, and fp8-e4m3 (fmt=7) keeps ceil(O/128)*ceil(I/128)
+ * block scales; everything else is per-row O. Using the per-row bound for a
+ * grouped/blocked format would reject a legitimate container (fmt=5 regressed
+ * exactly that way). */
st_read_f32_cap(&m->S,sn,t->s,
fmt==4 ? (int64_t)O*((I+gs-1)/gs) :
fmt==5 ? (int64_t)O*i3_groups(I) :
- fmt==6 ? (int64_t)1 : (int64_t)O, drop);
+ fmt==6 ? (int64_t)1 :
+ fmt==7 ? fp8_nblk(O)*fp8_nblk(I) : (int64_t)O, drop);
} else {
if(!t->qf && !t->q8 && !t->q4) qt_alloc(t,O,I,bits);
if(t->fmt==0) st_read_f32_cap(&m->S,name,t->qf,(int64_t)O*I,drop);
@@ -2598,10 +2832,19 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p
* (step_decode_batch) passes per-row kvs[]/positions[] with pos_base=0, so the kernel
* would rope every row at position 0 and attend over a 1-token window of the wrong
* cache -> greedy decode hits EOS at token 2 (mux answers truncated to 1 token).
- * Ragged rows take the CPU absorb path below, which reads kvs[s]/positions[s]. */
+ * Ragged rows take the CPU absorb path below, which reads kvs[s]/positions[s].
+ * fmt!=7 guards (q_a/q_b/kv_a/o): coli_metal_attn_decode's WP_() macro and the
+ * mm_gemv shader it dispatches through (bind_gemv) have no fmt=7 case in this
+ * build -- an fp8-passthrough tensor reaching here would be silently misread as
+ * f32 by the shader's default branch. Fail closed to the CPU path below instead
+ * (matmul_qt_ex there dispatches fmt=7 correctly via matmul_fp8). kv_b is already
+ * pinned to fmt==2 above for an unrelated reason (its absorb kernel is int4-only);
+ * these four checks are the same discipline extended to the tensors that flow
+ * through the generic per-fmt shader. */
if(g_metal_enabled && !kvs && S<=4 && (g_absorb==1||(g_absorb<0&&S<=4)) && m->kv_start[layer]==0
&& D==6144 && H==64 && c->q_lora==2048 && c->kv_lora==512 && c->qk_nope==192
- && c->qk_rope==64 && vh==256 && l->kv_b.fmt==2){
+ && c->qk_rope==64 && vh==256 && l->kv_b.fmt==2
+ && l->q_a.fmt!=7 && l->q_b.fmt!=7 && l->kv_a.fmt!=7 && l->o.fmt!=7){
int sel_active = m->has_dsa && layer<c->n_layers && c->idx_type[layer] && (pos_base+S) > c->index_topk;
if(!sel_active){
if(m->has_dsa && layer<c->n_layers && c->idx_type[layer]){ /* index keys for future selection */
@@ -4806,12 +5049,19 @@ static void layer_forward_rows(Model *m, Layer *l, int li, float *x, int S, int
* in un solo submit GPU; la CPU legge il routing e fa solo resolve/disk/expert-CB.
* Fallback: qualsiasi condizione mancante -> percorso CPU intero qui sotto.
* !kvs: ragged mux rows (per-row KV/position) are not expressible in this kernel's
- * single Lc/Rc + pos_base contract β€” see the matching guard in attention_rows. */
+ * single Lc/Rc + pos_base contract β€” see the matching guard in attention_rows.
+ * fmt!=7 guards (q_a/q_b/kv_a/o/sh_gate/sh_up/sh_down): same fail-closed discipline
+ * as attention_rows, extended to the shared-expert MLP this fused kernel also
+ * covers -- none of these seven bind_gemv-routed tensors has fmt=7 support in this
+ * build's mm_gemv shader, so any of them carrying fmt=7 must skip the whole fused
+ * path (CPU below dispatches fmt=7 correctly via matmul_qt_ex/matmul_fp8). */
if(g_metal_enabled && !kvs && S<=4 && li<c->n_layers && l->sparse
&& (g_absorb==1||(g_absorb<0&&S<=4)) && m->kv_start[li]==0
&& D==6144 && c->n_heads==64 && c->q_lora==2048 && c->kv_lora==512
&& c->qk_nope==192 && c->qk_rope==64 && c->v_head==256 && l->kv_b.fmt==2
- && c->n_experts==256 && c->topk==8 && c->n_shared==1 && c->moe_inter==2048){
+ && c->n_experts==256 && c->topk==8 && c->n_shared==1 && c->moe_inter==2048
+ && l->q_a.fmt!=7 && l->q_b.fmt!=7 && l->kv_a.fmt!=7 && l->o.fmt!=7
+ && l->sh_gate.fmt!=7 && l->sh_up.fmt!=7 && l->sh_down.fmt!=7){
int sel_active = m->has_dsa && c->idx_type[li] && (pos_base+S) > c->index_topk;
if(!sel_active){
static float *linrm,*lnrm,*lsh,*lw; static int *lidx,*lkeff;
diff --git a/c/quant.h b/c/quant.h
index d244d1b1..7dbf3936 100644
--- a/c/quant.h
+++ b/c/quant.h
@@ -402,6 +402,111 @@ static void matmul_i3(float *y, const float *x, const uint8_t *q3, const float *
}
}
+/* ---- fmt=7: native FP8-e4m3 passthrough (Z.ai GLM-5.2-FP8 read path) ------
+ * PUBLIC ordinal, assigned by the maintainer on #524. This format was minted
+ * fmt=6 during original development, then re-tagged fmt=100 (PRIVATE ORDINAL
+ * BLOCK, see colibri.c's QT struct comment) after #465 (E8/IQ3, above) claimed
+ * ordinal 6 upstream out from under it; fmt=7 is the number it graduated to at
+ * merge, per the private-block convention's own "find-and-replace, zero
+ * on-disk impact" promise. See qt_resolve_fmt in colibri.c ("THE DESIGN
+ * LANDMINE") for the disambiguation this needs against BOTH fmt=1 (int8) and
+ * fmt=6 (E8/IQ3).
+ *
+ * fmt=7 weight bytes are O*I raw e4m3 bytes -- byte-identical to int8 (fmt=1).
+ * What makes this a DIFFERENT format is the scale: one f32 per 128x128 BLOCK of
+ * the [O,I] weight matrix (shape [ceil(O/128),ceil(I/128)]), not one f32 per
+ * output row. Dequant is w[o,i] = e4m3_decode(byte) * scale[o/128, i/128]
+ * (MULTIPLY, not divide) -- mirrors tools/convert_fp8_to_int4.py's dequant()
+ * exactly, which is the authoritative reference for how Z.ai's checkpoints read
+ * on the source side. This f32 encoding is a declared PROPERTY of the format,
+ * not the only one it can carry -- DeepSeek-V4 ships this identical weight
+ * geometry with UE8M0 (1-byte, power-of-two) block scales instead; qt_resolve_fmt
+ * recognizes that byte signature and refuses it BY NAME rather than
+ * misreading it as f32 (UE8M0 decode itself is not implemented in this build).
+ *
+ * 256-entry decode LUT, cross-checked byte-for-byte against PyTorch's
+ * torch.float8_e4m3fn (the exact dtype safetensors reports for these shards):
+ * sign(1) exp(4,bias=7) mant(3), subnormal at exp==0, and the OCP E4M3-FN
+ * convention that exp==0xF is NOT reserved for infinity -- only mant==0x7 at
+ * exp==0xF is NaN (max finite is exp=0xF,mant=0x6 -> 448). A static compile-time
+ * table (not a lazily-initialized one) is used deliberately: matmul_fp8 below
+ * runs inside #pragma omp parallel for, and a lazy-init global would race under
+ * concurrent first use.
+ *
+ * NaN POLICY (decided, documented per the build spec): a NaN byte (0x7F/0xFF)
+ * decodes to a real IEEE NaN and is left to propagate through the dot product,
+ * exactly like any other source of a NaN weight (fmt=0/f32 tensors are never
+ * scrubbed either). The engine already has a dedicated, TESTED safety net for
+ * NaN reaching the sampler (argmax_v/dist_build, see tests/test_logit_nan.c:
+ * "degrade + diagnose, never silently corrupt") -- fmt=7 relies on that existing
+ * downstream net rather than adding a second, redundant weight-level scrub. */
+static const float E4M3_LUT[256] = {
+ 0x0.0p+0f,0x1.0000000000000p-9f,0x1.0000000000000p-8f,0x1.8000000000000p-8f,0x1.0000000000000p-7f,0x1.4000000000000p-7f,0x1.8000000000000p-7f,0x1.c000000000000p-7f,
+ 0x1.0000000000000p-6f,0x1.2000000000000p-6f,0x1.4000000000000p-6f,0x1.6000000000000p-6f,0x1.8000000000000p-6f,0x1.a000000000000p-6f,0x1.c000000000000p-6f,0x1.e000000000000p-6f,
+ 0x1.0000000000000p-5f,0x1.2000000000000p-5f,0x1.4000000000000p-5f,0x1.6000000000000p-5f,0x1.8000000000000p-5f,0x1.a000000000000p-5f,0x1.c000000000000p-5f,0x1.e000000000000p-5f,
+ 0x1.0000000000000p-4f,0x1.2000000000000p-4f,0x1.4000000000000p-4f,0x1.6000000000000p-4f,0x1.8000000000000p-4f,0x1.a000000000000p-4f,0x1.c000000000000p-4f,0x1.e000000000000p-4f,
+ 0x1.0000000000000p-3f,0x1.2000000000000p-3f,0x1.4000000000000p-3f,0x1.6000000000000p-3f,0x1.8000000000000p-3f,0x1.a000000000000p-3f,0x1.c000000000000p-3f,0x1.e000000000000p-3f,
+ 0x1.0000000000000p-2f,0x1.2000000000000p-2f,0x1.4000000000000p-2f,0x1.6000000000000p-2f,0x1.8000000000000p-2f,0x1.a000000000000p-2f,0x1.c000000000000p-2f,0x1.e000000000000p-2f,
+ 0x1.0000000000000p-1f,0x1.2000000000000p-1f,0x1.4000000000000p-1f,0x1.6000000000000p-1f,0x1.8000000000000p-1f,0x1.a000000000000p-1f,0x1.c000000000000p-1f,0x1.e000000000000p-1f,
+ 0x1.0000000000000p+0f,0x1.2000000000000p+0f,0x1.4000000000000p+0f,0x1.6000000000000p+0f,0x1.8000000000000p+0f,0x1.a000000000000p+0f,0x1.c000000000000p+0f,0x1.e000000000000p+0f,
+ 0x1.0000000000000p+1f,0x1.2000000000000p+1f,0x1.4000000000000p+1f,0x1.6000000000000p+1f,0x1.8000000000000p+1f,0x1.a000000000000p+1f,0x1.c000000000000p+1f,0x1.e000000000000p+1f,
+ 0x1.0000000000000p+2f,0x1.2000000000000p+2f,0x1.4000000000000p+2f,0x1.6000000000000p+2f,0x1.8000000000000p+2f,0x1.a000000000000p+2f,0x1.c000000000000p+2f,0x1.e000000000000p+2f,
+ 0x1.0000000000000p+3f,0x1.2000000000000p+3f,0x1.4000000000000p+3f,0x1.6000000000000p+3f,0x1.8000000000000p+3f,0x1.a000000000000p+3f,0x1.c000000000000p+3f,0x1.e000000000000p+3f,
+ 0x1.0000000000000p+4f,0x1.2000000000000p+4f,0x1.4000000000000p+4f,0x1.6000000000000p+4f,0x1.8000000000000p+4f,0x1.a000000000000p+4f,0x1.c000000000000p+4f,0x1.e000000000000p+4f,
+ 0x1.0000000000000p+5f,0x1.2000000000000p+5f,0x1.4000000000000p+5f,0x1.6000000000000p+5f,0x1.8000000000000p+5f,0x1.a000000000000p+5f,0x1.c000000000000p+5f,0x1.e000000000000p+5f,
+ 0x1.0000000000000p+6f,0x1.2000000000000p+6f,0x1.4000000000000p+6f,0x1.6000000000000p+6f,0x1.8000000000000p+6f,0x1.a000000000000p+6f,0x1.c000000000000p+6f,0x1.e000000000000p+6f,
+ 0x1.0000000000000p+7f,0x1.2000000000000p+7f,0x1.4000000000000p+7f,0x1.6000000000000p+7f,0x1.8000000000000p+7f,0x1.a000000000000p+7f,0x1.c000000000000p+7f,0x1.e000000000000p+7f,
+ 0x1.0000000000000p+8f,0x1.2000000000000p+8f,0x1.4000000000000p+8f,0x1.6000000000000p+8f,0x1.8000000000000p+8f,0x1.a000000000000p+8f,0x1.c000000000000p+8f, NAN,
+ -0x0.0p+0f,-0x1.0000000000000p-9f,-0x1.0000000000000p-8f,-0x1.8000000000000p-8f,-0x1.0000000000000p-7f,-0x1.4000000000000p-7f,-0x1.8000000000000p-7f,-0x1.c000000000000p-7f,
+ -0x1.0000000000000p-6f,-0x1.2000000000000p-6f,-0x1.4000000000000p-6f,-0x1.6000000000000p-6f,-0x1.8000000000000p-6f,-0x1.a000000000000p-6f,-0x1.c000000000000p-6f,-0x1.e000000000000p-6f,
+ -0x1.0000000000000p-5f,-0x1.2000000000000p-5f,-0x1.4000000000000p-5f,-0x1.6000000000000p-5f,-0x1.8000000000000p-5f,-0x1.a000000000000p-5f,-0x1.c000000000000p-5f,-0x1.e000000000000p-5f,
+ -0x1.0000000000000p-4f,-0x1.2000000000000p-4f,-0x1.4000000000000p-4f,-0x1.6000000000000p-4f,-0x1.8000000000000p-4f,-0x1.a000000000000p-4f,-0x1.c000000000000p-4f,-0x1.e000000000000p-4f,
+ -0x1.0000000000000p-3f,-0x1.2000000000000p-3f,-0x1.4000000000000p-3f,-0x1.6000000000000p-3f,-0x1.8000000000000p-3f,-0x1.a000000000000p-3f,-0x1.c000000000000p-3f,-0x1.e000000000000p-3f,
+ -0x1.0000000000000p-2f,-0x1.2000000000000p-2f,-0x1.4000000000000p-2f,-0x1.6000000000000p-2f,-0x1.8000000000000p-2f,-0x1.a000000000000p-2f,-0x1.c000000000000p-2f,-0x1.e000000000000p-2f,
+ -0x1.0000000000000p-1f,-0x1.2000000000000p-1f,-0x1.4000000000000p-1f,-0x1.6000000000000p-1f,-0x1.8000000000000p-1f,-0x1.a000000000000p-1f,-0x1.c000000000000p-1f,-0x1.e000000000000p-1f,
+ -0x1.0000000000000p+0f,-0x1.2000000000000p+0f,-0x1.4000000000000p+0f,-0x1.6000000000000p+0f,-0x1.8000000000000p+0f,-0x1.a000000000000p+0f,-0x1.c000000000000p+0f,-0x1.e000000000000p+0f,
+ -0x1.0000000000000p+1f,-0x1.2000000000000p+1f,-0x1.4000000000000p+1f,-0x1.6000000000000p+1f,-0x1.8000000000000p+1f,-0x1.a000000000000p+1f,-0x1.c000000000000p+1f,-0x1.e000000000000p+1f,
+ -0x1.0000000000000p+2f,-0x1.2000000000000p+2f,-0x1.4000000000000p+2f,-0x1.6000000000000p+2f,-0x1.8000000000000p+2f,-0x1.a000000000000p+2f,-0x1.c000000000000p+2f,-0x1.e000000000000p+2f,
+ -0x1.0000000000000p+3f,-0x1.2000000000000p+3f,-0x1.4000000000000p+3f,-0x1.6000000000000p+3f,-0x1.8000000000000p+3f,-0x1.a000000000000p+3f,-0x1.c000000000000p+3f,-0x1.e000000000000p+3f,
+ -0x1.0000000000000p+4f,-0x1.2000000000000p+4f,-0x1.4000000000000p+4f,-0x1.6000000000000p+4f,-0x1.8000000000000p+4f,-0x1.a000000000000p+4f,-0x1.c000000000000p+4f,-0x1.e000000000000p+4f,
+ -0x1.0000000000000p+5f,-0x1.2000000000000p+5f,-0x1.4000000000000p+5f,-0x1.6000000000000p+5f,-0x1.8000000000000p+5f,-0x1.a000000000000p+5f,-0x1.c000000000000p+5f,-0x1.e000000000000p+5f,
+ -0x1.0000000000000p+6f,-0x1.2000000000000p+6f,-0x1.4000000000000p+6f,-0x1.6000000000000p+6f,-0x1.8000000000000p+6f,-0x1.a000000000000p+6f,-0x1.c000000000000p+6f,-0x1.e000000000000p+6f,
+ -0x1.0000000000000p+7f,-0x1.2000000000000p+7f,-0x1.4000000000000p+7f,-0x1.6000000000000p+7f,-0x1.8000000000000p+7f,-0x1.a000000000000p+7f,-0x1.c000000000000p+7f,-0x1.e000000000000p+7f,
+ -0x1.0000000000000p+8f,-0x1.2000000000000p+8f,-0x1.4000000000000p+8f,-0x1.6000000000000p+8f,-0x1.8000000000000p+8f,-0x1.a000000000000p+8f,-0x1.c000000000000p+8f, NAN,
+};
+static inline float e4m3_decode(uint8_t b){ return E4M3_LUT[b]; }
+
+#define FP8_BLOCK 128
+static inline int64_t fp8_nblk(int n){ return ((int64_t)n + FP8_BLOCK - 1) / FP8_BLOCK; }
+
+/* y[S,O] = x[S,I] @ W^T, W raw e4m3 bytes (byte-identical layout to fmt=1) +
+ * per-128x128-BLOCK f32 scale [ceil(O/128),ceil(I/128)]. Scalar reference path
+ * (no SIMD in v1 -- BW-bound like the Metal kernel, and this format's hot path
+ * is the GPU one; a vectorized CPU kernel is future work if measured needed).
+ * Mirrors matmul_i3's double-accumulate-across-groups / float-within-group
+ * convention so cross-block cancellation doesn't cost precision unfairly. */
+static void matmul_fp8(float *y, const float *x, const uint8_t *q8, const float *bscale,
+ int S, int I, int O){
+ int64_t nblkI = fp8_nblk(I);
+ #pragma omp parallel for schedule(static)
+ for(int o=0;o<O;o++){
+ const uint8_t *w = q8 + (int64_t)o*I;
+ int64_t blkO = o / FP8_BLOCK;
+ const float *scl = bscale + blkO*nblkI;
+ for(int s=0;s<S;s++){
+ const float *xs = x + (int64_t)s*I;
+ double a=0;
+ for(int64_t bi=0; bi*FP8_BLOCK<I; bi++){
+ int base=(int)(bi*FP8_BLOCK); int blen=FP8_BLOCK; if(base+blen>I) blen=I-base;
+ float sc=scl[bi]; float acc=0;
+ for(int i=base;i<base+blen;i++) acc += e4m3_decode(w[i])*xs[i];
+ a += (double)acc*sc;
+ }
+ y[(int64_t)s*O+o]=(float)a;
+ }
+ }
+}
+
/* ---- IDOT: integer dot kernels (int8-quantized activations) --------------- */
#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
#define IDOT_KERNEL "avx512-vnni"
diff --git a/c/tests/test_fp8_load.c b/c/tests/test_fp8_load.c
new file mode 100644
index 00000000..f04cca02
--- /dev/null
+++ b/c/tests/test_fp8_load.c
@@ -0,0 +1,369 @@
+/* fmt=7 (native FP8-e4m3 passthrough) loader-seam tests.
+ *
+ * fmt=7, PUBLIC ordinal assigned by the maintainer on #524: this format was
+ * minted fmt=6 during original development of this branch, before dev's own
+ * #465 (E8/IQ3) claimed that ordinal upstream and merged it into dev as a
+ * REAL fmt=6 (see quant.h's E8 constants and e8_ helper functions, and
+ * qt_resolve_fmt's ns==4-tag early check); re-tagged fmt=100 (PRIVATE ORDINAL
+ * BLOCK, see colibri.c's QT struct comment) from that point forward -- there
+ * was never a build in this branch's history where this format was reachable
+ * as fmt=6 -- and graduated to fmt=7 at merge.
+ *
+ * Part A: qt_resolve_fmt disambiguation suite -- THE DESIGN LANDMINE. fmt=7
+ * weight bytes are byte-identical to fmt=1 (int8): both are O*I raw bytes.
+ * The two are told apart ONLY by the scale array's byte count (per-row O*4 for
+ * fmt=1, per-128x128-block ceil(O/128)*ceil(I/128)*4 for fmt=7, THIS build's
+ * implemented f32 scale encoding). For some shapes those two counts coincide
+ * exactly -- qt_resolve_fmt must REFUSE (exit(1)) rather than guess. Refusal
+ * is tested via fork()+waitpid(), mirroring tests/test_st_pread.c's
+ * established convention for exit(1)-terminated paths.
+ *
+ * Part A2: fmt=6 (E8/IQ3, upstream #465) vs fmt=7 collision at [O<=128 or
+ * O in a 128-block-count range, I=98] -- SECOND DESIGN LANDMINE.
+ *
+ * Part A3: fmt=7's scale ENCODING is a declared property, not a hardcoded
+ * constant -- f32 (Part A/A2 above) is what this build implements. A UE8M0
+ * (1 byte/block) encoding is a REAL, distinct byte signature (the DeepSeek-V4
+ * checkpoint format for this identical weight geometry) this build recognizes
+ * and refuses BY NAME rather than misreading.
+ *
+ * Part B: qt_from_disk loader-seam -- writes a real single-shard .safetensors
+ * file containing an fmt=7 tensor (U8 weight + per-block F32 .qs) next to an
+ * int8 control tensor of a DIFFERENT, non-colliding shape, loads both through
+ * qt_from_disk, and checks the byte-count/.qs-size inference picks fmt=7 vs
+ * fmt=1 correctly and the loaded weights dequantize identically to a reference.
+ * Mirrors tests/test_int3_load.c's structure for fmt=5.
+ *
+ * Part C: qt_bytes()/qt_scale_bytes() byte-accounting for fmt=7.
+ *
+ * This build has NO container metadata stamp (see qt_resolve_fmt's own header
+ * comment) -- every ambiguous/unimplemented-encoding shape below refuses
+ * unconditionally; stamp-resolves-ambiguity behavior is a separate, follow-up
+ * PR (registry + metadata stamp), not exercised here. */
+#define main coli_glm_main_unused
+#include "../colibri.c"
+#undef main
+
+#include <stdio.h>
+#include <string.h>
+#include <sys/stat.h>
+#ifndef _WIN32
+#include <unistd.h>
+#include <sys/wait.h>
+#endif
+
+static int fails = 0;
+#define CHECK(c) do{ if(!(c)){ printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #c); fails++; } }while(0)
+
+static uint64_t rng = 0xFEEDFACE0DDBA11ull;
+static float rndf(void){ rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17;
+ return ((int64_t)(rng & 0xFFFFF) - 0x80000) / (float)0x80000; }
+static uint8_t rndbyte_nonan(void){
+ for(;;){ rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17;
+ uint8_t b = (uint8_t)(rng & 0xFF);
+ if(b != 0x7F && b != 0xFF) return b; }
+}
+
+/* ---- Part A: qt_resolve_fmt disambiguation (in-process for non-refusing
+ * cases, fork+waitpid for refusing ones -- qt_resolve_fmt exit(1)s in place,
+ * it does not return an error code). ---- */
+
+static int expect_fmt(int O, int I, int64_t nb, int64_t ns, int expect_fmt_val, const char *tag){
+ int gs=0;
+ int fmt = qt_resolve_fmt(tag, O, I, nb, ns, &gs);
+ if(fmt != expect_fmt_val){
+ printf("FAIL %s: got fmt=%d, expected fmt=%d (O=%d I=%d nb=%lld ns=%lld)\n",
+ tag, fmt, expect_fmt_val, O, I, (long long)nb, (long long)ns);
+ return 0;
+ }
+ return 1;
+}
+
+static int expect_refuse(int O, int I, int64_t nb, int64_t ns, const char *tag){
+#ifndef _WIN32
+ int pipefd[2]; if(pipe(pipefd)!=0) return 0;
+ pid_t pid = fork();
+ if(pid < 0) return 0;
+ if(pid == 0){
+ dup2(pipefd[1],2); close(pipefd[0]); close(pipefd[1]);
+ int gs=0;
+ qt_resolve_fmt(tag, O, I, nb, ns, &gs); /* must exit(1) inside; must NOT return */
+ _exit(42); /* reaching here is the bug */
+ }
+ close(pipefd[1]);
+ char err[1024]={0}; ssize_t n=read(pipefd[0],err,sizeof(err)-1); (void)n;
+ close(pipefd[0]);
+ int status=0; waitpid(pid,&status,0);
+ int ok = WIFEXITED(status) && WEXITSTATUS(status)==1;
+ if(!ok){
+ printf("FAIL %s: expected exit(1) refusal, got status=%d, stderr=%.200s\n", tag, status, err);
+ return 0;
+ }
+ if(!strstr(err,"refus")){
+ printf("FAIL %s: exited(1) but message lacked a refusal explanation: %.200s\n", tag, err);
+ return 0;
+ }
+ return 1;
+#else
+ /* fork/pipe/waitpid are POSIX -- mirrors tests/test_st_pread.c's own
+ * Windows arm: skip the exit(1) subprocess check, print an explicit
+ * (never silent) skip line per case, count it as passing rather than
+ * failing. Every non-refusing case in this file (expect_fmt and its
+ * callers) still runs and asserts for real on Windows. */
+ printf("skipped on Windows (no fork): %s\n", tag);
+ (void)O; (void)I; (void)nb; (void)ns;
+ return 1;
+#endif
+}
+
+static void test_disambiguation(void){
+ /* --- non-degenerate golden paths (unambiguous either way) --- */
+ CHECK(expect_fmt(4096,4096,(int64_t)4096*4096,4096*4,1,"plain int8 4096x4096"));
+ /* spec's own worked example: [2048,6144] expert -> block scale [16,48] */
+ CHECK(expect_fmt(2048,6144,(int64_t)2048*6144,16LL*48*4,7,"fp8 2048x6144 (spec example)"));
+ CHECK(expect_fmt(384,6144,(int64_t)384*6144,3LL*48*4,7,"fp8 384x6144 non-square block grid"));
+
+ /* --- degenerate shapes: O<=128 makes nblkO==1, so ns_blk==nblkI*4 can
+ * equal ns_row==O*4 whenever nblkI==O. Exhaustive-in-spirit sweep of the
+ * boundary the design landmine describes. --- */
+ CHECK(expect_refuse(1,1, 1, 4, "degenerate O=1 I=1 (nblkI=1=O)"));
+ CHECK(expect_refuse(1,128, 128, 4, "degenerate O=1 I=128 (nblkI=1=O, I at block edge)"));
+ CHECK(expect_refuse(2,256, 2LL*256, 8, "degenerate O=2 I=256 (nblkI=2=O)"));
+ CHECK(expect_refuse(6,768, 6LL*768, 24, "degenerate O=6 I=768 (nblkI=6=O)"));
+ CHECK(expect_refuse(128,16384,128LL*16384, 512, "degenerate O=128 I=16384 (nblkO=1,nblkI=128=O)"));
+ /* O>128 degenerate case: nblkO=2, need nblkI=O/2 -- O=256,I=16384 -> nblkI=128, 2*128=256=O */
+ CHECK(expect_refuse(256,16384,256LL*16384, 1024, "degenerate O=256 I=16384 (nblkO=2,nblkI=128, product=O)"));
+ /* k=3: the ambiguity isn't a one-off O=256 coincidence -- it's structural for ANY
+ * O that's a multiple of 128 (nblkO=k), since nblkI==128 (I in (16256,16384]) always
+ * makes nblkO*nblkI == k*128 == O. One more multiple (O=384=3*128) confirms the
+ * condition generalizes past the k=2 worked example, not just a re-derivation. */
+ CHECK(expect_refuse(384,16384,384LL*16384, 1536, "degenerate O=384 I=16384 (nblkO=3,nblkI=128, k=3, product=O)"));
+
+ /* --- boundary-ADJACENT non-degenerate cases: one step past each
+ * degenerate case above, both interpretations now legitimately resolve. --- */
+ CHECK(expect_fmt(1,129, 129, 4, 1, "adjacent O=1 I=129 as fmt=1 (ns=row)"));
+ CHECK(expect_fmt(1,129, 129, 8, 7, "adjacent O=1 I=129 as fmt=7 (ns=block, nblkI=2)"));
+ CHECK(expect_fmt(2,257, 2LL*257, 8, 1, "adjacent O=2 I=257 as fmt=1 (ns=row)"));
+ CHECK(expect_fmt(2,257, 2LL*257, 12, 7, "adjacent O=2 I=257 as fmt=7 (ns=block, nblkI=3)"));
+
+ /* --- neither interpretation matches: garbage .qs size, must still refuse
+ * (the pre-existing generic-mismatch path, exercised through the fmt=7-aware
+ * function to confirm the new code didn't disturb it). --- */
+ CHECK(expect_refuse(10,10, 100, 999, "garbage ns matches neither row nor block layout"));
+}
+
+/* ---- Part A2: fmt=6 (E8/IQ3, upstream #465, merged into dev) vs fmt=7
+ * (this branch's fp8-e4m3-b128) collision -- SECOND DESIGN LANDMINE, see the
+ * derivation in qt_resolve_fmt's own comment. e8_rowbytes(I) is the constant
+ * 98 for every I in (0,256], so dev's fmt=6 tag check (ns==4 &&
+ * nb==O*e8_rowbytes(I)) collapses to nb==O*98 -- which coincides with
+ * fp8-e4m3-b128's raw weight bytes (O*I) at the ONE value I==98, where a
+ * SINGLE-BLOCK (O<=128) fp8 tensor with f32 block scales, OR a FOUR-BLOCK
+ * fp8 tensor with ue8m0 (1 byte/block) scales, ALSO carries exactly ns==4,
+ * same as the E8 tag. An unstamped [I=98] fp8-e4m3-b128 tensor at either of
+ * those shapes is therefore byte-for-byte indistinguishable from a genuine
+ * fmt=6 tensor: nb AND ns both coincide, not just ns (contrast the fmt=1/
+ * fmt=7 collision in Part A, where only ns ever coincides). O==1 stacks a
+ * THIRD candidate: fmt=1's per-row ns (O*4) is also 4 there. This build has
+ * no stamp to resolve any of these -- every one refuses. */
+static void test_fmt6_fp8_collision(void){
+ /* O=64, I=98: nblkO=nblkI=1 (fmt=7, single block, f32 scales) -> ns=4;
+ * e8_blocks(98)=1 -> nb=O*98=6272 for BOTH interpretations, and fmt=6's
+ * .qs tag is always exactly one f32 -> ns=4 too. Must refuse. */
+ int64_t nb64=(int64_t)64*98;
+ CHECK(expect_refuse(64,98, nb64, 4, "fmt=6/fmt=7(f32) collision O=64 I=98"));
+ /* boundary O=128 variant: still nblkO=1 for fmt=7 (128<=128), same collision. */
+ int64_t nb128=(int64_t)128*98;
+ CHECK(expect_refuse(128,98, nb128, 4, "fmt=6/fmt=7(f32) collision O=128 I=98"));
+
+ /* O=1, I=98: a THIRD candidate stacks on (fmt=1 plain int8 per-row, ns==O*4==4
+ * too) -- a genuine three-way ambiguity. Must refuse. */
+ int64_t nb1=(int64_t)1*98;
+ CHECK(expect_refuse(1,98, nb1, 4, "fmt=1/fmt=6/fmt=7(f32) THREE-way collision O=1 I=98"));
+
+ /* O in (384,512], I=98: nblkO=4, nblkI=1, product=4 -- a fmt=7 tensor with
+ * UE8M0 (1 byte/block) scales also lands at ns==4*1==4 here, the SAME tag
+ * fmt=6 uses. Must refuse (and, since this build doesn't implement ue8m0
+ * decode at all, would refuse on that basis alone even without the fmt=6
+ * collision -- this case exercises the OUTER fmt=6 guard specifically). */
+ int64_t nb400=(int64_t)400*98;
+ CHECK(expect_refuse(400,98, nb400, 4, "fmt=6/fmt=7(ue8m0, 4-block) collision O=400 I=98"));
+
+ /* regression guard: a GENUINE (non-colliding) fmt=6 fixture -- I!=98, so
+ * e8_rowbytes(I)==98 does NOT equal O*I -- must keep resolving to fmt=6.
+ * Mirrors test_e8_kernel.c's own O=24,I=512 shape and a small
+ * single-super-block shape (I=256, inside the (0,256] range where
+ * e8_rowbytes(I)==98 but I!=98, so still non-colliding). */
+ CHECK(expect_fmt(24,512, (int64_t)24*e8_rowbytes(512), 4, 6,
+ "genuine fmt=6 (non-colliding) O=24 I=512 -> still resolves to fmt=6"));
+ CHECK(expect_fmt(64,256, (int64_t)64*e8_rowbytes(256), 4, 6,
+ "genuine fmt=6 (non-colliding) O=64 I=256 (I!=98, no collision) -> fmt=6"));
+}
+
+/* ---- Part A3: fmt=7's scale ENCODING is a declared property -- UE8M0
+ * recognized, refused by name (not implemented in this build). ---- */
+static void test_ue8m0_scale_refusal(void){
+ /* [2048,6144] (spec example shape, same as Part A's fmt=7/f32 golden
+ * path): nblkO=16, nblkI=48, product=768 blocks. A UE8M0 sidecar is
+ * exactly 1 byte/block -> ns=768, distinct from BOTH fmt=1's per-row
+ * count (O*4=8192) and this build's f32 block-scale count (768*4=3072).
+ * Clean, unambiguous UE8M0 signature -- must name-refuse, not silently
+ * treat it as a truncated/corrupt f32 array or match it to fmt=1. */
+ int64_t nb=(int64_t)2048*6144;
+ CHECK(expect_refuse(2048,6144, nb, 768,
+ "fp8-e4m3-b128 with ue8m0 scales (spec-shaped, non-degenerate) -> recognized, refused by name"));
+
+ /* O=1, I=400: nblkO=1, nblkI=ceil(400/128)=4, product=4 -- a UE8M0 sidecar
+ * here is ns=4*1=4, which ALSO equals fmt=1's per-row count (O*4=4): the
+ * same small-O regime that produces the f32-vs-fmt=1 collision in Part A
+ * produces a ue8m0-vs-fmt=1 collision too. Must still refuse (combined
+ * message), not silently pick fmt=1. */
+ int64_t nb1=(int64_t)1*400;
+ CHECK(expect_refuse(1,400, nb1, 4,
+ "fp8-e4m3-b128 with ue8m0 scales, ALSO colliding with fmt=1 per-row (O=1) -> refused"));
+}
+
+/* ---- Part B: qt_from_disk loader-seam (real safetensors file) ---- */
+
+static void deq_fmt7(const QT *t, float *dq){
+ int64_t nblkI = fp8_nblk(t->I);
+ for(int o=0;o<t->O;o++){
+ int64_t blkO = o/FP8_BLOCK; const float *scl = t->s + blkO*nblkI;
+ for(int i=0;i<t->I;i++){
+ int64_t bi = i/FP8_BLOCK;
+ dq[(int64_t)o*t->I+i] = e4m3_decode(t->q8[(int64_t)o*t->I+i]) * scl[bi];
+ }
+ }
+}
+static void deq_fmt1(const QT *t, float *dq){
+ for(int o=0;o<t->O;o++){ float s=t->s[o];
+ for(int i=0;i<t->I;i++) dq[(int64_t)o*t->I+i]=(float)t->q8[(int64_t)o*t->I+i]*s; }
+}
+
+#define CDIV(n,d) (((n)+(d)-1)/(d))
+
+static void test_loader_seam(void){
+ enum { O7=8, I7=256 }; /* nblkO=1, nblkI=2 -> 2 block scales total */
+ enum { O1=5, I1=64 }; /* DIFFERENT shape from the fp8 tensor: O1*I1=320
+ * bytes, ns=O1*4=20 -- neither collides with the
+ * fp8 tensor's own byte counts (kept deliberately
+ * distinct so this is a plain, non-degenerate
+ * negative control, not another landmine case). */
+ enum { NBLK7 = CDIV(O7,128) * CDIV(I7,128) }; /* must be a compile-time constant expression for
+ * the static array below -- fp8_nblk() is a real
+ * function (runtime, not constexpr), so it can't
+ * size a `static` array even with literal inputs. */
+ static uint8_t q7[O7*I7]; static float s7[NBLK7];
+ for(int i=0;i<O7*I7;i++) q7[i]=rndbyte_nonan();
+ for(int i=0;i<(int)(sizeof s7/sizeof *s7);i++) s7[i]=0.01f+0.001f*(float)i;
+
+ static int8_t q1[O1*I1]; static float s1[O1];
+ for(int i=0;i<O1*I1;i++) q1[i]=(int8_t)(rndbyte_nonan()-128);
+ for(int i=0;i<O1;i++) s1[i]=0.02f+0.001f*(float)i;
+
+ const char *dir="tests/tmp_fp8_snap";
+#ifdef _WIN32
+ mkdir(dir);
+#else
+ mkdir(dir,0755);
+#endif
+ char path[300]; snprintf(path,sizeof path,"%s/model.safetensors",dir);
+ int64_t nb7=(int64_t)O7*I7, ns7=(int64_t)(sizeof s7);
+ int64_t nb1=(int64_t)O1*I1, ns1=(int64_t)O1*4;
+ char hdr[1024];
+ int hl=snprintf(hdr,sizeof hdr,
+ "{\"w7\":{\"dtype\":\"U8\",\"shape\":[%lld],\"data_offsets\":[0,%lld]},"
+ "\"w7.qs\":{\"dtype\":\"F32\",\"shape\":[%lld],\"data_offsets\":[%lld,%lld]},"
+ "\"w1\":{\"dtype\":\"U8\",\"shape\":[%lld],\"data_offsets\":[%lld,%lld]},"
+ "\"w1.qs\":{\"dtype\":\"F32\",\"shape\":[%lld],\"data_offsets\":[%lld,%lld]}}",
+ (long long)nb7,(long long)nb7,
+ (long long)(ns7/4),(long long)nb7,(long long)(nb7+ns7),
+ (long long)nb1,(long long)(nb7+ns7),(long long)(nb7+ns7+nb1),
+ (long long)O1,(long long)(nb7+ns7+nb1),(long long)(nb7+ns7+nb1+ns1));
+ FILE *f=fopen(path,"wb");
+ if(!f){ printf("FAIL: cannot create %s (run from c/, like tools/run_tests.py does)\n", path); fails++; return; }
+ uint64_t hlen=(uint64_t)hl;
+ fwrite(&hlen,8,1,f); fwrite(hdr,1,hl,f);
+ fwrite(q7,1,(size_t)nb7,f); fwrite(s7,1,(size_t)ns7,f);
+ fwrite(q1,1,(size_t)nb1,f); fwrite(s1,1,(size_t)ns1,f);
+ fclose(f);
+
+ static Model gm; /* only gm.S is used by qt_from_disk */
+ st_init(&gm.S, dir);
+
+ QT t7; memset(&t7,0,sizeof t7);
+ qt_from_disk(&gm,"w7",O7,I7,8,0,&t7);
+ CHECK(t7.fmt==7);
+ CHECK(t7.q8!=NULL && t7.s!=NULL); /* both weight and scale allocated (qalloc, not falloc) */
+ static float dq_load[O7*I7], dq_ref[O7*I7];
+ deq_fmt7(&t7,dq_load);
+ QT tr7={.fmt=7,.q8=(int8_t*)q7,.s=s7,.O=O7,.I=I7};
+ deq_fmt7(&tr7,dq_ref);
+ CHECK(memcmp(dq_load,dq_ref,sizeof dq_ref)==0);
+
+ QT t1; memset(&t1,0,sizeof t1);
+ qt_from_disk(&gm,"w1",O1,I1,8,0,&t1);
+ CHECK(t1.fmt==1); /* negative control: plain int8 still resolves as fmt=1 */
+ static float dq_load1[O1*I1], dq_ref1[O1*I1];
+ deq_fmt1(&t1,dq_load1);
+ QT tr1={.fmt=1,.q8=q1,.s=s1,.O=O1,.I=I1};
+ deq_fmt1(&tr1,dq_ref1);
+ CHECK(memcmp(dq_load1,dq_ref1,sizeof dq_ref1)==0);
+
+ unlink(path); rmdir(dir);
+}
+
+/* ---- Part C: qt_bytes()/qt_scale_bytes() byte-accounting for fmt=7 ----
+ *
+ * qt_bytes() must not fall through to the fmt=2 (packed int4, O*ceil(I/2)+O*4)
+ * default -- for a real fp8 tensor that would undercount the resident byte
+ * count by roughly half (an AUTOPIN/RAM-budget-feeding hazard). qt_wire_mmap/
+ * qt_unwire_mmap must not hardcode scale_b=O*4 (per-row) either -- wrong for
+ * fmt=7's per-128x128-block scale array -- hence the dedicated
+ * qt_scale_bytes() helper shared by qt_bytes() and both wire functions so
+ * there is exactly one place that knows each format's scale geometry. This
+ * test exercises the arithmetic directly (no qt_from_disk/disk I/O, no mlock
+ * syscall -- qt_wire_mmap's actual mem_wire() call is environment-dependent
+ * (RLIMIT_MEMLOCK) and not what changed; the byte-count formula is) across
+ * the shapes already used elsewhere in this file plus a block-edge
+ * (non-128-multiple) case. */
+static void check_fp8_bytes(int O, int I, const char *tag){
+ QT t; memset(&t,0,sizeof t); t.fmt=7; t.O=O; t.I=I; t.gs=0;
+ int64_t nblkO=fp8_nblk(O), nblkI=fp8_nblk(I), nblk=nblkO*nblkI;
+ int64_t want_total = (int64_t)O*I + nblk*4;
+ int64_t want_scale = nblk*4;
+ int64_t got_total = qt_bytes(&t);
+ int64_t got_scale = qt_scale_bytes(&t);
+ if(got_total != want_total)
+ printf("FAIL %s: qt_bytes=%lld want=%lld\n", tag, (long long)got_total, (long long)want_total);
+ CHECK(got_total == want_total);
+ if(got_scale != want_scale)
+ printf("FAIL %s: qt_scale_bytes=%lld want=%lld\n", tag, (long long)got_scale, (long long)want_scale);
+ CHECK(got_scale == want_scale);
+ /* weight_b, as qt_wire_mmap/qt_unwire_mmap now compute it, must land on the exact
+ * O*I raw-byte weight region -- not short (partial mlock) or long (mlock past the
+ * allocation, undefined behavior) by even one byte. */
+ CHECK(got_total - got_scale == (int64_t)O*I);
+ /* regression guard: the fmt=2 (packed-nibble) formula must NOT be what fmt=7
+ * returns -- confirm the value has actually MOVED off it (catches a silent
+ * revert of the fmt==7 branch order/placement, not just a formula typo). For
+ * O=1,I=1 the two formulas coincide by coincidence (both give 1+4=5), so that
+ * shape is skipped for this particular guard -- the other three shapes below are
+ * chosen to avoid the coincidence. */
+ int64_t old_wrong = (int64_t)O*((I+1)/2) + (int64_t)O*4;
+ if(!(O==1 && I==1)) CHECK(got_total != old_wrong);
+}
+
+int main(void){
+ test_disambiguation();
+ test_fmt6_fp8_collision();
+ test_ue8m0_scale_refusal();
+ test_loader_seam();
+ check_fp8_bytes(2048,6144, "qt_bytes fmt=7 gate/up-shaped O=2048 I=6144 (spec example)");
+ check_fp8_bytes(6144,2048, "qt_bytes fmt=7 down-shaped O=6144 I=2048");
+ check_fp8_bytes(130,200, "qt_bytes fmt=7 block edges O,I both non-mult-128");
+ check_fp8_bytes(1,1, "qt_bytes fmt=7 degenerate 1x1");
+ if(fails){ printf("fp8 loader-seam tests: %d FAILED\n", fails); return 1; }
+ printf("fp8 loader-seam tests: ok\n");
+ return 0;
+}
diff --git a/c/tests/test_fp8_passthrough.c b/c/tests/test_fp8_passthrough.c
new file mode 100644
index 00000000..61fb3709
--- /dev/null
+++ b/c/tests/test_fp8_passthrough.c
@@ -0,0 +1,165 @@
+/* fmt=7 (native FP8-e4m3 passthrough) CPU kernel tests: LUT exactness (all 256
+ * byte codes, incl. +-0, subnormals, NaN), matmul_fp8 vs a double-precision
+ * block-scale reference on block-edge (non-128-multiple O/I) shapes, a
+ * non-square-block-grid case with a distinct scale per block (catches a
+ * transposed/swapped block-index stride the same way test_backend_metal.mm's
+ * "stride-audit" case does for the GPU kernel), and the NaN-propagation policy
+ * (decode -> real IEEE NaN -> flows through the dot product; see quant.h's
+ * e4m3_decode/E4M3_LUT comment for the documented policy + rationale).
+ * Pure quant.h test: no Model/QT/disk dependency (see quant.h's own "pure
+ * compute" header comment) -- the loader-seam (qt_resolve_fmt disambiguation,
+ * qt_from_disk) is covered separately in test_fp8_load.c.
+ *
+ * fmt=7, PUBLIC ordinal assigned by the maintainer on #524: this format was
+ * minted fmt=6 during original development of this branch, before dev's own
+ * #465 (E8/IQ3) claimed that ordinal upstream; re-tagged fmt=100 (PRIVATE
+ * ORDINAL BLOCK, see colibri.c's QT struct comment) from that point forward --
+ * there was never a build in this branch's history where this format was
+ * reachable as fmt=6 -- and graduated to fmt=7 at merge. */
+#include "../quant.h"
+#include <stdio.h>
+#include <string.h>
+#include <math.h>
+
+static int fails = 0;
+#define CHECK(c) do{ if(!(c)){ printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #c); fails++; } }while(0)
+
+static uint64_t rng = 0xC0FFEE1234567ull;
+static float rndf(void){ rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17;
+ return ((int64_t)(rng & 0xFFFFF) - 0x80000) / (float)0x80000; }
+/* random byte EXCLUDING the two NaN codes (0x7F/0xFF) -- used by the accuracy/
+ * magnitude tests below so a stray NaN term doesn't turn `rel>tol` trivially
+ * false (NaN compares false against everything) and mask a real kernel bug.
+ * NaN handling itself is checked separately, deliberately, below. */
+static uint8_t rndbyte_nonan(void){
+ for(;;){ rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17;
+ uint8_t b = (uint8_t)(rng & 0xFF);
+ if(b != 0x7F && b != 0xFF) return b; }
+}
+
+/* independent reference decode (bit manipulation, not the LUT under test) --
+ * cross-checked offline against torch.float8_e4m3fn byte-for-byte (256/256
+ * match, RAN evidence, see the build report). Kept here so a transcription
+ * bug in E4M3_LUT's 256 hex-float literals is still caught by this test even
+ * though both were derived from the same formula. */
+static float ref_e4m3(uint8_t b){
+ unsigned sign=(b>>7)&1, exp=(b>>3)&0xF, mant=b&0x7;
+ if(exp==0xF && mant==0x7) return NAN;
+ float val;
+ if(exp==0) val = (float)mant * (1.0f/8.0f) * powf(2.0f, 1.0f-7.0f);
+ else val = (1.0f + (float)mant*(1.0f/8.0f)) * powf(2.0f, (float)exp-7.0f);
+ return sign ? -val : val;
+}
+
+static void test_lut(void){
+ int nan_codes=0;
+ for(int b=0;b<256;b++){
+ float got = e4m3_decode((uint8_t)b);
+ float ref = ref_e4m3((uint8_t)b);
+ if(isnan(ref)){ CHECK(isnan(got)); nan_codes++; continue; }
+ CHECK(got == ref);
+ if(got==0.f) CHECK(signbit(got)==signbit(ref)); /* +-0 must keep its sign bit */
+ }
+ CHECK(nan_codes==2); /* exactly 0x7F and 0xFF */
+ CHECK(isnan(e4m3_decode(0x7F)) && isnan(e4m3_decode(0xFF)));
+ CHECK(e4m3_decode(0x7E)==448.0f && e4m3_decode(0xFE)==-448.0f); /* max finite, OCP E4M3FN */
+ CHECK(e4m3_decode(0x01)==0x1.0p-9f); /* min positive subnormal = 2^-9 */
+ CHECK(e4m3_decode(0x00)==0.0f && !signbit(e4m3_decode(0x00)));
+ CHECK(e4m3_decode(0x80)==0.0f && signbit(e4m3_decode(0x80))); /* -0.0 */
+}
+
+/* double-precision block-scale reference: mirrors matmul_fp8's formula exactly
+ * (w[o,i] = e4m3_decode(byte) * scale[o/128,i/128]) but every intermediate is
+ * double so it can serve as ground truth for matmul_fp8's own float/double mix. */
+static void ref_matmul_fp8(double *y, double *mag, const float *x, const uint8_t *q8,
+ const float *bscale, int S, int I, int O){
+ int64_t nblkI = fp8_nblk(I);
+ for(int o=0;o<O;o++){
+ const uint8_t *w = q8 + (int64_t)o*I;
+ int64_t blkO = o / FP8_BLOCK;
+ const float *scl = bscale + blkO*nblkI;
+ for(int s=0;s<S;s++){
+ const float *xs = x + (int64_t)s*I;
+ double a=0, m=0;
+ for(int i=0;i<I;i++){
+ int64_t bi = i/FP8_BLOCK;
+ double term = (double)e4m3_decode(w[i]) * (double)xs[i] * (double)scl[bi];
+ a += term; m += fabs(term);
+ }
+ y[(int64_t)s*O+o]=a; mag[(int64_t)s*O+o]=m;
+ }
+ }
+}
+
+/* magnitude-relative Sigma|terms| oracle (the convention test_backend_metal.mm's
+ * cpu_ref_grouped/run_grouped established for fmt=4 and reuses for fmt=7 on the
+ * GPU side): compare error against the sum of |terms|, not the (possibly
+ * near-zero, cancellation-prone) result itself. */
+static int check_close(const float *got, const double *ref, const double *mag, int n, double tol, const char *tag){
+ double worst=0; int bad=0;
+ for(int i=0;i<n;i++){
+ double d = fabs((double)got[i]-ref[i]);
+ double rel = mag[i]>1e-30 ? d/mag[i] : d;
+ if(rel>worst) worst=rel;
+ if(rel>tol) bad++;
+ }
+ if(bad) printf(" %-40s worst_rel=%.3e FAIL (%d/%d over tol)\n", tag, worst, bad, n);
+ return bad==0;
+}
+
+static void run_block_case(int O, int I, const char *tag){
+ int64_t nblkO=fp8_nblk(O), nblkI=fp8_nblk(I), nblk=nblkO*nblkI;
+ uint8_t *q8 = malloc((size_t)O*I);
+ float *bscale = malloc((size_t)nblk*sizeof(float));
+ float *x = malloc((size_t)3*I*sizeof(float));
+ float *y = malloc((size_t)3*O*sizeof(float));
+ double *yr = malloc((size_t)3*O*sizeof(double));
+ double *mag = malloc((size_t)3*O*sizeof(double));
+ for(int64_t i=0;i<(int64_t)O*I;i++) q8[i]=rndbyte_nonan();
+ /* distinct, easily-distinguishable scale per block (b+1)*0.01 with alternating
+ * sign and a random jitter -- a swapped blkO/blkI stride or a wrong nblkI in the
+ * indexing picks up the WRONG block's scale, which this makes numerically loud
+ * rather than "close enough to pass by luck" (same idea as the GPU stride-audit
+ * case, applied here to the CPU kernel). */
+ for(int64_t b=0;b<nblk;b++) bscale[b] = (float)(b+1)*0.01f*((b&1)?-1.f:1.f) + rndf()*0.001f;
+ for(int i=0;i<3*I;i++) x[i]=rndf();
+ ref_matmul_fp8(yr, mag, x, q8, bscale, 3, I, O);
+ matmul_fp8(y, x, q8, bscale, 3, I, O);
+ CHECK(check_close(y, yr, mag, 3*O, 1e-5, tag));
+ free(q8); free(bscale); free(x); free(y); free(yr); free(mag);
+}
+
+static void test_nan_propagation(void){
+ /* one NaN byte in an otherwise-normal tensor -> that OUTPUT ROW is NaN (the
+ * documented policy: propagate, rely on the existing argmax_v/dist_build net
+ * downstream -- see quant.h's e4m3_decode comment). Rows that never touch the
+ * NaN byte stay finite: NaN contamination is per-dot-product, not global.
+ * O=3<=128 -> nblkO=1, so all three rows share the SAME nblkI=2-entry scale
+ * block-row (bscale[0..1]); q8 is the only thing that differs per row. */
+ enum { O=3, I=200 };
+ static uint8_t q8[O*I];
+ static float bscale[2]; /* nblkO(1)*nblkI(2) */
+ for(int i=0;i<O*I;i++) q8[i]=rndbyte_nonan();
+ q8[1*I + 37] = 0x7F; /* row 1 only, one NaN byte */
+ bscale[0]=0.1f; bscale[1]=0.11f;
+ static float x[I], y[O];
+ for(int i=0;i<I;i++) x[i]=rndf();
+ matmul_fp8(y, x, q8, bscale, 1, I, O);
+ CHECK(!isnan(y[0]) && !isnan(y[2])); /* untouched rows stay finite */
+ CHECK(isnan(y[1])); /* contaminated row propagates NaN */
+}
+
+int main(void){
+ test_lut();
+ run_block_case(2048, 6144, "block gate/up-shaped O=2048 I=6144 (spec example)");
+ run_block_case(6144, 2048, "block down-shaped O=6144 I=2048");
+ run_block_case(130, 200, "block edges: O,I both non-mult-128");
+ run_block_case(129, 128, "block edges: O just over 128, I exact");
+ run_block_case(128, 129, "block edges: O exact, I just over 128");
+ run_block_case(1, 1, "degenerate 1x1 (single sub-block)");
+ run_block_case(384, 6144, "non-square block grid nblkO=3 nblkI=48 (stride audit)");
+ test_nan_propagation();
+ if(fails){ printf("fp8 passthrough CPU tests: %d FAILED\n", fails); return 1; }
+ printf("fp8 passthrough CPU tests: ok\n");
+ return 0;
+}