Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
fp8: CPU read path + matmul_fp8, as fmt=7 (public ordinal, #524)
monotophic41fe5a7a7655
diff --git a/c/Makefile b/c/Makefileindex 10b1fa46..872b030a 100644--- a/c/Makefile+++ b/c/Makefile@@ -214,7 +214,7 @@ elsePYTHON ?= python3endifCUDA_OBJ =-TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_mirror$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_temp_env$(EXE) tests/test_tok_o200k$(EXE) tests/test_kv_alloc$(EXE) tests/test_int3$(EXE) tests/test_int3_load$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) tests/test_router_nan$(EXE) tests/test_pipe_block$(EXE) tests/test_e8_kernel$(EXE) tests/test_rans$(EXE) tests/test_corpus_draft$(EXE) tests/test_pilot_ring$(EXE)+TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_mirror$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_temp_env$(EXE) tests/test_tok_o200k$(EXE) tests/test_kv_alloc$(EXE) tests/test_int3$(EXE) tests/test_int3_load$(EXE) tests/test_fp8_passthrough$(EXE) tests/test_fp8_load$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) tests/test_router_nan$(EXE) tests/test_pipe_block$(EXE) tests/test_e8_kernel$(EXE) tests/test_rans$(EXE) tests/test_corpus_draft$(EXE) tests/test_pilot_ring$(EXE)ifneq (,$(LINUX))TEST_BINS += tests/test_uring$(EXE)endif@@ -542,6 +542,12 @@ tests/test_int3$(EXE): tests/test_int3.c colibri.c st.h uring.h json.h tok.h toktests/test_int3_load$(EXE): tests/test_int3_load.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_fp8_passthrough$(EXE): tests/test_fp8_passthrough.c quant.h+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)++tests/test_fp8_load$(EXE): tests/test_fp8_load.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_logit_nan$(EXE): tests/test_logit_nan.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)diff --git a/c/colibri.c b/c/colibri.cindex 65acbf30..10005197 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -103,7 +103,11 @@ typedef struct {* fmt=1 INT8 -> q8 (1 byte/param) + scala per riga* fmt=2 INT4 -> q4 (2 valori per byte, impacchettati) + scala per riga* INT4 e' cio' che fa stare la densa residente nei 15 GB (0.5 byte/param). */-/* fmt: 0 F32, 1 INT8, 2 INT4 (2/byte), 3 INT2 (4/byte), 4 INT4-GROUPED, 5 INT3-G64.+/* fmt: 0 F32, 1 INT8, 2 INT4 (2/byte), 3 INT2 (4/byte), 4 INT4-GROUPED, 5 INT3-G64,+ * 6 E8/IQ3 lattice, 7 FP8-E4M3 (native, passthrough -- see quant.h). fmt=7 is a+ * PUBLIC ordinal, assigned by the maintainer on #524; it developed under the+ * PRIVATE ORDINAL BLOCK convention below as fmt=100 and graduated out of that+ * block into this list at the same time this comment was written.* q4 ospita int4/int2/int3 packed. fmt=4 (grouped int4, #242): per-row nibbles + one f32* scale per group of `gs` inputs (s has O*ceil(I/gs) entries).* fmt=6 (E8/IQ3 lattice, #452): 98B per 256 weights = 3.0625 bits/weight, grid@@ -112,7 +116,53 @@ typedef struct {* fmt=5 (int3, per-GROUP scales, group=64, see quant.h I3_*): values in [-4,3] stored per* 64-input group as 24 bytes = 16B low plane (2 bits/val, int2 layout) + 8B high plane* (1 bit/val), plus ONE f32 scale PER GROUP (s has O*ceil(I/64) entries, not O). 3.5- * bits/weight effective β the quality/size sweet spot measured in the #132 ablation. */+ * bits/weight effective β the quality/size sweet spot measured in the #132 ablation.+ * fmt=7 (native FP8-e4m3 passthrough, resident/quality-core tier -- see quant.h's+ * FP8_BLOCK/e4m3_decode/matmul_fp8): q8 holds O*I raw e4m3 bytes, ONE byte per weight,+ * byte-identical layout to fmt=1's weight bytes (q4 is unused/NULL, same as fmt=1) --+ * disambiguated from fmt=1 (and, at small [O,I], from fmt=6 -- see below) purely by+ * scale geometry, see qt_resolve_fmt's "THE DESIGN LANDMINE" comment further down.+ * s holds ONE f32 scale PER 128x128 BLOCK, ceil(O/128)*ceil(I/128) entries total+ * (row-major: block-row-major then block-col), NOT O and NOT O*ceil(I/gs) --+ * qt_bytes()/qt_scale_bytes() below are the authoritative byte-count formulas. The+ * scale ENCODING is itself a declared PROPERTY of this format, not a hardcoded+ * constant -- f32 (4 bytes/block, what's above) is the value THIS build+ * implements; qt_resolve_fmt's "SCALE ENCODING IS A DECLARED PROPERTY" comment+ * documents why (a DeepSeek-V4 checkpoint ships this identical weight geometry+ * with a UE8M0 scale encoding instead) and how an unimplemented encoding is+ * recognized and refused by name rather than silently misread.+ * gs is unused (0) for fmt=7, same as fmt 1/2/3. */+/* ---- PRIVATE ORDINAL BLOCK CONVENTION ------------------------------------+ * fmt values 0-7 are upstream-assigned, public, stable ordinals -- do not+ * reuse or renumber them (fmt=7 is the newest member: assigned by the+ * maintainer on #524, after developing under this block as fmt=100 -- see+ * "graduated" above). fmt values 100+ remain this repo's PRIVATE/EXPERIMENTAL+ * block for any OTHER in-flight format proposal: ordinals a branch mints for+ * itself during development so it can't collide with a number upstream claims+ * out from under it -- exactly what happened to fmt=7's OWN earlier private+ * number, fmt=6, when #465's E8/IQ3 proposal claimed that ordinal upstream+ * while this branch was still developing against it, forcing a re-mint to+ * fmt=100 (see upstream_contribution/FORMATS_registry_draft.md for the+ * incident that prompted the rule).+ *+ * A PRIVATE-BLOCK ordinal is an internal enum value only -- qt_resolve_fmt+ * (below) infers format purely from byte arithmetic; the container on disk+ * carries no format ordinal at all. (A self-describing container stamp that+ * would persist a format's NAME, not its ordinal, is a follow-up proposal --+ * see qt_resolve_fmt's own note on where that plumbing would attach -- not+ * present in this build.) Nothing outside this binary's own compiled code+ * ever observes a 100+ number, so renumbering one later (e.g. when a format+ * is upstreamed and assigned a real public ordinal at merge, as just happened+ * for fmt=7) is a pure find-and-replace with zero on-disk or cross-version+ * compatibility impact.+ *+ * Rule for adding a new format to this branch or a future one: claim the next+ * unused 100+ integer, never a number already claimed upstream (check dev+ * before picking one -- this is exactly the mistake that forced fmt=7's own+ * proposal to renumber away from fmt=6) or by another in-flight private+ * format. Never ship a 100+ ordinal as a public default/committed-upstream+ * value -- the real ordinal is assigned by the maintainer at merge time,+ * exactly as fmt=7's was. */typedef struct {int fmt; float *qf; int8_t *q8; uint8_t *q4; float *s; int O, I, gs; /* gs=group size (0=per-row, 128=grouped) */#ifdef COLI_CUDA@@ -137,8 +187,52 @@ static int64_t qt_bytes(const QT *t){ /* byte residenti del tensore */return (int64_t)t->O*ng*24 + (int64_t)t->O*ng*4; }if(t->fmt==6) /* E8/IQ3: 98B per 256 weights, scales in-block, .qs is a 4-byte tag */return (int64_t)t->O*(((int64_t)t->I+255)/256)*98 + 4;+ if(t->fmt==7){ /* fp8-e4m3 passthrough: O*I raw e4m3 bytes (n, byte-identical layout+ * to fmt=1's weight bytes) + one f32 scale per 128x128 block+ * (FP8_BLOCK in quant.h, included below qt_bytes -- keep the+ * arithmetic literal here, same discipline as fmt=5's comment+ * above). Missing this branch would fall through to the fmt=2+ * default below (packed-nibble formula, ~half the real weight+ * bytes) and undercount a resident fp8 tensor's byte footprint --+ * feeds AUTOPIN/RAM-budget math, so this branch is load-bearing+ * from day one, not a later fix (contrast fmt=6 above, upstream's+ * own #452 review round 1 caught this exact bug shape for E8). */+ int64_t nblkO=((int64_t)t->O+127)/128, nblkI=((int64_t)t->I+127)/128;+ return n + nblkO*nblkI*4; }return (int64_t)t->O*((t->I+1)/2) + (int64_t)t->O*4; /* fmt=2 int4 per-row */}+/* scale-array byte count only, format-aware -- split out of qt_bytes() because+ * qt_wire_mmap/qt_unwire_mmap mlock the weight and scale ranges as TWO SEPARATE+ * regions (separate allocations, not one contiguous buffer: qt_from_disk always+ * qalloc's t->q8/t->q4 and t->s independently) and need the scale byte count on+ * its own to split qt_bytes(t) into a weight half and a scale half. The two call+ * sites used to hardcode scale_b=O*4 -- i.e. assume PER-ROW scale for every+ * format -- which is right for fmt 0/1/2/3 but wrong for fmt=4 (grouped,+ * O*ceil(I/gs) scales), fmt=5 (int3-g64, O*ceil(I/64) scales), and fmt=7+ * (per-128x128-block, ceil(O/128)*ceil(I/128) scales): any tensor in one of+ * those three formats reaching qt_wire_mmap/qt_unwire_mmap would mlock/munlock+ * the wrong byte ranges on BOTH halves (weight_b = qt_bytes(t)-scale_b shifts+ * too). fmt=6 (E8) is deliberately NOT covered here: its .qs is a fixed 4-byte+ * tag with no wire-mmap-relevant weight/scale split of its own (matches+ * qt_bytes' `+4` literal above), and t->fmt==6 tensors never reach+ * qt_wire_mmap/qt_unwire_mmap's mlock path in this build (int3-g64/E8 stay+ * CPU-side, see qt_cuda_upload -- MMAP wiring is a CUDA/Metal-adjacent memory+ * concern this build doesn't extend to them). fixed generically since the same+ * bug shape applies to fmt=4/5, not just the format that surfaced it -- fmt=4+ * is the routed-expert "int4-g64" format (qt_resolve_fmt assigns it to ESlot+ * g/u/d the same as any other tensor, see expert_load_impl), so this is not+ * purely a dormant fmt=7-only fix: any COLI_MMAP + mem_should_wire() session+ * pinning fmt=4 experts was already mlocking the wrong ranges before this+ * branch existed. UNVERIFIED at runtime (no model run performed -- static/+ * arithmetic fix only, see the report). */+static int64_t qt_scale_bytes(const QT *t){+ if(t->fmt==4){ int ng=(t->I+t->gs-1)/t->gs; return (int64_t)t->O*ng*4; }+ if(t->fmt==5){ int64_t ng=((int64_t)t->I+63)/64; return (int64_t)t->O*ng*4; }+ if(t->fmt==7){ int64_t nblkO=((int64_t)t->O+127)/128, nblkI=((int64_t)t->I+127)/128; return nblkO*nblkI*4; }+ return (int64_t)t->O*4; /* fmt 0 (t->s NULL, guarded by callers)/1/2/3/6: per-row (or, for+ * fmt=6, the 4-byte tag -- callers of this function never see fmt=6,+ * see the comment above; the fallback is harmless dead code for it). */+}typedef struct {float *in_ln, *post_ln;@@ -407,6 +501,14 @@ static int vk_matmul_pair_qt(QT *a, float *ya, QT *b, float *yb, const float *x,static int qt_cuda_upload(QT *t){if(t->fmt==5) return 0; /* int3-g64: no CUDA kernel yet β tensor stays CPU-side */if(t->fmt==6 && !g_cuda_e8_ready) return 0; /* E8 without its codebook would decode garbage */+ if(t->fmt==7) return 0; /* fp8-e4m3 passthrough: no CUDA kernel yet either (matches the+ * fmt=5/6 idiom above; matmul_qt_ex's own caller-side fmt exclusion+ * already keeps this from being reached with cuda_eligible tensors+ * in the exact-kernel path, but row_bytes()/weight_at() in+ * backend_cuda.cu have no fmt=7 case either -- explicit early-+ * return here so a future caller doesn't have to rediscover that by+ * tracing through to the CUDA source. UNVERIFIED at runtime: no CUDA+ * toolchain on this macOS build host to compile-check. */const void *weights = t->fmt==0 ? (const void*)t->qf: t->fmt==1 ? (const void*)t->q8 : (const void*)t->q4;if(t->fmt==4) /* grouped int4 (#334): scales are [O, ceil(I/gs)] β the plain@@ -684,13 +786,26 @@ static void matmul_i4_grouped_pair(float *yg, float *yu, const float *x,* (~+12% perplexity), measured. Every other prefill matmul keeps IDOT as before. */static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot){#ifdef COLI_METAL+ /* fmt=7 (fp8 passthrough) deliberately absent from this allowlist, same as fmt=5/6:+ * the S>=g_metal_gemm_min batched prefill GEMM path (coli_metal_gemm) only knows+ * fmt 1/2/4 in this build, so it fails CLOSED to the CPU branch below (matmul_fp8)+ * rather than being silently misread. coli_metal_gemm() also carries its own+ * internal fmt!=1&&fmt!=2&&fmt!=4 guard, so this is belt-and-braces. */if(g_metal_enabled && S>=g_metal_gemm_min && !spec_pinned() && (w->fmt==1||w->fmt==2||w->fmt==4) && !omp_in_parallel()){const void *wp = w->fmt==1 ? (const void*)w->q8 : (const void*)w->q4;if(coli_metal_gemm(y,x,wp,w->s,w->fmt,S,w->I,w->O,w->gs)) return;}#endif#ifdef COLI_CUDA- if(g_cuda_enabled && w->cuda_eligible && !w->cuda_failed && w->fmt!=5 && !omp_in_parallel()){+ /* fmt=7 excluded exactly like fmt=5/6: the CUDA backend's row_bytes()/weight_at()+ * (backend_cuda.cu) have no case for it and fall through to `return 0` / undefined+ * weight_at() behavior. row_bytes()==0 already makes coli_cuda_tensor_upload_g+ * refuse (defensive fail-closed), but that path is reached only after paying an+ * upload attempt and printing a "disabled after an error" message every load;+ * excluding it here up front is the same fmt=5/6 idiom, silent and immediate.+ * UNVERIFIED on this build (no CUDA toolchain on macOS to compile-check; traced+ * from backend_cuda.cu source only). */+ if(g_cuda_enabled && w->cuda_eligible && !w->cuda_failed && w->fmt!=5 && w->fmt!=7 && !omp_in_parallel()){const void *weights = w->fmt==0 ? (const void*)w->qf: w->fmt==1 ? (const void*)w->q8 : (const void*)w->q4;if(coli_cuda_matmul(&w->cuda,y,x,weights,w->s,w->fmt,S,w->I,w->O,w->cuda_device,w->gs)) return;@@ -716,6 +831,7 @@ static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot)if(w->fmt==1) matmul_q(y,x,w->q8,w->s,S,w->I,w->O);else if(w->fmt==3) matmul_i2(y,x,w->q4,w->s,S,w->I,w->O);else if(w->fmt==5) matmul_i3(y,x,w->q4,w->s,S,w->I,w->O);+ else if(w->fmt==7) matmul_fp8(y,x,(const uint8_t*)w->q8,w->s,S,w->I,w->O);else matmul_i4(y,x,w->q4,w->s,S,w->I,w->O);}@@ -1138,26 +1254,135 @@ static int detect_group_size(int O, int I, int64_t ns){* diventava un int2 valido e il matmul leggeva oltre il buffer (O*I nibble a* 4/byte). Qui i byte del peso devono corrispondere a un layout noto e i byte* della scala alla cardinalita' attesa (O per-row, O*ng per-gruppo) β altrimenti- * si termina invece di sforare. Ritorna fmt (1/2/3/4/5) e scrive *gs. */+ * si termina invece di sforare. Ritorna fmt (1/2/3/4/5/6/7) e scrive *gs.+ * No container-level format stamp is consulted here: a self-describing+ * __metadata__ stamp that could resolve a genuine byte-collision below+ * (instead of refusing it unconditionally) is a follow-up proposal, not+ * present in this build -- every ambiguous shape this function can see+ * refuses, full stop. */static int qt_resolve_fmt(const char *name, int O, int I, int64_t nb, int64_t ns, int *gs){int64_t exp_i8=(int64_t)O*I, exp_i4=(int64_t)O*((I+1)/2), exp_i2=(int64_t)O*((I+3)/4);int64_t exp_i3=(int64_t)O*i3_rowbytes(I); /* int3-g64 (fmt=5): 24B per 64-input group *//* fmt=6 (E8/IQ3, #452): scales live inside the 98B super-blocks, so the .qs* convention is kept with a single-float tag β ns==4 is the discriminator- * (every other format carries at least O floats of real scales). */- if(ns==4 && nb==(int64_t)O*e8_rowbytes(I)){ *gs=0; return 6; }+ * (every other format carries at least O floats of real scales).+ *+ * SECOND DESIGN LANDMINE: e8_rowbytes(I) = ceil(I/256)*98 is the constant 98+ * for every I in (0,256], so this check's nb==O*e8_rowbytes(I) collapses to+ * nb==O*98 -- which is ALSO fp8-e4m3-b128's raw-byte weight count (O*I) at+ * the ONE value of I where I itself equals 98 (solving 98k==I for I in+ * ((k-1)*256,k*256] only admits k=1, I=98). At that same I=98 a handful of+ * fp8 scale-array shapes ALSO carry exactly ns==4 bytes, same as this+ * check's own tag, and each is a genuine collision candidate this block+ * must catch before the unconditional `return 6` below:+ * - a SINGLE-BLOCK fp8 tensor (O<=128, fp8_nblk(O)*fp8_nblk(I)==1) with+ * f32 block scales: ns==1*4==4.+ * - a FOUR-BLOCK fp8 tensor (fp8_nblk(O)*fp8_nblk(I)==4, e.g. O in+ * (384,512] at this I) with UE8M0 (1 byte/block) scales: ns==4*1==4 --+ * the SAME arithmetic collision the fmt=1-vs-fmt=7 landmine below has+ * with UE8M0, just against fmt=6's tag instead of fmt=1's per-row+ * count; see that landmine's own comment for the general shape.+ * - at O==1 specifically, fmt=1's own per-row tag (O*4) is also 4.+ * Refuse unconditionally when any of these coincide with the E8/IQ3 tag --+ * this build has no stamp to break the tie (see the function's own header+ * comment) -- rather than let dev's own unconditional `return 6` silently+ * misread an fp8-e4m3-b128 (or, at O==1, plain int8) tensor as E8/IQ3-+ * lattice-decoded garbage with no error. No real GLM tensor has these+ * shapes; the discipline exists for untrusted containers. */+ if(ns==4 && nb==(int64_t)O*e8_rowbytes(I)){+ int fp8_blk_f32_also = (nb==(int64_t)O*I) && (fp8_nblk(O)*fp8_nblk(I)==1);+ int fp8_blk_ue8m0_also = (nb==(int64_t)O*I) && (fp8_nblk(O)*fp8_nblk(I)==4);+ int i8_row_also = (nb==(int64_t)O*I) && (O==1); /* ns==O*4==4 iff O==1 */+ if(fp8_blk_f32_also || fp8_blk_ue8m0_also || i8_row_also){+ fprintf(stderr,"%s: [%d,%d] byte layout (nb=%lld ns=%lld) matches E8/IQ3 "+ "(fmt=6, 4-byte tag)%s%s%s; refusing rather than guessing (untrusted "+ "container, fmt=6 collision at I=98)\n",+ name,O,I,(long long)nb,(long long)ns,+ fp8_blk_f32_also ? " AND per-128x128-block FP8 f32 scales (fmt=7, single block)" : "",+ fp8_blk_ue8m0_also ? " AND per-128x128-block FP8 ue8m0 scales (fmt=7, 4 blocks, recognized-not-implemented)" : "",+ i8_row_also ? " AND plain int8 per-row (fmt=1, O=1)" : "");+ exit(1);+ }+ *gs=0; return 6;+ }/* Row formats take precedence: for tiny I the int3-g64 byte count can coincide with* a row layout (e.g. [O,48]: ceil(48/2)=24=1*24). For real tensor shapes the counts* are distinct, and the weight bytes β not the scale size β are the int3 tag, because* int3-g64 and grouped-int4-at-gs=64 carry the SAME scale cardinality O*ceil(I/64). */int fmt = (nb==exp_i8)?1 : (nb==exp_i4)?2 : (nb==exp_i2)?3 : (nb==exp_i3)?5 : 0;if(!fmt){- fprintf(stderr,"%s: quantized weight is %lld bytes β no int8/int4/int2/int3-g64 layout for [%d,%d], refusing (untrusted container)\n",+ fprintf(stderr,"%s: quantized weight is %lld bytes β no int8/int4/int2/int3-g64/fp8 layout for [%d,%d], refusing (untrusted container)\n",name,(long long)nb,O,I); exit(1); }*gs=0;if(fmt==2){ int g=detect_group_size(O,I,ns); if(g>0){ fmt=4; *gs=g; } }+ /* fmt=1 vs fmt=7 (native FP8-e4m3 passthrough): THE DESIGN LANDMINE. Weight+ * bytes are IDENTICAL (O*I raw bytes, both matched exp_i8 above) -- fmt=1+ * and fmt=7 can only be told apart by the SCALE array's geometry: fmt=1 is+ * per-row (ns==O*4 bytes); fmt=7 is per-128x128-BLOCK, ns==ceil(O/128)*+ * ceil(I/128)*4 bytes for THIS build's implemented f32 scale encoding. For+ * most real shapes these two byte counts are distinct and the match is+ * unambiguous. But for small O (<=128) and/or small I, ceil(O/128)*+ * ceil(I/128) can equal O exactly (e.g. O=1,I<=128 -> 1*1=1=O; O=2,I in+ * [129,256] -> 1*2=2=O; O=256,I in (16256,16384] -> 2*128=256=O) -- a+ * tensor whose scale array satisfies BOTH conventions at once. Guessing+ * either way risks silently reading a genuine per-row-int8 tensor as+ * block-scaled FP8 (or vice versa), which would corrupt every weight+ * without any error. Refuse instead (same "untrusted container"+ * discipline as the rest of this function).+ *+ * SCALE ENCODING IS A DECLARED PROPERTY, not a hardcoded constant: f32 (4+ * bytes/block, above) is what THIS build implements, but it is not the+ * only encoding real fp8-e4m3-b128 weight geometry ships with. DeepSeek-V4+ * ships the SAME weight layout (FP8 E4M3, 128x128 blocks) with UE8M0+ * scales instead -- one byte per block, a power-of-two exponent, dtype+ * F8_E8M0 (maintainer finding, colibri #524) -- so a fp8-e4m3-b128+ * container's scale sidecar can legitimately be ceil(O/128)*ceil(I/128)*1+ * bytes, not *4. That byte count is a REAL, distinct signature (never+ * equal to the f32 block-scale count above for any O,I>=1, since one is+ * exactly 4x the other and 4n==n only at n==0) -- recognized here rather+ * than silently misread as a truncated or corrupt f32 scale array. This+ * build does not implement UE8M0 decode: recognizing the signature and+ * refusing BY NAME (rather than falling through to the generic+ * "wrong byte count" refusal below, or worse, matching it against the+ * wrong candidate) is the whole point -- a future decoder lands into this+ * seam rather than a near-duplicate format. Checked for collision against+ * every OTHER format's ns arithmetic reachable from this nb==O*I branch+ * (fmt=1 and fmt=7-f32, both above): the byte counts are realistically+ * distinct (nblkO*nblkI is orders of magnitude smaller than O*4 for any+ * real GLM-sized matrix), but NOT categorically distinct -- the same+ * small-O regime that makes the fmt=1-vs-fmt=7-f32 collision above+ * possible also makes a fmt=1-vs-fmt=7-ue8m0 collision possible (e.g.+ * O=1, I in (384,512]: nblkO=1, nblkI=4, ue8m0 ns=4=O*4=fmt=1's own+ * per-row count) -- handled below by refusing whenever the ue8m0+ * signature matches, noting the row collision too when it also applies.+ * No real GLM tensor has these shapes; see also the SECOND DESIGN+ * LANDMINE above for the analogous ue8m0-vs-fmt=6 corner this same+ * signature can hit. */+ if(fmt==1){+ int64_t nblkO=fp8_nblk(O), nblkI=fp8_nblk(I);+ int64_t ns_row=(int64_t)O*4, ns_blk=nblkO*nblkI*4, ns_blk_ue8m0=nblkO*nblkI;+ int is_row=(ns==ns_row), is_blk=(ns==ns_blk), is_blk_ue8m0=(ns==ns_blk_ue8m0);+ if(is_row && is_blk){+ fprintf(stderr,"%s: [%d,%d] scale array is %lld bytes β matches BOTH per-row "+ "int8 (fmt=1) and per-128x128-block FP8 (fmt=7) scale geometry; refusing "+ "rather than guessing (untrusted container, THE DESIGN LANDMINE)\n",+ name,O,I,(long long)ns); exit(1);+ }+ if(is_blk_ue8m0){+ fprintf(stderr,"%s: [%d,%d] fp8-e4m3-b128 with ue8m0 scales recognized but not "+ "implemented; only f32 block scales are supported in this build (nb=%lld "+ "bytes matches raw e4m3 weight bytes, ns=%lld bytes matches %lld blocks x "+ "1 byte/block)%s\n",+ name,O,I,(long long)nb,(long long)ns,(long long)(nblkO*nblkI),+ is_row ? " -- scale array ALSO matches per-row int8 (fmt=1); refusing either way (untrusted container)" : "");+ exit(1);+ }+ if(is_blk && !is_row) fmt=7;+ }int64_t exp_scale = (fmt==4)? (int64_t)O*((I+*gs-1)/(*gs))- : (fmt==5)? (int64_t)O*i3_groups(I) : (int64_t)O; /* in FLOAT */+ : (fmt==5)? (int64_t)O*i3_groups(I)+ : (fmt==7)? fp8_nblk(O)*fp8_nblk(I)+ : (int64_t)O; /* in FLOAT */if(ns != exp_scale*4){fprintf(stderr,"%s: scale array is %lld bytes β expected %lld for [%d,%d] fmt=%d, refusing (untrusted container)\n",name,(long long)ns,(long long)(exp_scale*4),O,I,fmt); exit(1); }@@ -1200,16 +1425,25 @@ static void qt_from_disk(Model *m, const char *name, int O, int I, int bits, intelse if(fmt==6){ /* E8/IQ3: everything in-block, .qs is the 4-byte tag */if(t->fmt!=6||!t->q4){ t->fmt=6; t->O=O; t->I=I; t->gs=0; t->q4=qalloc(nb); t->s=qsalloc(1); }st_read_raw(&m->S,name,t->q4,drop); }+ else if(fmt==7){ int64_t nblk=fp8_nblk(O)*fp8_nblk(I); /* fp8-e4m3: raw bytes (q8, like fmt=1)+ * + one f32 scale per 128x128 block. BOTH weights and scale via qalloc, not falloc,+ * from day one -- the GPU-visibility lesson fmt=4 and fmt=5 each had to learn+ * separately (see review round 1 audit history in the report). */+ if(t->fmt!=7||!t->q8){ t->fmt=7; t->O=O; t->I=I; t->gs=0; t->q8=qalloc(nb); t->s=(float*)qalloc((size_t)nblk*sizeof(float)); }+ st_read_raw(&m->S,name,t->q8,drop); }else { if(t->fmt!=fmt||!t->q4){ t->fmt=fmt; t->O=O; t->I=I; t->gs=0; t->q4=qalloc(nb); t->s=qsalloc(O); } st_read_raw(&m->S,name,t->q4,drop); }/* cap MUST match the scale cardinality qt_resolve_fmt already validated and* the falloc above actually reserved, per format: grouped-int4 (fmt=4) keeps- * O*ceil(I/gs) scales and int3-g64 (fmt=5) keeps O*i3_groups(I); everything- * else is per-row O. Using the per-row bound for a grouped format would- * reject a legitimate container (fmt=5 regressed exactly that way). */+ * O*ceil(I/gs) scales, int3-g64 (fmt=5) keeps O*i3_groups(I), E8/IQ3 (fmt=6)+ * keeps the 1-float tag, and fp8-e4m3 (fmt=7) keeps ceil(O/128)*ceil(I/128)+ * block scales; everything else is per-row O. Using the per-row bound for a+ * grouped/blocked format would reject a legitimate container (fmt=5 regressed+ * exactly that way). */st_read_f32_cap(&m->S,sn,t->s,fmt==4 ? (int64_t)O*((I+gs-1)/gs) :fmt==5 ? (int64_t)O*i3_groups(I) :- fmt==6 ? (int64_t)1 : (int64_t)O, drop);+ fmt==6 ? (int64_t)1 :+ fmt==7 ? fp8_nblk(O)*fp8_nblk(I) : (int64_t)O, drop);} else {if(!t->qf && !t->q8 && !t->q4) qt_alloc(t,O,I,bits);if(t->fmt==0) st_read_f32_cap(&m->S,name,t->qf,(int64_t)O*I,drop);@@ -2598,10 +2832,19 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p* (step_decode_batch) passes per-row kvs[]/positions[] with pos_base=0, so the kernel* would rope every row at position 0 and attend over a 1-token window of the wrong* cache -> greedy decode hits EOS at token 2 (mux answers truncated to 1 token).- * Ragged rows take the CPU absorb path below, which reads kvs[s]/positions[s]. */+ * Ragged rows take the CPU absorb path below, which reads kvs[s]/positions[s].+ * fmt!=7 guards (q_a/q_b/kv_a/o): coli_metal_attn_decode's WP_() macro and the+ * mm_gemv shader it dispatches through (bind_gemv) have no fmt=7 case in this+ * build -- an fp8-passthrough tensor reaching here would be silently misread as+ * f32 by the shader's default branch. Fail closed to the CPU path below instead+ * (matmul_qt_ex there dispatches fmt=7 correctly via matmul_fp8). kv_b is already+ * pinned to fmt==2 above for an unrelated reason (its absorb kernel is int4-only);+ * these four checks are the same discipline extended to the tensors that flow+ * through the generic per-fmt shader. */if(g_metal_enabled && !kvs && S<=4 && (g_absorb==1||(g_absorb<0&&S<=4)) && m->kv_start[layer]==0&& D==6144 && H==64 && c->q_lora==2048 && c->kv_lora==512 && c->qk_nope==192- && c->qk_rope==64 && vh==256 && l->kv_b.fmt==2){+ && c->qk_rope==64 && vh==256 && l->kv_b.fmt==2+ && l->q_a.fmt!=7 && l->q_b.fmt!=7 && l->kv_a.fmt!=7 && l->o.fmt!=7){int sel_active = m->has_dsa && layer<c->n_layers && c->idx_type[layer] && (pos_base+S) > c->index_topk;if(!sel_active){if(m->has_dsa && layer<c->n_layers && c->idx_type[layer]){ /* index keys for future selection */@@ -4806,12 +5049,19 @@ static void layer_forward_rows(Model *m, Layer *l, int li, float *x, int S, int* in un solo submit GPU; la CPU legge il routing e fa solo resolve/disk/expert-CB.* Fallback: qualsiasi condizione mancante -> percorso CPU intero qui sotto.* !kvs: ragged mux rows (per-row KV/position) are not expressible in this kernel's- * single Lc/Rc + pos_base contract β see the matching guard in attention_rows. */+ * single Lc/Rc + pos_base contract β see the matching guard in attention_rows.+ * fmt!=7 guards (q_a/q_b/kv_a/o/sh_gate/sh_up/sh_down): same fail-closed discipline+ * as attention_rows, extended to the shared-expert MLP this fused kernel also+ * covers -- none of these seven bind_gemv-routed tensors has fmt=7 support in this+ * build's mm_gemv shader, so any of them carrying fmt=7 must skip the whole fused+ * path (CPU below dispatches fmt=7 correctly via matmul_qt_ex/matmul_fp8). */if(g_metal_enabled && !kvs && S<=4 && li<c->n_layers && l->sparse&& (g_absorb==1||(g_absorb<0&&S<=4)) && m->kv_start[li]==0&& D==6144 && c->n_heads==64 && c->q_lora==2048 && c->kv_lora==512&& c->qk_nope==192 && c->qk_rope==64 && c->v_head==256 && l->kv_b.fmt==2- && c->n_experts==256 && c->topk==8 && c->n_shared==1 && c->moe_inter==2048){+ && c->n_experts==256 && c->topk==8 && c->n_shared==1 && c->moe_inter==2048+ && l->q_a.fmt!=7 && l->q_b.fmt!=7 && l->kv_a.fmt!=7 && l->o.fmt!=7+ && l->sh_gate.fmt!=7 && l->sh_up.fmt!=7 && l->sh_down.fmt!=7){int sel_active = m->has_dsa && c->idx_type[li] && (pos_base+S) > c->index_topk;if(!sel_active){static float *linrm,*lnrm,*lsh,*lw; static int *lidx,*lkeff;diff --git a/c/quant.h b/c/quant.hindex d244d1b1..7dbf3936 100644--- a/c/quant.h+++ b/c/quant.h@@ -402,6 +402,111 @@ static void matmul_i3(float *y, const float *x, const uint8_t *q3, const float *}}+/* ---- fmt=7: native FP8-e4m3 passthrough (Z.ai GLM-5.2-FP8 read path) ------+ * PUBLIC ordinal, assigned by the maintainer on #524. This format was minted+ * fmt=6 during original development, then re-tagged fmt=100 (PRIVATE ORDINAL+ * BLOCK, see colibri.c's QT struct comment) after #465 (E8/IQ3, above) claimed+ * ordinal 6 upstream out from under it; fmt=7 is the number it graduated to at+ * merge, per the private-block convention's own "find-and-replace, zero+ * on-disk impact" promise. See qt_resolve_fmt in colibri.c ("THE DESIGN+ * LANDMINE") for the disambiguation this needs against BOTH fmt=1 (int8) and+ * fmt=6 (E8/IQ3).+ *+ * fmt=7 weight bytes are O*I raw e4m3 bytes -- byte-identical to int8 (fmt=1).+ * What makes this a DIFFERENT format is the scale: one f32 per 128x128 BLOCK of+ * the [O,I] weight matrix (shape [ceil(O/128),ceil(I/128)]), not one f32 per+ * output row. Dequant is w[o,i] = e4m3_decode(byte) * scale[o/128, i/128]+ * (MULTIPLY, not divide) -- mirrors tools/convert_fp8_to_int4.py's dequant()+ * exactly, which is the authoritative reference for how Z.ai's checkpoints read+ * on the source side. This f32 encoding is a declared PROPERTY of the format,+ * not the only one it can carry -- DeepSeek-V4 ships this identical weight+ * geometry with UE8M0 (1-byte, power-of-two) block scales instead; qt_resolve_fmt+ * recognizes that byte signature and refuses it BY NAME rather than+ * misreading it as f32 (UE8M0 decode itself is not implemented in this build).+ *+ * 256-entry decode LUT, cross-checked byte-for-byte against PyTorch's+ * torch.float8_e4m3fn (the exact dtype safetensors reports for these shards):+ * sign(1) exp(4,bias=7) mant(3), subnormal at exp==0, and the OCP E4M3-FN+ * convention that exp==0xF is NOT reserved for infinity -- only mant==0x7 at+ * exp==0xF is NaN (max finite is exp=0xF,mant=0x6 -> 448). A static compile-time+ * table (not a lazily-initialized one) is used deliberately: matmul_fp8 below+ * runs inside #pragma omp parallel for, and a lazy-init global would race under+ * concurrent first use.+ *+ * NaN POLICY (decided, documented per the build spec): a NaN byte (0x7F/0xFF)+ * decodes to a real IEEE NaN and is left to propagate through the dot product,+ * exactly like any other source of a NaN weight (fmt=0/f32 tensors are never+ * scrubbed either). The engine already has a dedicated, TESTED safety net for+ * NaN reaching the sampler (argmax_v/dist_build, see tests/test_logit_nan.c:+ * "degrade + diagnose, never silently corrupt") -- fmt=7 relies on that existing+ * downstream net rather than adding a second, redundant weight-level scrub. */+static const float E4M3_LUT[256] = {+ 0x0.0p+0f,0x1.0000000000000p-9f,0x1.0000000000000p-8f,0x1.8000000000000p-8f,0x1.0000000000000p-7f,0x1.4000000000000p-7f,0x1.8000000000000p-7f,0x1.c000000000000p-7f,+ 0x1.0000000000000p-6f,0x1.2000000000000p-6f,0x1.4000000000000p-6f,0x1.6000000000000p-6f,0x1.8000000000000p-6f,0x1.a000000000000p-6f,0x1.c000000000000p-6f,0x1.e000000000000p-6f,+ 0x1.0000000000000p-5f,0x1.2000000000000p-5f,0x1.4000000000000p-5f,0x1.6000000000000p-5f,0x1.8000000000000p-5f,0x1.a000000000000p-5f,0x1.c000000000000p-5f,0x1.e000000000000p-5f,+ 0x1.0000000000000p-4f,0x1.2000000000000p-4f,0x1.4000000000000p-4f,0x1.6000000000000p-4f,0x1.8000000000000p-4f,0x1.a000000000000p-4f,0x1.c000000000000p-4f,0x1.e000000000000p-4f,+ 0x1.0000000000000p-3f,0x1.2000000000000p-3f,0x1.4000000000000p-3f,0x1.6000000000000p-3f,0x1.8000000000000p-3f,0x1.a000000000000p-3f,0x1.c000000000000p-3f,0x1.e000000000000p-3f,+ 0x1.0000000000000p-2f,0x1.2000000000000p-2f,0x1.4000000000000p-2f,0x1.6000000000000p-2f,0x1.8000000000000p-2f,0x1.a000000000000p-2f,0x1.c000000000000p-2f,0x1.e000000000000p-2f,+ 0x1.0000000000000p-1f,0x1.2000000000000p-1f,0x1.4000000000000p-1f,0x1.6000000000000p-1f,0x1.8000000000000p-1f,0x1.a000000000000p-1f,0x1.c000000000000p-1f,0x1.e000000000000p-1f,+ 0x1.0000000000000p+0f,0x1.2000000000000p+0f,0x1.4000000000000p+0f,0x1.6000000000000p+0f,0x1.8000000000000p+0f,0x1.a000000000000p+0f,0x1.c000000000000p+0f,0x1.e000000000000p+0f,+ 0x1.0000000000000p+1f,0x1.2000000000000p+1f,0x1.4000000000000p+1f,0x1.6000000000000p+1f,0x1.8000000000000p+1f,0x1.a000000000000p+1f,0x1.c000000000000p+1f,0x1.e000000000000p+1f,+ 0x1.0000000000000p+2f,0x1.2000000000000p+2f,0x1.4000000000000p+2f,0x1.6000000000000p+2f,0x1.8000000000000p+2f,0x1.a000000000000p+2f,0x1.c000000000000p+2f,0x1.e000000000000p+2f,+ 0x1.0000000000000p+3f,0x1.2000000000000p+3f,0x1.4000000000000p+3f,0x1.6000000000000p+3f,0x1.8000000000000p+3f,0x1.a000000000000p+3f,0x1.c000000000000p+3f,0x1.e000000000000p+3f,+ 0x1.0000000000000p+4f,0x1.2000000000000p+4f,0x1.4000000000000p+4f,0x1.6000000000000p+4f,0x1.8000000000000p+4f,0x1.a000000000000p+4f,0x1.c000000000000p+4f,0x1.e000000000000p+4f,+ 0x1.0000000000000p+5f,0x1.2000000000000p+5f,0x1.4000000000000p+5f,0x1.6000000000000p+5f,0x1.8000000000000p+5f,0x1.a000000000000p+5f,0x1.c000000000000p+5f,0x1.e000000000000p+5f,+ 0x1.0000000000000p+6f,0x1.2000000000000p+6f,0x1.4000000000000p+6f,0x1.6000000000000p+6f,0x1.8000000000000p+6f,0x1.a000000000000p+6f,0x1.c000000000000p+6f,0x1.e000000000000p+6f,+ 0x1.0000000000000p+7f,0x1.2000000000000p+7f,0x1.4000000000000p+7f,0x1.6000000000000p+7f,0x1.8000000000000p+7f,0x1.a000000000000p+7f,0x1.c000000000000p+7f,0x1.e000000000000p+7f,+ 0x1.0000000000000p+8f,0x1.2000000000000p+8f,0x1.4000000000000p+8f,0x1.6000000000000p+8f,0x1.8000000000000p+8f,0x1.a000000000000p+8f,0x1.c000000000000p+8f, NAN,+ -0x0.0p+0f,-0x1.0000000000000p-9f,-0x1.0000000000000p-8f,-0x1.8000000000000p-8f,-0x1.0000000000000p-7f,-0x1.4000000000000p-7f,-0x1.8000000000000p-7f,-0x1.c000000000000p-7f,+ -0x1.0000000000000p-6f,-0x1.2000000000000p-6f,-0x1.4000000000000p-6f,-0x1.6000000000000p-6f,-0x1.8000000000000p-6f,-0x1.a000000000000p-6f,-0x1.c000000000000p-6f,-0x1.e000000000000p-6f,+ -0x1.0000000000000p-5f,-0x1.2000000000000p-5f,-0x1.4000000000000p-5f,-0x1.6000000000000p-5f,-0x1.8000000000000p-5f,-0x1.a000000000000p-5f,-0x1.c000000000000p-5f,-0x1.e000000000000p-5f,+ -0x1.0000000000000p-4f,-0x1.2000000000000p-4f,-0x1.4000000000000p-4f,-0x1.6000000000000p-4f,-0x1.8000000000000p-4f,-0x1.a000000000000p-4f,-0x1.c000000000000p-4f,-0x1.e000000000000p-4f,+ -0x1.0000000000000p-3f,-0x1.2000000000000p-3f,-0x1.4000000000000p-3f,-0x1.6000000000000p-3f,-0x1.8000000000000p-3f,-0x1.a000000000000p-3f,-0x1.c000000000000p-3f,-0x1.e000000000000p-3f,+ -0x1.0000000000000p-2f,-0x1.2000000000000p-2f,-0x1.4000000000000p-2f,-0x1.6000000000000p-2f,-0x1.8000000000000p-2f,-0x1.a000000000000p-2f,-0x1.c000000000000p-2f,-0x1.e000000000000p-2f,+ -0x1.0000000000000p-1f,-0x1.2000000000000p-1f,-0x1.4000000000000p-1f,-0x1.6000000000000p-1f,-0x1.8000000000000p-1f,-0x1.a000000000000p-1f,-0x1.c000000000000p-1f,-0x1.e000000000000p-1f,+ -0x1.0000000000000p+0f,-0x1.2000000000000p+0f,-0x1.4000000000000p+0f,-0x1.6000000000000p+0f,-0x1.8000000000000p+0f,-0x1.a000000000000p+0f,-0x1.c000000000000p+0f,-0x1.e000000000000p+0f,+ -0x1.0000000000000p+1f,-0x1.2000000000000p+1f,-0x1.4000000000000p+1f,-0x1.6000000000000p+1f,-0x1.8000000000000p+1f,-0x1.a000000000000p+1f,-0x1.c000000000000p+1f,-0x1.e000000000000p+1f,+ -0x1.0000000000000p+2f,-0x1.2000000000000p+2f,-0x1.4000000000000p+2f,-0x1.6000000000000p+2f,-0x1.8000000000000p+2f,-0x1.a000000000000p+2f,-0x1.c000000000000p+2f,-0x1.e000000000000p+2f,+ -0x1.0000000000000p+3f,-0x1.2000000000000p+3f,-0x1.4000000000000p+3f,-0x1.6000000000000p+3f,-0x1.8000000000000p+3f,-0x1.a000000000000p+3f,-0x1.c000000000000p+3f,-0x1.e000000000000p+3f,+ -0x1.0000000000000p+4f,-0x1.2000000000000p+4f,-0x1.4000000000000p+4f,-0x1.6000000000000p+4f,-0x1.8000000000000p+4f,-0x1.a000000000000p+4f,-0x1.c000000000000p+4f,-0x1.e000000000000p+4f,+ -0x1.0000000000000p+5f,-0x1.2000000000000p+5f,-0x1.4000000000000p+5f,-0x1.6000000000000p+5f,-0x1.8000000000000p+5f,-0x1.a000000000000p+5f,-0x1.c000000000000p+5f,-0x1.e000000000000p+5f,+ -0x1.0000000000000p+6f,-0x1.2000000000000p+6f,-0x1.4000000000000p+6f,-0x1.6000000000000p+6f,-0x1.8000000000000p+6f,-0x1.a000000000000p+6f,-0x1.c000000000000p+6f,-0x1.e000000000000p+6f,+ -0x1.0000000000000p+7f,-0x1.2000000000000p+7f,-0x1.4000000000000p+7f,-0x1.6000000000000p+7f,-0x1.8000000000000p+7f,-0x1.a000000000000p+7f,-0x1.c000000000000p+7f,-0x1.e000000000000p+7f,+ -0x1.0000000000000p+8f,-0x1.2000000000000p+8f,-0x1.4000000000000p+8f,-0x1.6000000000000p+8f,-0x1.8000000000000p+8f,-0x1.a000000000000p+8f,-0x1.c000000000000p+8f, NAN,+};+static inline float e4m3_decode(uint8_t b){ return E4M3_LUT[b]; }++#define FP8_BLOCK 128+static inline int64_t fp8_nblk(int n){ return ((int64_t)n + FP8_BLOCK - 1) / FP8_BLOCK; }++/* y[S,O] = x[S,I] @ W^T, W raw e4m3 bytes (byte-identical layout to fmt=1) ++ * per-128x128-BLOCK f32 scale [ceil(O/128),ceil(I/128)]. Scalar reference path+ * (no SIMD in v1 -- BW-bound like the Metal kernel, and this format's hot path+ * is the GPU one; a vectorized CPU kernel is future work if measured needed).+ * Mirrors matmul_i3's double-accumulate-across-groups / float-within-group+ * convention so cross-block cancellation doesn't cost precision unfairly. */+static void matmul_fp8(float *y, const float *x, const uint8_t *q8, const float *bscale,+ int S, int I, int O){+ int64_t nblkI = fp8_nblk(I);+ #pragma omp parallel for schedule(static)+ for(int o=0;o<O;o++){+ const uint8_t *w = q8 + (int64_t)o*I;+ int64_t blkO = o / FP8_BLOCK;+ const float *scl = bscale + blkO*nblkI;+ for(int s=0;s<S;s++){+ const float *xs = x + (int64_t)s*I;+ double a=0;+ for(int64_t bi=0; bi*FP8_BLOCK<I; bi++){+ int base=(int)(bi*FP8_BLOCK); int blen=FP8_BLOCK; if(base+blen>I) blen=I-base;+ float sc=scl[bi]; float acc=0;+ for(int i=base;i<base+blen;i++) acc += e4m3_decode(w[i])*xs[i];+ a += (double)acc*sc;+ }+ y[(int64_t)s*O+o]=(float)a;+ }+ }+}+/* ---- IDOT: integer dot kernels (int8-quantized activations) --------------- */#if defined(__AVX512VNNI__) && defined(__AVX512BW__)#define IDOT_KERNEL "avx512-vnni"diff --git a/c/tests/test_fp8_load.c b/c/tests/test_fp8_load.cnew file mode 100644index 00000000..f04cca02--- /dev/null+++ b/c/tests/test_fp8_load.c@@ -0,0 +1,369 @@+/* fmt=7 (native FP8-e4m3 passthrough) loader-seam tests.+ *+ * fmt=7, PUBLIC ordinal assigned by the maintainer on #524: this format was+ * minted fmt=6 during original development of this branch, before dev's own+ * #465 (E8/IQ3) claimed that ordinal upstream and merged it into dev as a+ * REAL fmt=6 (see quant.h's E8 constants and e8_ helper functions, and+ * qt_resolve_fmt's ns==4-tag early check); re-tagged fmt=100 (PRIVATE ORDINAL+ * BLOCK, see colibri.c's QT struct comment) from that point forward -- there+ * was never a build in this branch's history where this format was reachable+ * as fmt=6 -- and graduated to fmt=7 at merge.+ *+ * Part A: qt_resolve_fmt disambiguation suite -- THE DESIGN LANDMINE. fmt=7+ * weight bytes are byte-identical to fmt=1 (int8): both are O*I raw bytes.+ * The two are told apart ONLY by the scale array's byte count (per-row O*4 for+ * fmt=1, per-128x128-block ceil(O/128)*ceil(I/128)*4 for fmt=7, THIS build's+ * implemented f32 scale encoding). For some shapes those two counts coincide+ * exactly -- qt_resolve_fmt must REFUSE (exit(1)) rather than guess. Refusal+ * is tested via fork()+waitpid(), mirroring tests/test_st_pread.c's+ * established convention for exit(1)-terminated paths.+ *+ * Part A2: fmt=6 (E8/IQ3, upstream #465) vs fmt=7 collision at [O<=128 or+ * O in a 128-block-count range, I=98] -- SECOND DESIGN LANDMINE.+ *+ * Part A3: fmt=7's scale ENCODING is a declared property, not a hardcoded+ * constant -- f32 (Part A/A2 above) is what this build implements. A UE8M0+ * (1 byte/block) encoding is a REAL, distinct byte signature (the DeepSeek-V4+ * checkpoint format for this identical weight geometry) this build recognizes+ * and refuses BY NAME rather than misreading.+ *+ * Part B: qt_from_disk loader-seam -- writes a real single-shard .safetensors+ * file containing an fmt=7 tensor (U8 weight + per-block F32 .qs) next to an+ * int8 control tensor of a DIFFERENT, non-colliding shape, loads both through+ * qt_from_disk, and checks the byte-count/.qs-size inference picks fmt=7 vs+ * fmt=1 correctly and the loaded weights dequantize identically to a reference.+ * Mirrors tests/test_int3_load.c's structure for fmt=5.+ *+ * Part C: qt_bytes()/qt_scale_bytes() byte-accounting for fmt=7.+ *+ * This build has NO container metadata stamp (see qt_resolve_fmt's own header+ * comment) -- every ambiguous/unimplemented-encoding shape below refuses+ * unconditionally; stamp-resolves-ambiguity behavior is a separate, follow-up+ * PR (registry + metadata stamp), not exercised here. */+#define main coli_glm_main_unused+#include "../colibri.c"+#undef main++#include <stdio.h>+#include <string.h>+#include <sys/stat.h>+#ifndef _WIN32+#include <unistd.h>+#include <sys/wait.h>+#endif++static int fails = 0;+#define CHECK(c) do{ if(!(c)){ printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #c); fails++; } }while(0)++static uint64_t rng = 0xFEEDFACE0DDBA11ull;+static float rndf(void){ rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17;+ return ((int64_t)(rng & 0xFFFFF) - 0x80000) / (float)0x80000; }+static uint8_t rndbyte_nonan(void){+ for(;;){ rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17;+ uint8_t b = (uint8_t)(rng & 0xFF);+ if(b != 0x7F && b != 0xFF) return b; }+}++/* ---- Part A: qt_resolve_fmt disambiguation (in-process for non-refusing+ * cases, fork+waitpid for refusing ones -- qt_resolve_fmt exit(1)s in place,+ * it does not return an error code). ---- */++static int expect_fmt(int O, int I, int64_t nb, int64_t ns, int expect_fmt_val, const char *tag){+ int gs=0;+ int fmt = qt_resolve_fmt(tag, O, I, nb, ns, &gs);+ if(fmt != expect_fmt_val){+ printf("FAIL %s: got fmt=%d, expected fmt=%d (O=%d I=%d nb=%lld ns=%lld)\n",+ tag, fmt, expect_fmt_val, O, I, (long long)nb, (long long)ns);+ return 0;+ }+ return 1;+}++static int expect_refuse(int O, int I, int64_t nb, int64_t ns, const char *tag){+#ifndef _WIN32+ int pipefd[2]; if(pipe(pipefd)!=0) return 0;+ pid_t pid = fork();+ if(pid < 0) return 0;+ if(pid == 0){+ dup2(pipefd[1],2); close(pipefd[0]); close(pipefd[1]);+ int gs=0;+ qt_resolve_fmt(tag, O, I, nb, ns, &gs); /* must exit(1) inside; must NOT return */+ _exit(42); /* reaching here is the bug */+ }+ close(pipefd[1]);+ char err[1024]={0}; ssize_t n=read(pipefd[0],err,sizeof(err)-1); (void)n;+ close(pipefd[0]);+ int status=0; waitpid(pid,&status,0);+ int ok = WIFEXITED(status) && WEXITSTATUS(status)==1;+ if(!ok){+ printf("FAIL %s: expected exit(1) refusal, got status=%d, stderr=%.200s\n", tag, status, err);+ return 0;+ }+ if(!strstr(err,"refus")){+ printf("FAIL %s: exited(1) but message lacked a refusal explanation: %.200s\n", tag, err);+ return 0;+ }+ return 1;+#else+ /* fork/pipe/waitpid are POSIX -- mirrors tests/test_st_pread.c's own+ * Windows arm: skip the exit(1) subprocess check, print an explicit+ * (never silent) skip line per case, count it as passing rather than+ * failing. Every non-refusing case in this file (expect_fmt and its+ * callers) still runs and asserts for real on Windows. */+ printf("skipped on Windows (no fork): %s\n", tag);+ (void)O; (void)I; (void)nb; (void)ns;+ return 1;+#endif+}++static void test_disambiguation(void){+ /* --- non-degenerate golden paths (unambiguous either way) --- */+ CHECK(expect_fmt(4096,4096,(int64_t)4096*4096,4096*4,1,"plain int8 4096x4096"));+ /* spec's own worked example: [2048,6144] expert -> block scale [16,48] */+ CHECK(expect_fmt(2048,6144,(int64_t)2048*6144,16LL*48*4,7,"fp8 2048x6144 (spec example)"));+ CHECK(expect_fmt(384,6144,(int64_t)384*6144,3LL*48*4,7,"fp8 384x6144 non-square block grid"));++ /* --- degenerate shapes: O<=128 makes nblkO==1, so ns_blk==nblkI*4 can+ * equal ns_row==O*4 whenever nblkI==O. Exhaustive-in-spirit sweep of the+ * boundary the design landmine describes. --- */+ CHECK(expect_refuse(1,1, 1, 4, "degenerate O=1 I=1 (nblkI=1=O)"));+ CHECK(expect_refuse(1,128, 128, 4, "degenerate O=1 I=128 (nblkI=1=O, I at block edge)"));+ CHECK(expect_refuse(2,256, 2LL*256, 8, "degenerate O=2 I=256 (nblkI=2=O)"));+ CHECK(expect_refuse(6,768, 6LL*768, 24, "degenerate O=6 I=768 (nblkI=6=O)"));+ CHECK(expect_refuse(128,16384,128LL*16384, 512, "degenerate O=128 I=16384 (nblkO=1,nblkI=128=O)"));+ /* O>128 degenerate case: nblkO=2, need nblkI=O/2 -- O=256,I=16384 -> nblkI=128, 2*128=256=O */+ CHECK(expect_refuse(256,16384,256LL*16384, 1024, "degenerate O=256 I=16384 (nblkO=2,nblkI=128, product=O)"));+ /* k=3: the ambiguity isn't a one-off O=256 coincidence -- it's structural for ANY+ * O that's a multiple of 128 (nblkO=k), since nblkI==128 (I in (16256,16384]) always+ * makes nblkO*nblkI == k*128 == O. One more multiple (O=384=3*128) confirms the+ * condition generalizes past the k=2 worked example, not just a re-derivation. */+ CHECK(expect_refuse(384,16384,384LL*16384, 1536, "degenerate O=384 I=16384 (nblkO=3,nblkI=128, k=3, product=O)"));++ /* --- boundary-ADJACENT non-degenerate cases: one step past each+ * degenerate case above, both interpretations now legitimately resolve. --- */+ CHECK(expect_fmt(1,129, 129, 4, 1, "adjacent O=1 I=129 as fmt=1 (ns=row)"));+ CHECK(expect_fmt(1,129, 129, 8, 7, "adjacent O=1 I=129 as fmt=7 (ns=block, nblkI=2)"));+ CHECK(expect_fmt(2,257, 2LL*257, 8, 1, "adjacent O=2 I=257 as fmt=1 (ns=row)"));+ CHECK(expect_fmt(2,257, 2LL*257, 12, 7, "adjacent O=2 I=257 as fmt=7 (ns=block, nblkI=3)"));++ /* --- neither interpretation matches: garbage .qs size, must still refuse+ * (the pre-existing generic-mismatch path, exercised through the fmt=7-aware+ * function to confirm the new code didn't disturb it). --- */+ CHECK(expect_refuse(10,10, 100, 999, "garbage ns matches neither row nor block layout"));+}++/* ---- Part A2: fmt=6 (E8/IQ3, upstream #465, merged into dev) vs fmt=7+ * (this branch's fp8-e4m3-b128) collision -- SECOND DESIGN LANDMINE, see the+ * derivation in qt_resolve_fmt's own comment. e8_rowbytes(I) is the constant+ * 98 for every I in (0,256], so dev's fmt=6 tag check (ns==4 &&+ * nb==O*e8_rowbytes(I)) collapses to nb==O*98 -- which coincides with+ * fp8-e4m3-b128's raw weight bytes (O*I) at the ONE value I==98, where a+ * SINGLE-BLOCK (O<=128) fp8 tensor with f32 block scales, OR a FOUR-BLOCK+ * fp8 tensor with ue8m0 (1 byte/block) scales, ALSO carries exactly ns==4,+ * same as the E8 tag. An unstamped [I=98] fp8-e4m3-b128 tensor at either of+ * those shapes is therefore byte-for-byte indistinguishable from a genuine+ * fmt=6 tensor: nb AND ns both coincide, not just ns (contrast the fmt=1/+ * fmt=7 collision in Part A, where only ns ever coincides). O==1 stacks a+ * THIRD candidate: fmt=1's per-row ns (O*4) is also 4 there. This build has+ * no stamp to resolve any of these -- every one refuses. */+static void test_fmt6_fp8_collision(void){+ /* O=64, I=98: nblkO=nblkI=1 (fmt=7, single block, f32 scales) -> ns=4;+ * e8_blocks(98)=1 -> nb=O*98=6272 for BOTH interpretations, and fmt=6's+ * .qs tag is always exactly one f32 -> ns=4 too. Must refuse. */+ int64_t nb64=(int64_t)64*98;+ CHECK(expect_refuse(64,98, nb64, 4, "fmt=6/fmt=7(f32) collision O=64 I=98"));+ /* boundary O=128 variant: still nblkO=1 for fmt=7 (128<=128), same collision. */+ int64_t nb128=(int64_t)128*98;+ CHECK(expect_refuse(128,98, nb128, 4, "fmt=6/fmt=7(f32) collision O=128 I=98"));++ /* O=1, I=98: a THIRD candidate stacks on (fmt=1 plain int8 per-row, ns==O*4==4+ * too) -- a genuine three-way ambiguity. Must refuse. */+ int64_t nb1=(int64_t)1*98;+ CHECK(expect_refuse(1,98, nb1, 4, "fmt=1/fmt=6/fmt=7(f32) THREE-way collision O=1 I=98"));++ /* O in (384,512], I=98: nblkO=4, nblkI=1, product=4 -- a fmt=7 tensor with+ * UE8M0 (1 byte/block) scales also lands at ns==4*1==4 here, the SAME tag+ * fmt=6 uses. Must refuse (and, since this build doesn't implement ue8m0+ * decode at all, would refuse on that basis alone even without the fmt=6+ * collision -- this case exercises the OUTER fmt=6 guard specifically). */+ int64_t nb400=(int64_t)400*98;+ CHECK(expect_refuse(400,98, nb400, 4, "fmt=6/fmt=7(ue8m0, 4-block) collision O=400 I=98"));++ /* regression guard: a GENUINE (non-colliding) fmt=6 fixture -- I!=98, so+ * e8_rowbytes(I)==98 does NOT equal O*I -- must keep resolving to fmt=6.+ * Mirrors test_e8_kernel.c's own O=24,I=512 shape and a small+ * single-super-block shape (I=256, inside the (0,256] range where+ * e8_rowbytes(I)==98 but I!=98, so still non-colliding). */+ CHECK(expect_fmt(24,512, (int64_t)24*e8_rowbytes(512), 4, 6,+ "genuine fmt=6 (non-colliding) O=24 I=512 -> still resolves to fmt=6"));+ CHECK(expect_fmt(64,256, (int64_t)64*e8_rowbytes(256), 4, 6,+ "genuine fmt=6 (non-colliding) O=64 I=256 (I!=98, no collision) -> fmt=6"));+}++/* ---- Part A3: fmt=7's scale ENCODING is a declared property -- UE8M0+ * recognized, refused by name (not implemented in this build). ---- */+static void test_ue8m0_scale_refusal(void){+ /* [2048,6144] (spec example shape, same as Part A's fmt=7/f32 golden+ * path): nblkO=16, nblkI=48, product=768 blocks. A UE8M0 sidecar is+ * exactly 1 byte/block -> ns=768, distinct from BOTH fmt=1's per-row+ * count (O*4=8192) and this build's f32 block-scale count (768*4=3072).+ * Clean, unambiguous UE8M0 signature -- must name-refuse, not silently+ * treat it as a truncated/corrupt f32 array or match it to fmt=1. */+ int64_t nb=(int64_t)2048*6144;+ CHECK(expect_refuse(2048,6144, nb, 768,+ "fp8-e4m3-b128 with ue8m0 scales (spec-shaped, non-degenerate) -> recognized, refused by name"));++ /* O=1, I=400: nblkO=1, nblkI=ceil(400/128)=4, product=4 -- a UE8M0 sidecar+ * here is ns=4*1=4, which ALSO equals fmt=1's per-row count (O*4=4): the+ * same small-O regime that produces the f32-vs-fmt=1 collision in Part A+ * produces a ue8m0-vs-fmt=1 collision too. Must still refuse (combined+ * message), not silently pick fmt=1. */+ int64_t nb1=(int64_t)1*400;+ CHECK(expect_refuse(1,400, nb1, 4,+ "fp8-e4m3-b128 with ue8m0 scales, ALSO colliding with fmt=1 per-row (O=1) -> refused"));+}++/* ---- Part B: qt_from_disk loader-seam (real safetensors file) ---- */++static void deq_fmt7(const QT *t, float *dq){+ int64_t nblkI = fp8_nblk(t->I);+ for(int o=0;o<t->O;o++){+ int64_t blkO = o/FP8_BLOCK; const float *scl = t->s + blkO*nblkI;+ for(int i=0;i<t->I;i++){+ int64_t bi = i/FP8_BLOCK;+ dq[(int64_t)o*t->I+i] = e4m3_decode(t->q8[(int64_t)o*t->I+i]) * scl[bi];+ }+ }+}+static void deq_fmt1(const QT *t, float *dq){+ for(int o=0;o<t->O;o++){ float s=t->s[o];+ for(int i=0;i<t->I;i++) dq[(int64_t)o*t->I+i]=(float)t->q8[(int64_t)o*t->I+i]*s; }+}++#define CDIV(n,d) (((n)+(d)-1)/(d))++static void test_loader_seam(void){+ enum { O7=8, I7=256 }; /* nblkO=1, nblkI=2 -> 2 block scales total */+ enum { O1=5, I1=64 }; /* DIFFERENT shape from the fp8 tensor: O1*I1=320+ * bytes, ns=O1*4=20 -- neither collides with the+ * fp8 tensor's own byte counts (kept deliberately+ * distinct so this is a plain, non-degenerate+ * negative control, not another landmine case). */+ enum { NBLK7 = CDIV(O7,128) * CDIV(I7,128) }; /* must be a compile-time constant expression for+ * the static array below -- fp8_nblk() is a real+ * function (runtime, not constexpr), so it can't+ * size a `static` array even with literal inputs. */+ static uint8_t q7[O7*I7]; static float s7[NBLK7];+ for(int i=0;i<O7*I7;i++) q7[i]=rndbyte_nonan();+ for(int i=0;i<(int)(sizeof s7/sizeof *s7);i++) s7[i]=0.01f+0.001f*(float)i;++ static int8_t q1[O1*I1]; static float s1[O1];+ for(int i=0;i<O1*I1;i++) q1[i]=(int8_t)(rndbyte_nonan()-128);+ for(int i=0;i<O1;i++) s1[i]=0.02f+0.001f*(float)i;++ const char *dir="tests/tmp_fp8_snap";+#ifdef _WIN32+ mkdir(dir);+#else+ mkdir(dir,0755);+#endif+ char path[300]; snprintf(path,sizeof path,"%s/model.safetensors",dir);+ int64_t nb7=(int64_t)O7*I7, ns7=(int64_t)(sizeof s7);+ int64_t nb1=(int64_t)O1*I1, ns1=(int64_t)O1*4;+ char hdr[1024];+ int hl=snprintf(hdr,sizeof hdr,+ "{\"w7\":{\"dtype\":\"U8\",\"shape\":[%lld],\"data_offsets\":[0,%lld]},"+ "\"w7.qs\":{\"dtype\":\"F32\",\"shape\":[%lld],\"data_offsets\":[%lld,%lld]},"+ "\"w1\":{\"dtype\":\"U8\",\"shape\":[%lld],\"data_offsets\":[%lld,%lld]},"+ "\"w1.qs\":{\"dtype\":\"F32\",\"shape\":[%lld],\"data_offsets\":[%lld,%lld]}}",+ (long long)nb7,(long long)nb7,+ (long long)(ns7/4),(long long)nb7,(long long)(nb7+ns7),+ (long long)nb1,(long long)(nb7+ns7),(long long)(nb7+ns7+nb1),+ (long long)O1,(long long)(nb7+ns7+nb1),(long long)(nb7+ns7+nb1+ns1));+ FILE *f=fopen(path,"wb");+ if(!f){ printf("FAIL: cannot create %s (run from c/, like tools/run_tests.py does)\n", path); fails++; return; }+ uint64_t hlen=(uint64_t)hl;+ fwrite(&hlen,8,1,f); fwrite(hdr,1,hl,f);+ fwrite(q7,1,(size_t)nb7,f); fwrite(s7,1,(size_t)ns7,f);+ fwrite(q1,1,(size_t)nb1,f); fwrite(s1,1,(size_t)ns1,f);+ fclose(f);++ static Model gm; /* only gm.S is used by qt_from_disk */+ st_init(&gm.S, dir);++ QT t7; memset(&t7,0,sizeof t7);+ qt_from_disk(&gm,"w7",O7,I7,8,0,&t7);+ CHECK(t7.fmt==7);+ CHECK(t7.q8!=NULL && t7.s!=NULL); /* both weight and scale allocated (qalloc, not falloc) */+ static float dq_load[O7*I7], dq_ref[O7*I7];+ deq_fmt7(&t7,dq_load);+ QT tr7={.fmt=7,.q8=(int8_t*)q7,.s=s7,.O=O7,.I=I7};+ deq_fmt7(&tr7,dq_ref);+ CHECK(memcmp(dq_load,dq_ref,sizeof dq_ref)==0);++ QT t1; memset(&t1,0,sizeof t1);+ qt_from_disk(&gm,"w1",O1,I1,8,0,&t1);+ CHECK(t1.fmt==1); /* negative control: plain int8 still resolves as fmt=1 */+ static float dq_load1[O1*I1], dq_ref1[O1*I1];+ deq_fmt1(&t1,dq_load1);+ QT tr1={.fmt=1,.q8=q1,.s=s1,.O=O1,.I=I1};+ deq_fmt1(&tr1,dq_ref1);+ CHECK(memcmp(dq_load1,dq_ref1,sizeof dq_ref1)==0);++ unlink(path); rmdir(dir);+}++/* ---- Part C: qt_bytes()/qt_scale_bytes() byte-accounting for fmt=7 ----+ *+ * qt_bytes() must not fall through to the fmt=2 (packed int4, O*ceil(I/2)+O*4)+ * default -- for a real fp8 tensor that would undercount the resident byte+ * count by roughly half (an AUTOPIN/RAM-budget-feeding hazard). qt_wire_mmap/+ * qt_unwire_mmap must not hardcode scale_b=O*4 (per-row) either -- wrong for+ * fmt=7's per-128x128-block scale array -- hence the dedicated+ * qt_scale_bytes() helper shared by qt_bytes() and both wire functions so+ * there is exactly one place that knows each format's scale geometry. This+ * test exercises the arithmetic directly (no qt_from_disk/disk I/O, no mlock+ * syscall -- qt_wire_mmap's actual mem_wire() call is environment-dependent+ * (RLIMIT_MEMLOCK) and not what changed; the byte-count formula is) across+ * the shapes already used elsewhere in this file plus a block-edge+ * (non-128-multiple) case. */+static void check_fp8_bytes(int O, int I, const char *tag){+ QT t; memset(&t,0,sizeof t); t.fmt=7; t.O=O; t.I=I; t.gs=0;+ int64_t nblkO=fp8_nblk(O), nblkI=fp8_nblk(I), nblk=nblkO*nblkI;+ int64_t want_total = (int64_t)O*I + nblk*4;+ int64_t want_scale = nblk*4;+ int64_t got_total = qt_bytes(&t);+ int64_t got_scale = qt_scale_bytes(&t);+ if(got_total != want_total)+ printf("FAIL %s: qt_bytes=%lld want=%lld\n", tag, (long long)got_total, (long long)want_total);+ CHECK(got_total == want_total);+ if(got_scale != want_scale)+ printf("FAIL %s: qt_scale_bytes=%lld want=%lld\n", tag, (long long)got_scale, (long long)want_scale);+ CHECK(got_scale == want_scale);+ /* weight_b, as qt_wire_mmap/qt_unwire_mmap now compute it, must land on the exact+ * O*I raw-byte weight region -- not short (partial mlock) or long (mlock past the+ * allocation, undefined behavior) by even one byte. */+ CHECK(got_total - got_scale == (int64_t)O*I);+ /* regression guard: the fmt=2 (packed-nibble) formula must NOT be what fmt=7+ * returns -- confirm the value has actually MOVED off it (catches a silent+ * revert of the fmt==7 branch order/placement, not just a formula typo). For+ * O=1,I=1 the two formulas coincide by coincidence (both give 1+4=5), so that+ * shape is skipped for this particular guard -- the other three shapes below are+ * chosen to avoid the coincidence. */+ int64_t old_wrong = (int64_t)O*((I+1)/2) + (int64_t)O*4;+ if(!(O==1 && I==1)) CHECK(got_total != old_wrong);+}++int main(void){+ test_disambiguation();+ test_fmt6_fp8_collision();+ test_ue8m0_scale_refusal();+ test_loader_seam();+ check_fp8_bytes(2048,6144, "qt_bytes fmt=7 gate/up-shaped O=2048 I=6144 (spec example)");+ check_fp8_bytes(6144,2048, "qt_bytes fmt=7 down-shaped O=6144 I=2048");+ check_fp8_bytes(130,200, "qt_bytes fmt=7 block edges O,I both non-mult-128");+ check_fp8_bytes(1,1, "qt_bytes fmt=7 degenerate 1x1");+ if(fails){ printf("fp8 loader-seam tests: %d FAILED\n", fails); return 1; }+ printf("fp8 loader-seam tests: ok\n");+ return 0;+}diff --git a/c/tests/test_fp8_passthrough.c b/c/tests/test_fp8_passthrough.cnew file mode 100644index 00000000..61fb3709--- /dev/null+++ b/c/tests/test_fp8_passthrough.c@@ -0,0 +1,165 @@+/* fmt=7 (native FP8-e4m3 passthrough) CPU kernel tests: LUT exactness (all 256+ * byte codes, incl. +-0, subnormals, NaN), matmul_fp8 vs a double-precision+ * block-scale reference on block-edge (non-128-multiple O/I) shapes, a+ * non-square-block-grid case with a distinct scale per block (catches a+ * transposed/swapped block-index stride the same way test_backend_metal.mm's+ * "stride-audit" case does for the GPU kernel), and the NaN-propagation policy+ * (decode -> real IEEE NaN -> flows through the dot product; see quant.h's+ * e4m3_decode/E4M3_LUT comment for the documented policy + rationale).+ * Pure quant.h test: no Model/QT/disk dependency (see quant.h's own "pure+ * compute" header comment) -- the loader-seam (qt_resolve_fmt disambiguation,+ * qt_from_disk) is covered separately in test_fp8_load.c.+ *+ * fmt=7, PUBLIC ordinal assigned by the maintainer on #524: this format was+ * minted fmt=6 during original development of this branch, before dev's own+ * #465 (E8/IQ3) claimed that ordinal upstream; re-tagged fmt=100 (PRIVATE+ * ORDINAL BLOCK, see colibri.c's QT struct comment) from that point forward --+ * there was never a build in this branch's history where this format was+ * reachable as fmt=6 -- and graduated to fmt=7 at merge. */+#include "../quant.h"+#include <stdio.h>+#include <string.h>+#include <math.h>++static int fails = 0;+#define CHECK(c) do{ if(!(c)){ printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #c); fails++; } }while(0)++static uint64_t rng = 0xC0FFEE1234567ull;+static float rndf(void){ rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17;+ return ((int64_t)(rng & 0xFFFFF) - 0x80000) / (float)0x80000; }+/* random byte EXCLUDING the two NaN codes (0x7F/0xFF) -- used by the accuracy/+ * magnitude tests below so a stray NaN term doesn't turn `rel>tol` trivially+ * false (NaN compares false against everything) and mask a real kernel bug.+ * NaN handling itself is checked separately, deliberately, below. */+static uint8_t rndbyte_nonan(void){+ for(;;){ rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17;+ uint8_t b = (uint8_t)(rng & 0xFF);+ if(b != 0x7F && b != 0xFF) return b; }+}++/* independent reference decode (bit manipulation, not the LUT under test) --+ * cross-checked offline against torch.float8_e4m3fn byte-for-byte (256/256+ * match, RAN evidence, see the build report). Kept here so a transcription+ * bug in E4M3_LUT's 256 hex-float literals is still caught by this test even+ * though both were derived from the same formula. */+static float ref_e4m3(uint8_t b){+ unsigned sign=(b>>7)&1, exp=(b>>3)&0xF, mant=b&0x7;+ if(exp==0xF && mant==0x7) return NAN;+ float val;+ if(exp==0) val = (float)mant * (1.0f/8.0f) * powf(2.0f, 1.0f-7.0f);+ else val = (1.0f + (float)mant*(1.0f/8.0f)) * powf(2.0f, (float)exp-7.0f);+ return sign ? -val : val;+}++static void test_lut(void){+ int nan_codes=0;+ for(int b=0;b<256;b++){+ float got = e4m3_decode((uint8_t)b);+ float ref = ref_e4m3((uint8_t)b);+ if(isnan(ref)){ CHECK(isnan(got)); nan_codes++; continue; }+ CHECK(got == ref);+ if(got==0.f) CHECK(signbit(got)==signbit(ref)); /* +-0 must keep its sign bit */+ }+ CHECK(nan_codes==2); /* exactly 0x7F and 0xFF */+ CHECK(isnan(e4m3_decode(0x7F)) && isnan(e4m3_decode(0xFF)));+ CHECK(e4m3_decode(0x7E)==448.0f && e4m3_decode(0xFE)==-448.0f); /* max finite, OCP E4M3FN */+ CHECK(e4m3_decode(0x01)==0x1.0p-9f); /* min positive subnormal = 2^-9 */+ CHECK(e4m3_decode(0x00)==0.0f && !signbit(e4m3_decode(0x00)));+ CHECK(e4m3_decode(0x80)==0.0f && signbit(e4m3_decode(0x80))); /* -0.0 */+}++/* double-precision block-scale reference: mirrors matmul_fp8's formula exactly+ * (w[o,i] = e4m3_decode(byte) * scale[o/128,i/128]) but every intermediate is+ * double so it can serve as ground truth for matmul_fp8's own float/double mix. */+static void ref_matmul_fp8(double *y, double *mag, const float *x, const uint8_t *q8,+ const float *bscale, int S, int I, int O){+ int64_t nblkI = fp8_nblk(I);+ for(int o=0;o<O;o++){+ const uint8_t *w = q8 + (int64_t)o*I;+ int64_t blkO = o / FP8_BLOCK;+ const float *scl = bscale + blkO*nblkI;+ for(int s=0;s<S;s++){+ const float *xs = x + (int64_t)s*I;+ double a=0, m=0;+ for(int i=0;i<I;i++){+ int64_t bi = i/FP8_BLOCK;+ double term = (double)e4m3_decode(w[i]) * (double)xs[i] * (double)scl[bi];+ a += term; m += fabs(term);+ }+ y[(int64_t)s*O+o]=a; mag[(int64_t)s*O+o]=m;+ }+ }+}++/* magnitude-relative Sigma|terms| oracle (the convention test_backend_metal.mm's+ * cpu_ref_grouped/run_grouped established for fmt=4 and reuses for fmt=7 on the+ * GPU side): compare error against the sum of |terms|, not the (possibly+ * near-zero, cancellation-prone) result itself. */+static int check_close(const float *got, const double *ref, const double *mag, int n, double tol, const char *tag){+ double worst=0; int bad=0;+ for(int i=0;i<n;i++){+ double d = fabs((double)got[i]-ref[i]);+ double rel = mag[i]>1e-30 ? d/mag[i] : d;+ if(rel>worst) worst=rel;+ if(rel>tol) bad++;+ }+ if(bad) printf(" %-40s worst_rel=%.3e FAIL (%d/%d over tol)\n", tag, worst, bad, n);+ return bad==0;+}++static void run_block_case(int O, int I, const char *tag){+ int64_t nblkO=fp8_nblk(O), nblkI=fp8_nblk(I), nblk=nblkO*nblkI;+ uint8_t *q8 = malloc((size_t)O*I);+ float *bscale = malloc((size_t)nblk*sizeof(float));+ float *x = malloc((size_t)3*I*sizeof(float));+ float *y = malloc((size_t)3*O*sizeof(float));+ double *yr = malloc((size_t)3*O*sizeof(double));+ double *mag = malloc((size_t)3*O*sizeof(double));+ for(int64_t i=0;i<(int64_t)O*I;i++) q8[i]=rndbyte_nonan();+ /* distinct, easily-distinguishable scale per block (b+1)*0.01 with alternating+ * sign and a random jitter -- a swapped blkO/blkI stride or a wrong nblkI in the+ * indexing picks up the WRONG block's scale, which this makes numerically loud+ * rather than "close enough to pass by luck" (same idea as the GPU stride-audit+ * case, applied here to the CPU kernel). */+ for(int64_t b=0;b<nblk;b++) bscale[b] = (float)(b+1)*0.01f*((b&1)?-1.f:1.f) + rndf()*0.001f;+ for(int i=0;i<3*I;i++) x[i]=rndf();+ ref_matmul_fp8(yr, mag, x, q8, bscale, 3, I, O);+ matmul_fp8(y, x, q8, bscale, 3, I, O);+ CHECK(check_close(y, yr, mag, 3*O, 1e-5, tag));+ free(q8); free(bscale); free(x); free(y); free(yr); free(mag);+}++static void test_nan_propagation(void){+ /* one NaN byte in an otherwise-normal tensor -> that OUTPUT ROW is NaN (the+ * documented policy: propagate, rely on the existing argmax_v/dist_build net+ * downstream -- see quant.h's e4m3_decode comment). Rows that never touch the+ * NaN byte stay finite: NaN contamination is per-dot-product, not global.+ * O=3<=128 -> nblkO=1, so all three rows share the SAME nblkI=2-entry scale+ * block-row (bscale[0..1]); q8 is the only thing that differs per row. */+ enum { O=3, I=200 };+ static uint8_t q8[O*I];+ static float bscale[2]; /* nblkO(1)*nblkI(2) */+ for(int i=0;i<O*I;i++) q8[i]=rndbyte_nonan();+ q8[1*I + 37] = 0x7F; /* row 1 only, one NaN byte */+ bscale[0]=0.1f; bscale[1]=0.11f;+ static float x[I], y[O];+ for(int i=0;i<I;i++) x[i]=rndf();+ matmul_fp8(y, x, q8, bscale, 1, I, O);+ CHECK(!isnan(y[0]) && !isnan(y[2])); /* untouched rows stay finite */+ CHECK(isnan(y[1])); /* contaminated row propagates NaN */+}++int main(void){+ test_lut();+ run_block_case(2048, 6144, "block gate/up-shaped O=2048 I=6144 (spec example)");+ run_block_case(6144, 2048, "block down-shaped O=6144 I=2048");+ run_block_case(130, 200, "block edges: O,I both non-mult-128");+ run_block_case(129, 128, "block edges: O just over 128, I exact");+ run_block_case(128, 129, "block edges: O exact, I just over 128");+ run_block_case(1, 1, "degenerate 1x1 (single sub-block)");+ run_block_case(384, 6144, "non-square block grid nblkO=3 nblkI=48 (stride audit)");+ test_nan_propagation();+ if(fails){ printf("fp8 passthrough CPU tests: %d FAILED\n", fails); return 1; }+ printf("fp8 passthrough CPU tests: ok\n");+ return 0;+}