Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

metal: widen the fmt-gate notice to all fused-bound tensors, share the gates' predicate

monotophicddf5cbbd2ebe
diff --git a/c/colibri.c b/c/colibri.c
index d0e6f515..19a60329 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -83,12 +83,15 @@ static inline void omp_set_num_threads(int n){ (void)n; }
#ifdef COLI_VULKAN
#include "backend_vulkan.h"
#endif
-/* Declared unconditionally (not just under COLI_METAL): on a non-Metal build it just sits
- * at 0 forever, which is the correct value there (no Metal backend => never enabled). Kept
- * outside the #ifdef so portable code β€” e.g. kvb_fmt_gate_notice below β€” can read "is Metal
- * active" without needing COLI_METAL (and the backend_metal.h / Metal framework link it
- * would drag in) on every platform's test build. */
+/* Declared unconditionally (not just under COLI_METAL): on a non-Metal build they just sit
+ * at 0 forever, which is the correct value there (no Metal backend => never enabled, and
+ * COLI_METAL_MOE_EXACT is only parsed under COLI_METAL). Kept outside the #ifdef so
+ * portable code β€” the shared fused-path format predicate metal_fused_layer_fmt_miss and
+ * metal_fmt_gate_notice below β€” can read "is Metal active" and the MOE-exact mode without
+ * needing COLI_METAL (and the backend_metal.h / Metal framework link it would drag in) on
+ * every platform's test build. */
static int g_metal_enabled;
+static int g_moe_exact=0; /* COLI_METAL_MOE_EXACT: exact (ungrouped) MoE kernels only */
/* COLI_SLAB_SHRINK=0 disables the #856 slab shrink at runtime.
*
* The shrink stops a wide slab migrating into a narrow row through the
@@ -116,7 +119,9 @@ static int g_slab_shrink = 1;
* read it; the duplicate that used to sit here was a tentative definition the
* linker merged, hence no warning and no symptom. */
static int g_metal_gemm_min=16; /* COLI_METAL_GEMM_MIN: min rows to send a matmul_qt GEMM to GPU */
-static int g_moe_exact=0;
+/* g_moe_exact moved next to g_metal_enabled above, outside the #ifdef: the shared
+ * fused-path format predicate (metal_fused_layer_fmt_miss) consumes it and must
+ * compile on every platform's test build. */
/* output dello shared expert gia' calcolato su GPU (solo Metal layer-CB) */
static const float *g_pre_sh;
#endif
@@ -1811,27 +1816,81 @@ static void layer_cuda_shard_kvb(Layer *l,int H,int Q,int V){
}
#endif
-/* KV_B FMT-GATE NOTICE (#kvb): kernel contract β€” the fused Metal attention kernels
- * (attention_rows' coli_metal_attn_decode and layer_forward_rows' coli_metal_layer_decode,
- * both gated on `l->kv_b.fmt==2||l->kv_b.fmt==4`) only run against kv_b_proj stored INT4,
- * either per-row (fmt=2) or grouped (fmt=4, since #587's kv_b grouped-int4 addition); any
- * OTHER format silently falls through to the CPU absorb path for decode attention on that
- * layer. v1-class mixed-precision containers can mint kv_b_proj at a format outside that
- * pair, so this is a real trap, not a hypothetical: print it once (called from model_init,
- * after all layer tensors are resolved) so it isn't silent. NOTICE ONLY β€” no gate/behavior
- * change here. */
-static void kvb_fmt_gate_notice(Model *m){
+/* One bit per weight tensor the fused Metal decode kernels bind, as reported by
+ * metal_fused_layer_fmt_miss() (defined next to metal_fused_fmt_ok below β€” the single
+ * per-layer format predicate both fused gates AND metal_fmt_gate_notice consume). The
+ * two masks name exactly which of the 8 each fused entry binds. */
+enum {
+ METAL_FUSED_KV_B = 1<<0, /* the gates' own kv_b term: fmt==2, or fmt==4 with g_moe_exact off */
+ METAL_FUSED_Q_A = 1<<1, /* the rest: metal_fused_fmt_ok() allowlist */
+ METAL_FUSED_Q_B = 1<<2,
+ METAL_FUSED_KV_A = 1<<3,
+ METAL_FUSED_O = 1<<4,
+ METAL_FUSED_SH_GATE = 1<<5, /* sh_*: checked on sparse layers only β€” dense layers */
+ METAL_FUSED_SH_UP = 1<<6, /* never load them (fmt stays 0) and never reach the */
+ METAL_FUSED_SH_DOWN = 1<<7, /* fused layer CB, so a miss there would be noise */
+};
+#define METAL_FUSED_ATTN_TENSORS (METAL_FUSED_KV_B|METAL_FUSED_Q_A|METAL_FUSED_Q_B|METAL_FUSED_KV_A|METAL_FUSED_O)
+#define METAL_FUSED_LAYER_TENSORS (METAL_FUSED_ATTN_TENSORS|METAL_FUSED_SH_GATE|METAL_FUSED_SH_UP|METAL_FUSED_SH_DOWN)
+static unsigned metal_fused_layer_fmt_miss(const Layer *l);
+
+/* FUSED-METAL FORMAT-GATE NOTICE (#kvb, widened): kernel contract β€” both fused Metal
+ * decode entries (attention_rows' coli_metal_attn_decode and layer_forward_rows'
+ * coli_metal_layer_decode) gate per layer on metal_fused_layer_fmt_miss() over the weight
+ * tensors they bind: kv_b_proj on the gates' own two-format+mode term (int4 per-row
+ * fmt==2 always; grouped fmt==4 only while g_moe_exact is off β€” #587's grouped-int4 kv_b
+ * is mode-gated), the rest on the metal_fused_fmt_ok() allowlist (fmt 1/2/3/4). Any miss
+ * silently drops that layer off the fused path onto the CPU implementation, with no
+ * signal anywhere. v1-class mixed-precision containers can mint any of these tensors at
+ * an off-allowlist format, so print it once per offending tensor KIND (never per layer β€”
+ * bounded at 8 lines) from model_init, after all layer tensors are resolved. The
+ * condition is the gates' own predicate, so notice and gates cannot drift apart again
+ * (the kv_b-only notice this replaces re-typed the gate condition and had already gone
+ * stale: it missed the `!g_moe_exact` term, staying silent on fmt=4 kv_b under
+ * COLI_METAL_MOE_EXACT=1 while both gates closed). NOTICE ONLY β€” no gate/behavior change
+ * here. Main layers only (m->L): the MTP head (m->mtpL) is deliberately excluded β€”
+ * common containers ship it at INT8 by design, and the fused gates never see it.
+ * FORMAT-ONLY SEMANTICS: a line here names a FORMAT (or format-mode) obstacle and
+ * nothing else β€” it does not claim the fused path would otherwise engage. The gates
+ * carry further runtime preconditions (GLM-5.2 dims, batch shape/S, ragged-mux guard,
+ * expert config) that this notice deliberately does not duplicate: those are properties
+ * of the call, not of the container, and re-stating them here would recreate exactly the
+ * duplicated-condition drift the shared predicate exists to kill. On a container/arch
+ * the fused path could never serve anyway, an off-allowlist tensor still gets its line β€”
+ * the format is still worth curing before it bites on a config that DOES reach the
+ * gates. sh_* lines count over the SPARSE-layer population (the only layers that load
+ * them); every other kind counts over all main layers. */
+static void metal_fmt_gate_notice(Model *m){
Cfg *c=&m->c;
if(!g_metal_enabled) return;
- int bad=0, first_fmt=-1;
- for(int i=0;i<c->n_layers;i++) if(m->L[i].kv_b.fmt!=2 && m->L[i].kv_b.fmt!=4){
- bad++; if(first_fmt<0) first_fmt=m->L[i].kv_b.fmt; }
- if(bad) fprintf(stderr,
- "[METAL] kv_b_proj is fmt=%d (not int4) on %d/%d layer%s: the fused Metal "
- "attention path requires kv_b in int4, per-row (fmt=2) or grouped (fmt=4), so "
- "those layers run decode attention on the CPU absorb path instead. Requantize "
- "kv_b to int4 (per-row --kvb-bits 4, or grouped) to re-enable the fused path.\n",
- first_fmt, bad, c->n_layers, bad==1?"":"s");
+ enum { NK=8 };
+ int nbad[NK]={0}, ffmt[NK]={-1,-1,-1,-1,-1,-1,-1,-1}, nsparse=0;
+ for(int i=0;i<c->n_layers;i++){
+ Layer *l=&m->L[i];
+ if(l->sparse) nsparse++;
+ unsigned miss=metal_fused_layer_fmt_miss(l);
+ if(!miss) continue;
+ const int f[NK]={ l->kv_b.fmt, l->q_a.fmt, l->q_b.fmt, l->kv_a.fmt,
+ l->o.fmt, l->sh_gate.fmt, l->sh_up.fmt, l->sh_down.fmt };
+ for(int k=0;k<NK;k++) if(miss>>k & 1u){ nbad[k]++; if(ffmt[k]<0) ffmt[k]=f[k]; }
+ }
+ if(nbad[0]) fprintf(stderr,
+ "[METAL] kv_b_proj is fmt=%d on %d/%d layer%s: the fused Metal attention path "
+ "serves kv_b only as int4, per-row (fmt=2) or grouped (fmt=4; grouped is served "
+ "only while COLI_METAL_MOE_EXACT is off), so those layers run decode attention "
+ "on the CPU absorb path instead. Requantize kv_b to int4 (--kvb-bits 4) to "
+ "re-enable the fused path.\n",
+ ffmt[0], nbad[0], c->n_layers, nbad[0]==1?"":"s");
+ static const char *const kind_name[NK]={ "kv_b_proj" /* [0] has its own line above */,
+ "q_a_proj","q_b_proj","kv_a_proj_with_mqa","o_proj",
+ "shared_experts.gate_proj","shared_experts.up_proj","shared_experts.down_proj" };
+ for(int k=1;k<NK;k++) if(nbad[k]) fprintf(stderr,
+ "[METAL] %s is fmt=%d on %d/%d %slayer%s: outside the fused-path allowlist "
+ "(fmt 1/2/3/4), so the fused Metal decode kernels that bind this tensor skip "
+ "those layers and they run on the CPU path instead.\n",
+ kind_name[k], ffmt[k], nbad[k],
+ k>=5?nsparse:c->n_layers, /* sh_* (k>=5): only sparse layers load them */
+ k>=5?"sparse ":"", nbad[k]==1?"":"s");
}
static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits){
@@ -1909,7 +1968,7 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits
}
#undef P
}
- kvb_fmt_gate_notice(m); /* once per load, after all layer kv_b resolved */
+ metal_fmt_gate_notice(m); /* once per load, after all layer tensors resolved */
/* testa MTP (layer n_layers): presente solo se convertita con --mtp */
{
/* MTP attiva SOLO se il set e' COMPLETO (i tensori vivono su 3 shard: durante la
@@ -3306,6 +3365,36 @@ static int attn_pipe_prefix(Model *m,Layer *l,const float *x,int S,float *QR,flo
* can unit-test the truth table (tests/test_fp8_load.c Part F). */
static inline int metal_fused_fmt_ok(int fmt){ return fmt==1 || fmt==2 || fmt==3 || fmt==4; }
+/* Per-layer companion to metal_fused_fmt_ok: ONE predicate over all the weight tensors
+ * the fused decode kernels bind, returning a bitmask of offending tensors
+ * (METAL_FUSED_*, 0 == fully fused-eligible). Both fused gates test it against the mask
+ * of tensors their kernel actually binds (METAL_FUSED_ATTN_TENSORS /
+ * METAL_FUSED_LAYER_TENSORS), and metal_fmt_gate_notice (model_init) reports every set
+ * bit β€” the notice consumes the gates' own condition, so the two can never drift apart
+ * again (the previous kv_b-only notice re-typed the gate condition and was already
+ * stale: it omitted the `!g_moe_exact` term below). kv_b's term is the gates' own:
+ * fmt==2 (int4 per-row, its absorb kernel's native format) always serves; fmt==4
+ * (grouped int4, #587) serves only while g_moe_exact is off β€” COLI_METAL_MOE_EXACT=1
+ * keeps grouped kv_b off the fused path. sh_gate/sh_up/sh_down are checked on sparse
+ * layers only: dense layers never load them (fmt stays 0) and layer_forward_rows' fused
+ * path already requires l->sparse, so flagging them there would be pure false positive.
+ * Reads only the layer's fmt/sparse fields and the g_moe_exact mode flag β€” no side
+ * effects. */
+static unsigned metal_fused_layer_fmt_miss(const Layer *l){
+ unsigned miss=0;
+ if(!(l->kv_b.fmt==2 || (l->kv_b.fmt==4 && !g_moe_exact))) miss|=METAL_FUSED_KV_B;
+ if(!metal_fused_fmt_ok(l->q_a.fmt)) miss|=METAL_FUSED_Q_A;
+ if(!metal_fused_fmt_ok(l->q_b.fmt)) miss|=METAL_FUSED_Q_B;
+ if(!metal_fused_fmt_ok(l->kv_a.fmt)) miss|=METAL_FUSED_KV_A;
+ if(!metal_fused_fmt_ok(l->o.fmt)) miss|=METAL_FUSED_O;
+ if(l->sparse){
+ if(!metal_fused_fmt_ok(l->sh_gate.fmt)) miss|=METAL_FUSED_SH_GATE;
+ if(!metal_fused_fmt_ok(l->sh_up.fmt)) miss|=METAL_FUSED_SH_UP;
+ if(!metal_fused_fmt_ok(l->sh_down.fmt)) miss|=METAL_FUSED_SH_DOWN;
+ }
+ return miss;
+}
+
static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int pos_base,
KVState *const *kvs, const int *positions, float *out){
Cfg *c=&m->c; int H=c->n_heads, D=c->hidden, qh=c->qk_head, vh=c->v_head;
@@ -3320,28 +3409,30 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p
* would rope every row at position 0 and attend over a 1-token window of the wrong
* cache -> greedy decode hits EOS at token 2 (mux answers truncated to 1 token).
* Ragged rows take the CPU absorb path below, which reads kvs[s]/positions[s].
- * metal_fused_fmt_ok(q_a/q_b/kv_a/o): POSITIVE allowlist (fmt 1/2/4, see the
- * predicate's own comment above attention_rows). Previously these were
- * negative fmt!=8 guards -- fail-OPEN for everything they didn't name: for
- * fmt=8 specifically the WP_() macro two lines below picks q8 only for
- * fmt==1, else q4 -- and q4 is NULL/unallocated for fmt=8 (same convention
- * as fmt=1, see the QT struct comment), so an fmt=8 tensor reaching
- * coli_metal_attn_decode would hand the kernel a NULL weight pointer; for
- * fmt=5/6 the hazard is WORSE -- their real weight bytes live in q4, so
- * WP_() hands the shader a valid pointer and mm_gemv's terminal f32
- * fallback branch silently misreads them as f32 garbage. The allowlist
- * fails CLOSED to the CPU path below for all of them (matmul_qt_ex there
- * dispatches every format correctly, incl. fmt=8 via matmul_fp8); wiring
- * fmt=8 through bind_gemv/coli_metal_attn_decode stays a deferred
- * follow-up (see this PR's GPU-path note). kv_b is already pinned to
- * fmt==2 above for an unrelated reason (its absorb kernel is int4-only);
- * these four checks are the same discipline extended to the tensors that
- * flow through the shared per-fmt shader. */
+ * metal_fused_layer_fmt_miss & METAL_FUSED_ATTN_TENSORS: kv_b on its
+ * two-format+mode term (fmt==2, or fmt==4 with g_moe_exact off) plus the
+ * POSITIVE allowlist (fmt 1/2/3/4) over q_a/q_b/kv_a/o -- see the shared
+ * predicate's own comment above attention_rows. The allowlist checks were
+ * previously negative fmt!=8 guards -- fail-OPEN for everything they
+ * didn't name: for fmt=8 specifically the WP_() macro two lines below
+ * picks q8 only for fmt==1, else q4 -- and q4 is NULL/unallocated for
+ * fmt=8 (same convention as fmt=1, see the QT struct comment), so an
+ * fmt=8 tensor reaching coli_metal_attn_decode would hand the kernel a
+ * NULL weight pointer; for fmt=5/6 the hazard is WORSE -- their real
+ * weight bytes live in q4, so WP_() hands the shader a valid pointer and
+ * mm_gemv's terminal f32 fallback branch silently misreads them as f32
+ * garbage. The allowlist fails CLOSED to the CPU path below for all of
+ * them (matmul_qt_ex there dispatches every format correctly, incl. fmt=8
+ * via matmul_fp8); wiring fmt=8 through bind_gemv/coli_metal_attn_decode
+ * stays a deferred follow-up (see this PR's GPU-path note). kv_b's
+ * stricter term lives inside the same predicate for an unrelated reason
+ * (its absorb kernel is int4-only, grouped served only outside MOE-exact
+ * mode); the four allowlist checks are the same discipline extended to
+ * the tensors that flow through the shared per-fmt shader. */
if(g_metal_enabled && !kvs && g_absorb!=0 && (S<=4 || g_metal_prefill) && m->kv_start[layer]==0
&& D==6144 && H==64 && c->q_lora==2048 && c->kv_lora==512 && c->qk_nope==192
- && c->qk_rope==64 && vh==256 && (l->kv_b.fmt==2||(l->kv_b.fmt==4&&!g_moe_exact))
- && metal_fused_fmt_ok(l->q_a.fmt) && metal_fused_fmt_ok(l->q_b.fmt)
- && metal_fused_fmt_ok(l->kv_a.fmt) && metal_fused_fmt_ok(l->o.fmt)){
+ && c->qk_rope==64 && vh==256
+ && !(metal_fused_layer_fmt_miss(l) & METAL_FUSED_ATTN_TENSORS)){
int sel_active = m->has_dsa && layer<c->n_layers && c->idx_type[layer] && (pos_base+S) > c->index_topk;
if(!sel_active){
if(m->has_dsa && layer<c->n_layers && c->idx_type[layer]){ /* index keys for future selection */
@@ -5665,8 +5756,10 @@ static void layer_forward_rows(Model *m, Layer *l, int li, float *x, int S, int
* Fallback: qualsiasi condizione mancante -> percorso CPU intero qui sotto.
* !kvs: ragged mux rows (per-row KV/position) are not expressible in this kernel's
* single Lc/Rc + pos_base contract β€” see the matching guard in attention_rows.
- * metal_fused_fmt_ok(q_a/q_b/kv_a/o/sh_gate/sh_up/sh_down): POSITIVE allowlist
- * (fmt 1/2/4), same fail-closed discipline as attention_rows, extended to the
+ * metal_fused_layer_fmt_miss & METAL_FUSED_LAYER_TENSORS: kv_b on its
+ * two-format+mode term (fmt==2, or fmt==4 with g_moe_exact off) plus the
+ * POSITIVE allowlist (fmt 1/2/3/4) over q_a/q_b/kv_a/o/sh_gate/sh_up/sh_down,
+ * same fail-closed discipline as attention_rows, extended to the
* shared-expert MLP this fused kernel also covers. The mm_gemv shader these
* bind_gemv calls dispatch through has a real fmt==8 branch (this PR's own
* Metal kernel commit), but the WP_() macro below picks q8 only for fmt==1
@@ -5681,12 +5774,9 @@ static void layer_forward_rows(Model *m, Layer *l, int li, float *x, int S, int
if(g_metal_enabled && !kvs && S<=4 && li<c->n_layers && l->sparse
&& (g_absorb==1||(g_absorb<0&&S<=4)) && m->kv_start[li]==0
&& D==6144 && c->n_heads==64 && c->q_lora==2048 && c->kv_lora==512
- && c->qk_nope==192 && c->qk_rope==64 && c->v_head==256 && (l->kv_b.fmt==2||(l->kv_b.fmt==4&&!g_moe_exact))
+ && c->qk_nope==192 && c->qk_rope==64 && c->v_head==256
&& c->n_experts==256 && c->topk==8 && c->n_shared==1 && c->moe_inter==2048
- && metal_fused_fmt_ok(l->q_a.fmt) && metal_fused_fmt_ok(l->q_b.fmt)
- && metal_fused_fmt_ok(l->kv_a.fmt) && metal_fused_fmt_ok(l->o.fmt)
- && metal_fused_fmt_ok(l->sh_gate.fmt) && metal_fused_fmt_ok(l->sh_up.fmt)
- && metal_fused_fmt_ok(l->sh_down.fmt)){
+ && !(metal_fused_layer_fmt_miss(l) & METAL_FUSED_LAYER_TENSORS)){
int sel_active = m->has_dsa && c->idx_type[li] && (pos_base+S) > c->index_topk;
if(!sel_active){
static float *linrm,*lnrm,*lsh,*lw; static int *lidx,*lkeff;
diff --git a/c/tests/test_fp8_load.c b/c/tests/test_fp8_load.c
index 6fcd9361..a333de0c 100644
--- a/c/tests/test_fp8_load.c
+++ b/c/tests/test_fp8_load.c
@@ -745,8 +745,14 @@ static void test_metal_fused_allowlist(void){
}
CHECK(!metal_fused_fmt_ok(100)); /* private-block ordinals stay out too */
/* (2) grep-pin: run_tests.py runs us with cwd=c/ (Part B's fixtures rely
- * on the same fact). 11 per-tensor call sites = 4 (attention_rows:
- * q_a/q_b/kv_a/o) + 7 (layer_forward_rows: those + sh_gate/sh_up/sh_down). */
+ * on the same fact). The per-tensor calls live in ONE place now --
+ * metal_fused_layer_fmt_miss(), the shared per-layer predicate both gates
+ * and metal_fmt_gate_notice consume -- so the pin is: exactly 7
+ * metal_fused_fmt_ok(l->...) call sites (q_a/q_b/kv_a/o +
+ * sh_gate/sh_up/sh_down, each once, inside the helper; kv_b uses its own
+ * two-format+mode term there, not the allowlist), and BOTH gate sites
+ * consult the helper against the mask of tensors their kernel binds.
+ * (test_kvb_notice.c pins the helper's own truth table.) */
FILE *f = fopen("colibri.c", "r");
if(!f){ printf("FAIL Part F grep-pin: cannot open colibri.c (cwd not c/?)\n"); CHECK(0); return; }
static char src[16*1024*1024];
@@ -754,8 +760,14 @@ static void test_metal_fused_allowlist(void){
CHECK(n > 100000); /* sanity: we read the real file */
int calls=0; const char *p=src;
while((p=strstr(p,"metal_fused_fmt_ok(l->"))!=NULL){ calls++; p++; }
- if(calls!=11) printf("FAIL Part F grep-pin: %d metal_fused_fmt_ok(l->...) call sites, want 11\n", calls);
- CHECK(calls==11);
+ if(calls!=7) printf("FAIL Part F grep-pin: %d metal_fused_fmt_ok(l->...) call sites, want 7\n", calls);
+ CHECK(calls==7);
+ int attn=0; p=src;
+ while((p=strstr(p,"metal_fused_layer_fmt_miss(l) & METAL_FUSED_ATTN_TENSORS"))!=NULL){ attn++; p++; }
+ CHECK(attn==1); /* attention_rows' gate consults the shared predicate */
+ int lay=0; p=src;
+ while((p=strstr(p,"metal_fused_layer_fmt_miss(l) & METAL_FUSED_LAYER_TENSORS"))!=NULL){ lay++; p++; }
+ CHECK(lay==1); /* layer_forward_rows' gate consults the shared predicate */
/* the old fail-open idiom must be gone from the gate sites (the CUDA
* eligibility guard's `w->fmt!=8` is a different site and untouched). */
CHECK(strstr(src,"l->q_a.fmt!=8")==NULL);
diff --git a/c/tests/test_kvb_notice.c b/c/tests/test_kvb_notice.c
index f21e0d81..c7a3b731 100644
--- a/c/tests/test_kvb_notice.c
+++ b/c/tests/test_kvb_notice.c
@@ -1,19 +1,24 @@
-/* kv_b fmt-gate NOTICE test (see kvb_fmt_gate_notice in colibri.c, called once from
- * model_init after all layer tensors are resolved).
+/* Fused-Metal format-gate NOTICE test (see metal_fmt_gate_notice in colibri.c, called
+ * once from model_init after all layer tensors are resolved).
*
- * The trap: a v1-class mixed-precision container can mint kv_b_proj at a format outside
- * the pair the fused Metal attention kernels actually serve. BOTH fused entries
- * (attention_rows' fused decode and layer_forward_rows' fused layer decode) hard-gate on
- * `l->kv_b.fmt==2||l->kv_b.fmt==4` (per-row or grouped int4 -- fmt=4 became a valid
- * GPU-served kv_b configuration once #587's grouped-int4 kv_b support landed), so a kv_b
- * outside {2,4} silently forces that layer's decode attention onto the CPU absorb path --
- * with no notice at all if this fires. This test proves the fix (a stderr notice, NOT a
- * behavior change): it fires exactly once when it should, and stays silent for BOTH valid
- * formats (fmt=2 and fmt=4) as well as for the MTP head (excluded by design: the notice
- * only scans m->L[0..n_layers), never m->mtpL). It calls kvb_fmt_gate_notice() directly
- * against a hand-built Model, so it needs no real snapshot on disk and no Metal backend
- * link -- g_metal_enabled is declared unconditionally in colibri.c precisely so this
- * stays portable.
+ * The trap: a v1-class mixed-precision container can mint any fused-path weight tensor
+ * at a format the fused Metal decode kernels cannot bind. BOTH fused entries
+ * (attention_rows' fused decode and layer_forward_rows' fused layer decode) gate per
+ * layer on metal_fused_layer_fmt_miss(): kv_b_proj on the gates' two-format+mode term
+ * (int4 per-row fmt==2 always; grouped fmt==4 only while g_moe_exact is off), the other
+ * bound tensors (q_a/q_b/kv_a/o and, on sparse layers, sh_gate/sh_up/sh_down) on the
+ * metal_fused_fmt_ok() allowlist (fmt 1/2/3/4). A miss silently drops the layer off the
+ * fused path onto the CPU implementation -- historically with no notice at all, and the
+ * kv_b-only notice this one replaced had itself drifted (it ignored g_moe_exact and
+ * stayed silent on fmt=4 kv_b under COLI_METAL_MOE_EXACT=1 while both gates closed).
+ * This test proves the notice (stderr only, NOT a behavior change): one line per
+ * offending tensor KIND (bounded, never per-layer), silent when clean, silent without
+ * Metal, MTP head excluded, and mode-aware exactly like the gates. It also pins the
+ * shared predicate's truth table and the gate masks' bit membership, since the gates
+ * consume the exact same function. It calls the functions directly against a hand-built
+ * Model, so it needs no real snapshot on disk and no Metal backend link --
+ * g_metal_enabled and g_moe_exact are declared unconditionally in colibri.c precisely so
+ * this stays portable.
*
* Portable stderr capture: freopen(stderr) onto a temp file, dup()/dup2() to save and
* restore the real fd 2 around it. Deliberately no child-process spawn/reap primitives
@@ -55,6 +60,20 @@ static void restore_stderr(int saved, char *buf, size_t bufsz){
close(saved);
}
+/* Runs the notice against `m` with stderr captured into buf. */
+static void run_notice(Model *m, char *buf, size_t bufsz){
+ int saved = redirect_stderr("tests/tmp_kvb_notice.stderr");
+ metal_fmt_gate_notice(m);
+ restore_stderr(saved, buf, bufsz);
+ remove("tests/tmp_kvb_notice.stderr");
+}
+
+static int count_sub(const char *hay, const char *needle){
+ int n=0; const char *p=hay;
+ while((p=strstr(p,needle))){ n++; p++; }
+ return n;
+}
+
static Model make_model(int n_layers){
Model m; memset(&m,0,sizeof m);
m.c.n_layers = n_layers;
@@ -62,127 +81,346 @@ static Model make_model(int n_layers){
return m;
}
+/* Every fused-bound tensor at an allowlisted format -- deliberately a MIX of all four
+ * admitted formats (1/2/3/4) so the allowlist itself is what keeps this silent. */
+static void set_clean(Model *m){
+ for(int i=0;i<m->c.n_layers;i++){
+ Layer *l=&m->L[i];
+ l->sparse=1;
+ l->kv_b.fmt=2;
+ l->q_a.fmt=1; l->q_b.fmt=4; l->kv_a.fmt=3; l->o.fmt=1;
+ l->sh_gate.fmt=1; l->sh_up.fmt=2; l->sh_down.fmt=4;
+ }
+}
+
+/* Kind index k follows the METAL_FUSED_* bit order (1..7 = the allowlist kinds). */
+static void set_kind_fmt(Layer *l, int k, int fmt){
+ switch(k){
+ case 1: l->q_a.fmt=fmt; break;
+ case 2: l->q_b.fmt=fmt; break;
+ case 3: l->kv_a.fmt=fmt; break;
+ case 4: l->o.fmt=fmt; break;
+ case 5: l->sh_gate.fmt=fmt; break;
+ case 6: l->sh_up.fmt=fmt; break;
+ case 7: l->sh_down.fmt=fmt; break;
+ }
+}
+static const char *const kind_str[8] = { "kv_b_proj",
+ "q_a_proj","q_b_proj","kv_a_proj_with_mqa","o_proj",
+ "shared_experts.gate_proj","shared_experts.up_proj","shared_experts.down_proj" };
+
int main(void){
- char buf[4096];
+ char buf[8192];
+ g_moe_exact = 0; /* default mode unless a case says otherwise */
- /* (a) fmt=1 kv_b on 2/3 layers (the other is fmt=2) + Metal enabled -> notice fires
- * exactly once, for a format OUTSIDE {2,4} -- fmt=1 (v1-class INT8 container) is
- * chosen specifically because it's neither of the two now-valid GPU-served formats. */
+ /* (a) fmt=1 kv_b on 2/3 layers + Metal enabled -> the kv_b notice appears exactly
+ * once, with fmt, count, requirement, consequence, remedy -- and nothing else.
+ * fmt=1 (v1-class INT8 container) is chosen specifically because it misses the
+ * kv_b term in BOTH modes. */
{
Model m = make_model(3);
- m.L[0].kv_b.fmt = 1; /* v1-class INT8 container: outside {2,4} */
- m.L[1].kv_b.fmt = 2;
+ set_clean(&m);
+ m.L[0].kv_b.fmt = 1; /* v1-class INT8 container: not int4 */
m.L[2].kv_b.fmt = 1;
g_metal_enabled = 1;
+ run_notice(&m, buf, sizeof buf);
- int saved = redirect_stderr("tests/tmp_kvb_notice_a.stderr");
- kvb_fmt_gate_notice(&m);
- restore_stderr(saved, buf, sizeof buf);
- remove("tests/tmp_kvb_notice_a.stderr");
-
- int occurrences = 0; const char *p = buf;
- while((p = strstr(p, "[METAL] kv_b_proj"))){ occurrences++; p++; }
- CHECK(occurrences == 1); /* exactly one notice, not one per layer */
+ CHECK(count_sub(buf, "[METAL] kv_b_proj") == 1); /* exactly one notice, not one per layer */
+ CHECK(count_sub(buf, "[METAL]") == 1); /* and no allowlist lines: kv_b only */
CHECK(strstr(buf, "fmt=1") != NULL); /* which fmt kv_b actually has */
CHECK(strstr(buf, "2/3 layer") != NULL); /* count of affected layers */
CHECK(strstr(buf, "fmt=2") != NULL); /* mentions the per-row option */
- CHECK(strstr(buf, "fmt=4") != NULL); /* mentions the grouped option -- fmt=4 is now valid */
+ CHECK(strstr(buf, "fmt=4") != NULL); /* mentions the grouped option -- fmt=4 is valid outside MOE-exact */
CHECK(strstr(buf, "CPU absorb path") != NULL); /* attention runs the CPU path */
CHECK(strstr(buf, "--kvb-bits 4") != NULL); /* remedy hint */
/* the pre-#587 message's PER-ROW-only warning and fmt=4-trap sentence must be
- * gone -- fmt=4 is a valid GPU-served kv_b now, not a second way to miss the gate. */
+ * gone -- fmt=4 is a valid GPU-served kv_b outside MOE-exact mode, not a second
+ * way to miss the gate. (These pins are carried from dev's test verbatim.) */
CHECK(strstr(buf, "PER-ROW") == NULL);
CHECK(strstr(buf, "keeping kv_b ungrouped") == NULL);
CHECK(strstr(buf, "also misses this fmt=2 gate") == NULL);
free(m.L);
}
- /* (b) fmt=2 kv_b on every layer -> no notice, even with Metal enabled */
+ /* (a2) fmt=2 kv_b on every layer -> no notice (dev-compat case (b)) */
{
Model m = make_model(3);
+ set_clean(&m);
for(int i=0;i<3;i++) m.L[i].kv_b.fmt = 2;
g_metal_enabled = 1;
-
- int saved = redirect_stderr("tests/tmp_kvb_notice_b.stderr");
- kvb_fmt_gate_notice(&m);
- restore_stderr(saved, buf, sizeof buf);
- remove("tests/tmp_kvb_notice_b.stderr");
-
+ run_notice(&m, buf, sizeof buf);
CHECK(buf[0] == 0);
free(m.L);
}
- /* (b2) fmt=4 (grouped int4) kv_b on every layer -> no notice either: fmt=4 is a valid
- * GPU-served kv_b configuration post-#587, not a trap (this is the semantic change
- * under test -- the pre-#587 notice fired here). */
+ /* (a3) fmt=4 (grouped int4) kv_b on every layer, default mode -> no notice: fmt=4
+ * is a valid GPU-served kv_b configuration post-#587 while g_moe_exact is off
+ * (dev-compat case (b2)). */
{
Model m = make_model(3);
+ set_clean(&m);
for(int i=0;i<3;i++) m.L[i].kv_b.fmt = 4;
g_metal_enabled = 1;
+ run_notice(&m, buf, sizeof buf);
+ CHECK(buf[0] == 0);
+ free(m.L);
+ }
- int saved = redirect_stderr("tests/tmp_kvb_notice_b2.stderr");
- kvb_fmt_gate_notice(&m);
- restore_stderr(saved, buf, sizeof buf);
- remove("tests/tmp_kvb_notice_b2.stderr");
-
+ /* (a4) mixed fmt=2/fmt=4 across layers, default mode -> still no notice (the OR
+ * term, not just either format alone, is what the fused path requires per-layer;
+ * dev-compat case (b3)). */
+ {
+ Model m = make_model(4);
+ set_clean(&m);
+ m.L[0].kv_b.fmt = 2; m.L[1].kv_b.fmt = 4; m.L[2].kv_b.fmt = 4; m.L[3].kv_b.fmt = 2;
+ g_metal_enabled = 1;
+ run_notice(&m, buf, sizeof buf);
CHECK(buf[0] == 0);
free(m.L);
}
- /* (b3) mixed fmt=2/fmt=4 across layers -> still no notice (the OR gate, not just
- * either format alone, is what the fused path actually requires per-layer). */
+ /* (a5) MODE TERM (the r4 drift cure): under g_moe_exact=1 (COLI_METAL_MOE_EXACT=1)
+ * the gates' kv_b term is fmt==2 EXACTLY -- grouped fmt=4 closes both gates, so the
+ * notice must fire for it. Mixed 2/4: only the fmt=4 layers are misses. The
+ * pre-r4 notice (fmt!=2 && fmt!=4) stayed silent here: gates closed, no signal. */
{
Model m = make_model(4);
+ set_clean(&m);
m.L[0].kv_b.fmt = 2; m.L[1].kv_b.fmt = 4; m.L[2].kv_b.fmt = 4; m.L[3].kv_b.fmt = 2;
g_metal_enabled = 1;
+ g_moe_exact = 1;
+ run_notice(&m, buf, sizeof buf);
+
+ CHECK(count_sub(buf, "[METAL] kv_b_proj") == 1);
+ CHECK(count_sub(buf, "[METAL]") == 1);
+ CHECK(strstr(buf, "fmt=4") != NULL); /* the offending format is grouped int4 */
+ CHECK(strstr(buf, "2/4 layer") != NULL); /* only the fmt=4 layers count */
+ CHECK(strstr(buf, "COLI_METAL_MOE_EXACT") != NULL); /* names the mode that excludes fmt=4 */
- int saved = redirect_stderr("tests/tmp_kvb_notice_b3.stderr");
- kvb_fmt_gate_notice(&m);
- restore_stderr(saved, buf, sizeof buf);
- remove("tests/tmp_kvb_notice_b3.stderr");
+ /* ... and all-fmt=4 under the same mode: 3/3 miss */
+ for(int i=0;i<4;i++) m.L[i].kv_b.fmt = 4;
+ m.c.n_layers = 3; /* reuse first 3 layers */
+ run_notice(&m, buf, sizeof buf);
+ CHECK(count_sub(buf, "[METAL] kv_b_proj") == 1);
+ CHECK(strstr(buf, "3/3 layer") != NULL);
+ /* fmt=2 stays served under MOE-exact: silent */
+ for(int i=0;i<3;i++) m.L[i].kv_b.fmt = 2;
+ run_notice(&m, buf, sizeof buf);
CHECK(buf[0] == 0);
+
+ g_moe_exact = 0;
free(m.L);
}
- /* (c) fmt=1 kv_b but Metal not enabled -> no notice (the fused paths this warns
- * about are Metal-only, so with Metal off there is nothing to warn about) */
+ /* (b) each allowlist tensor kind individually off-allowlist (fmt=8) on 2/4 layers
+ * -> exactly ONE line, naming that kind, its fmt, the layer count, the allowlist
+ * requirement, and the CPU consequence. No kv_b line, no other kind named. */
+ for(int k=1;k<8;k++){
+ Model m = make_model(4);
+ set_clean(&m);
+ set_kind_fmt(&m.L[1], k, 8);
+ set_kind_fmt(&m.L[3], k, 8);
+ g_metal_enabled = 1;
+ run_notice(&m, buf, sizeof buf);
+
+ char head[128];
+ snprintf(head, sizeof head, "[METAL] %s is fmt=8", kind_str[k]);
+ CHECK(count_sub(buf, "[METAL]") == 1); /* one line for the one bad kind */
+ CHECK(strstr(buf, head) != NULL); /* names the kind + actual fmt */
+ /* count of affected layers: sh_* kinds count over the sparse population (all 4
+ * layers are sparse here), every other kind over all main layers */
+ CHECK(strstr(buf, k>=5 ? "2/4 sparse layer" : "2/4 layer") != NULL);
+ if(k<5) CHECK(strstr(buf, "sparse") == NULL); /* attn-bound kinds never say sparse */
+ CHECK(strstr(buf, "fmt 1/2/3/4") != NULL); /* the allowlist requirement */
+ CHECK(strstr(buf, "CPU path") != NULL); /* the fused-path consequence */
+ CHECK(strstr(buf, "kv_b_proj") == NULL); /* clean kv_b stays unmentioned */
+ for(int j=1;j<8;j++) if(j!=k)
+ CHECK(strstr(buf, kind_str[j]) == NULL); /* no other kind named */
+ free(m.L);
+ }
+
+ /* (c) EVERYTHING bad on EVERY layer of a 40-layer model -> bounded output: exactly
+ * 8 lines (kv_b + 7 kinds), each kind named exactly once, counts say 40/40 --
+ * never per-layer spam (which would be 320 lines here). */
+ {
+ Model m = make_model(40);
+ set_clean(&m);
+ for(int i=0;i<40;i++){
+ m.L[i].kv_b.fmt = 1;
+ for(int k=1;k<8;k++) set_kind_fmt(&m.L[i], k, 9);
+ }
+ g_metal_enabled = 1;
+ run_notice(&m, buf, sizeof buf);
+
+ CHECK(count_sub(buf, "[METAL]") == 8); /* bounded: one line per kind */
+ for(int k=0;k<8;k++) CHECK(count_sub(buf, kind_str[k]) == 1);
+ CHECK(count_sub(buf, "40/40 layer") == 5); /* kv_b + q_a/q_b/kv_a/o: all layers */
+ CHECK(count_sub(buf, "40/40 sparse layer") == 3); /* sh_*: all 40 layers are sparse */
+ free(m.L);
+ }
+
+ /* (d) all-pass silence: every fused-bound tensor on the allowlist -- across all
+ * four admitted formats (1/2/3/4, incl. fmt=3) -- and kv_b at fmt=2 -> no output. */
+ {
+ Model m = make_model(3);
+ set_clean(&m);
+ g_metal_enabled = 1;
+ run_notice(&m, buf, sizeof buf);
+ CHECK(buf[0] == 0);
+ free(m.L);
+ }
+
+ /* (e) Metal not enabled -> no notice even with everything bad (the fused paths this
+ * warns about are Metal-only, so with Metal off there is nothing to warn about) */
{
Model m = make_model(2);
- m.L[0].kv_b.fmt = 1; m.L[1].kv_b.fmt = 1;
+ set_clean(&m);
+ for(int i=0;i<2;i++){
+ m.L[i].kv_b.fmt = 1;
+ for(int k=1;k<8;k++) set_kind_fmt(&m.L[i], k, 8);
+ }
g_metal_enabled = 0;
+ run_notice(&m, buf, sizeof buf);
+ CHECK(buf[0] == 0);
+ free(m.L);
+ }
- int saved = redirect_stderr("tests/tmp_kvb_notice_c.stderr");
- kvb_fmt_gate_notice(&m);
- restore_stderr(saved, buf, sizeof buf);
- remove("tests/tmp_kvb_notice_c.stderr");
-
+ /* (f) MTP-head exclusion: main layers clean, the MTP head (m->mtpL, ships INT8 by
+ * design) entirely off-allowlist -> still silent. The fused gates never see the
+ * MTP head and neither must the notice. */
+ {
+ Model m = make_model(3);
+ set_clean(&m);
+ m.has_mtp = 1;
+ m.mtpL.sparse = 1;
+ m.mtpL.kv_b.fmt = 1;
+ for(int k=1;k<8;k++) set_kind_fmt(&m.mtpL, k, 8);
+ g_metal_enabled = 1;
+ run_notice(&m, buf, sizeof buf);
CHECK(buf[0] == 0);
free(m.L);
}
- /* (d) MTP-exclusion: the notice scans m->L[0..n_layers) only. A bad format planted
- * on m->mtpL (the separate MTP-head Layer, never part of m->L) must NOT be counted or
- * reported, even with every real layer at a valid format and Metal enabled -- the
- * fused layer-decode gate itself already excludes the MTP head (li<n_layers), and
- * common containers ship it at INT8 by design, so warning about it here would be both
- * wrong (this notice's gate doesn't apply to it) and noise on every ordinary load. */
+ /* (g) dense-layer sh_* exemption: dense layers never load the shared-expert MLP
+ * (sh_* fmt stays 0) and never reach the fused layer CB, so their sh_* must NOT
+ * be reported -- a real GLM container has dense layer(s) and would false-positive
+ * on every load otherwise. */
{
Model m = make_model(3);
- for(int i=0;i<3;i++) m.L[i].kv_b.fmt = 2;
- m.mtpL.kv_b.fmt = 1; /* bad format, but on the excluded MTP head */
+ set_clean(&m);
+ m.L[0].sparse = 0;
+ m.L[0].sh_gate.fmt = 0; m.L[0].sh_up.fmt = 0; m.L[0].sh_down.fmt = 0;
g_metal_enabled = 1;
+ run_notice(&m, buf, sizeof buf);
+ CHECK(buf[0] == 0);
+ free(m.L);
+ }
- int saved = redirect_stderr("tests/tmp_kvb_notice_d.stderr");
- kvb_fmt_gate_notice(&m);
- restore_stderr(saved, buf, sizeof buf);
- remove("tests/tmp_kvb_notice_d.stderr");
+ /* (h) shared-predicate truth table: the gates consume metal_fused_layer_fmt_miss
+ * directly, so pin its bit semantics (this is what "the notice cannot drift from
+ * the gates" rests on) -- including the kv_b mode term in BOTH g_moe_exact states. */
+ {
+ Layer l; memset(&l,0,sizeof l);
+ l.sparse=1;
+ l.kv_b.fmt=2; l.q_a.fmt=1; l.q_b.fmt=2; l.kv_a.fmt=3; l.o.fmt=4;
+ l.sh_gate.fmt=1; l.sh_up.fmt=1; l.sh_down.fmt=1;
+ CHECK(metal_fused_layer_fmt_miss(&l) == 0); /* fully eligible */
+ l.kv_b.fmt=4; /* grouped int4: */
+ CHECK(metal_fused_layer_fmt_miss(&l) == 0); /* served while MOE-exact off */
+ g_moe_exact = 1;
+ CHECK(metal_fused_layer_fmt_miss(&l) == METAL_FUSED_KV_B); /* MOE-exact: fmt==2 exactly */
+ l.kv_b.fmt=2;
+ CHECK(metal_fused_layer_fmt_miss(&l) == 0); /* fmt=2 serves in both modes */
+ g_moe_exact = 0;
+ l.kv_b.fmt=1;
+ CHECK(metal_fused_layer_fmt_miss(&l) == METAL_FUSED_KV_B); /* non-int4 misses in both modes */
+ l.kv_b.fmt=2; l.q_a.fmt=8;
+ CHECK(metal_fused_layer_fmt_miss(&l) == METAL_FUSED_Q_A);
+ l.q_a.fmt=1; l.o.fmt=0;
+ CHECK(metal_fused_layer_fmt_miss(&l) == METAL_FUSED_O); /* fmt=0 never fused */
+ l.o.fmt=4; l.sh_down.fmt=5;
+ CHECK(metal_fused_layer_fmt_miss(&l) == METAL_FUSED_SH_DOWN);
+ CHECK(!(metal_fused_layer_fmt_miss(&l) & METAL_FUSED_ATTN_TENSORS)); /* attn mask blind to sh_* */
+ l.sparse=0;
+ CHECK(metal_fused_layer_fmt_miss(&l) == 0); /* dense: sh_* exempt */
+ }
- CHECK(buf[0] == 0);
+ /* (i) mixed dense/sparse, GLM shape: 61 layers, layer 0 dense, everything bad on
+ * every layer it can be bad on -> still exactly 8 bounded lines; attn-bound kinds
+ * count over ALL 61 layers, sh_* kinds over the 60-layer sparse population only
+ * (a 61/61 there would claim a miss on a layer that never loads the tensor). */
+ {
+ Model m = make_model(61);
+ set_clean(&m);
+ m.L[0].sparse = 0;
+ m.L[0].sh_gate.fmt = 0; m.L[0].sh_up.fmt = 0; m.L[0].sh_down.fmt = 0;
+ for(int i=0;i<61;i++){
+ m.L[i].kv_b.fmt = 1;
+ for(int k=1;k<5;k++) set_kind_fmt(&m.L[i], k, 8);
+ if(m.L[i].sparse) for(int k=5;k<8;k++) set_kind_fmt(&m.L[i], k, 8);
+ }
+ g_metal_enabled = 1;
+ run_notice(&m, buf, sizeof buf);
+
+ CHECK(count_sub(buf, "[METAL]") == 8); /* bounded on the mixed shape too */
+ CHECK(count_sub(buf, "61/61 layer") == 5); /* kv_b + q_a/q_b/kv_a/o: all layers */
+ CHECK(count_sub(buf, "60/60 sparse layer") == 3); /* sh_*: sparse population only */
free(m.L);
}
- if(fails){ printf("kvb fmt-gate notice tests: %d FAILED\n", fails); return 1; }
- printf("kvb fmt-gate notice tests: ok\n");
+ /* (j) mask BIT COMPOSITION: the truth table in (h) proves the helper's bits, but
+ * not which bits each GATE mask contains -- a mask silently dropping a member
+ * (e.g. METAL_FUSED_O falling out of METAL_FUSED_ATTN_TENSORS) would fail open on
+ * the fused path with every other test still green. Assert, per kind, the masked
+ * gate booleans the gates actually compute against membership stated independently
+ * of the enum definitions: attn mask = kv_b+q_a+q_b+kv_a+o and NOT sh_*; layer
+ * mask = all 8. */
+ {
+ static const int in_attn[8] = {1,1,1,1,1,0,0,0};
+ static const int in_layer[8] = {1,1,1,1,1,1,1,1};
+ for(int k=0;k<8;k++){
+ Layer l; memset(&l,0,sizeof l);
+ l.sparse=1;
+ l.kv_b.fmt=2; l.q_a.fmt=1; l.q_b.fmt=2; l.kv_a.fmt=3; l.o.fmt=4;
+ l.sh_gate.fmt=1; l.sh_up.fmt=1; l.sh_down.fmt=1;
+ if(k==0) l.kv_b.fmt=1; else set_kind_fmt(&l, k, 8); /* exactly kind k misses */
+ unsigned miss = metal_fused_layer_fmt_miss(&l);
+ int attn_pass = !(miss & METAL_FUSED_ATTN_TENSORS); /* the gates' own tests */
+ int layer_pass = !(miss & METAL_FUSED_LAYER_TENSORS);
+ if(attn_pass != !in_attn[k] || layer_pass != !in_layer[k])
+ printf("FAIL mask membership: kind %s attn_pass=%d layer_pass=%d\n",
+ kind_str[k], attn_pass, layer_pass);
+ CHECK(attn_pass == !in_attn[k]);
+ CHECK(layer_pass == !in_layer[k]);
+ }
+ /* the kv_b MODE miss flows through the same masks: fmt=4 + MOE-exact closes
+ * both masked gates, exactly like the fused gates themselves. */
+ {
+ Layer l; memset(&l,0,sizeof l);
+ l.sparse=1;
+ l.kv_b.fmt=4; l.q_a.fmt=1; l.q_b.fmt=2; l.kv_a.fmt=3; l.o.fmt=4;
+ l.sh_gate.fmt=1; l.sh_up.fmt=1; l.sh_down.fmt=1;
+ g_moe_exact = 1;
+ unsigned miss = metal_fused_layer_fmt_miss(&l);
+ CHECK((miss & METAL_FUSED_ATTN_TENSORS) != 0);
+ CHECK((miss & METAL_FUSED_LAYER_TENSORS) != 0);
+ g_moe_exact = 0;
+ miss = metal_fused_layer_fmt_miss(&l);
+ CHECK(!(miss & METAL_FUSED_ATTN_TENSORS));
+ CHECK(!(miss & METAL_FUSED_LAYER_TENSORS));
+ }
+ /* and a fully clean layer passes both masked gates */
+ Layer l; memset(&l,0,sizeof l);
+ l.sparse=1;
+ l.kv_b.fmt=2; l.q_a.fmt=1; l.q_b.fmt=2; l.kv_a.fmt=3; l.o.fmt=4;
+ l.sh_gate.fmt=1; l.sh_up.fmt=1; l.sh_down.fmt=1;
+ unsigned miss = metal_fused_layer_fmt_miss(&l);
+ CHECK(!(miss & METAL_FUSED_ATTN_TENSORS));
+ CHECK(!(miss & METAL_FUSED_LAYER_TENSORS));
+ }
+
+ if(fails){ printf("fused-metal fmt-gate notice tests: %d FAILED\n", fails); return 1; }
+ printf("fused-metal fmt-gate notice tests: ok\n");
return 0;
}
diff --git a/docs/FORMATS.md b/docs/FORMATS.md
index 51ac48b0..8cb4b48c 100644
--- a/docs/FORMATS.md
+++ b/docs/FORMATS.md
@@ -84,6 +84,22 @@ this format moved to 8). Before picking an ordinal β€” or relying on one β€”
proposers must scan both dev and the open PRs. This registry is the
coordination point: a row lands here when the format lands in dev.
+**Format consumers: the fused Metal decode path.** Beyond *decoding*, some
+engine paths place format *requirements* on specific tensors. The fused Metal
+decode kernels engage per layer only when `kv_b_proj` satisfies the gates'
+two-format+mode term β€” fmt=2 (`int4-row`) always, fmt=4 (`int4-grouped`) only
+while `COLI_METAL_MOE_EXACT` is off (MOE-exact mode keeps grouped kv_b off the
+fused path) β€” AND the other fused-bound tensors (`q_a`, `q_b`, `kv_a`, `o`,
+and on sparse layers `sh_gate`/`sh_up`/`sh_down`) sit on the fmt 1/2/3/4
+allowlist; any other format on any of those tensors makes the affected layers'
+decode take the CPU path instead, announced by a one-line-per-tensor-kind
+`[METAL]` stderr notice at load. Single source of truth (all anchors
+`c/colibri.c` at branch head `kvb/fmt-gate-notice-r4`): the shared per-layer
+predicate `metal_fused_layer_fmt_miss` (:3366, over the `metal_fused_fmt_ok`
+allowlist, :3349), consulted by both gate sites β€” `attention_rows` (:3418) and
+`layer_forward_rows` (:5738) β€” and by the load-time notice
+`metal_fmt_gate_notice` (:1848, called from `model_init`).
+
Sources for all rows (`c/quant.h`/`c/colibri.c` line numbers at this PR
pair's current restack, base dev `292ed4c`):