Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

moe: gs-consistency guard for fmt=4 GPU batches (review F2)

monotophic95fdfda2ca57
diff --git a/c/Makefile b/c/Makefile
index 97bf41ea..c7ae0588 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -961,6 +961,8 @@ tests/test_pipe_block$(EXE): tests/test_pipe_block.c colibri.c st.h uring.h json
tests/test_pilot_ring$(EXE): tests/test_pilot_ring.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+tests/test_moe_gs_guard$(EXE): tests/test_moe_gs_guard.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_omp_tune$(EXE): tests/test_omp_tune.c omp_tune.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
diff --git a/c/colibri.c b/c/colibri.c
index 1c402e15..83a53848 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -3823,6 +3823,18 @@ static void metal_stage_rot_e8(float *mxg, const int *mrows, int p, int D){
}
}
#endif
+/* moe()'s MB_BUILD subset builder (review F2): coli_metal_moe_block(_begin) take ONE
+ * fmt/qgs for an entire GPU batch of experts (+ optionally the fused shared expert), so
+ * a candidate member at (fmt,gs) is only safe to fold into a batch already established
+ * at (est_fmt,est_gs) if it can't disagree about layout: fmt=4 (grouped int4) is the
+ * only format with a group size to disagree ON, so this is a no-op (always compatible)
+ * for est_fmt!=4 (nothing established yet, or the batch isn't grouped) or fmt!=4 (the
+ * candidate itself isn't grouped -- its own fmt mismatch is a separate, pre-existing gap
+ * this guard does not cover, see MB_BUILD's comment). Pulled out of MB_BUILD as its own
+ * function so it's independently testable (see tests/test_moe_gs_guard.c). */
+static int mb_gs_compat(int est_fmt, int est_gs, int fmt, int gs){
+ return !(est_fmt==4 && fmt==4 && gs!=est_gs);
+}
static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int with_shared){
if(g_pilot_real){ /* barriera cross-layer: prendi possesso di QUESTO layer e aspetta
@@ -4253,19 +4265,37 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
* batch's rows all share s, so this is one FWHT per token in practice. The down
* input is rotated on-GPU by moe_fwht inside moe_submit. */
int is_miss[64]={0}; ColiMetalMoeHandle *mh=NULL;
- int cpu_res=1, cpu_miss=1, mh_shared=0, nbb=0, Rtot=0, mfmt=-1, mgs=0, sh_in=0;
+ int cpu_res=1, cpu_miss=1, mh_shared=0, nbb=0, Rtot=0, mfmt=-1, mgs=0, mgs_ok=1, sh_in=0;
const void *MG[65],*MU[65],*MD[65]; const float *MGS[65],*MUS[65],*MDS[65];
int xoffb[65],nrb[65];
float *mxg=NULL; int *mrows=NULL; float *mrw=NULL;
- /* subset builder: experts with is_miss==WANTMISS (+ shared expert when TRY_SH) */
+ /* subset builder: experts with is_miss==WANTMISS (+ shared expert when TRY_SH).
+ * mgs_ok (review F2): moe_submit takes ONE fmt/qgs for the WHOLE batch, so a
+ * fmt=4 subset needs every member's group size to agree with the first expert's
+ * (mgs) -- fmt equality alone (already checked below for the shared expert) isn't
+ * enough once fmt=4 has a group size to disagree on. Routed experts get the same
+ * guard against each other (first-expert-gs consistency): a v1-class mixed-
+ * precision container could in principle mint routed experts at different gs
+ * within one layer (qt_resolve_fmt derives fmt/gs per-tensor from the file, with
+ * no uniformity enforced across experts at load time), even though a normal
+ * single-pass conversion never would. mgs_ok=0 does NOT drop the mismatched
+ * expert's rows: it leaves nbb's bookkeeping untouched and instead suppresses the
+ * GPU submit call at the two call sites below, which leaves cpu_res/cpu_miss at
+ * their initial 1 -- the same state a genuine GPU submission failure leaves them
+ * in, so the existing CPU fallback loop (metal_done false) redoes this whole
+ * nb-expert block correctly regardless of the mismatch. (Not checked, and out of
+ * scope for this guard: per-expert fmt agreement across g/u/d, and gs agreement
+ * across sh_gate/sh_up/sh_down or across a single expert's own g/u/d -- see
+ * WORKER_REPORT UNCERTAINTIES.) */
#define MB_BUILD(WANTMISS, TRY_SH) do{ \
- nbb=0; Rtot=0; mfmt=-1; mgs=0; sh_in=0; \
+ nbb=0; Rtot=0; mfmt=-1; mgs=0; mgs_ok=1; sh_in=0; \
for(int j=0;j<nb;j++){ if(is_miss[j]!=(WANTMISS)) continue; \
int eid=uniq[base+j]; ESlot *e=use[j]; int cnt=0; \
for(int s=0;s<S;s++) for(int kk=0;kk<keff[s];kk++) \
if(idxs[(int64_t)s*K+kk]==eid){ cnt++; break; } \
if(!cnt) continue; \
if(mfmt<0){ mfmt=e->g.fmt; mgs=e->g.gs; } \
+ else if(!mb_gs_compat(mfmt,mgs,e->g.fmt,e->g.gs)) mgs_ok=0; \
MG[nbb]=e->g.fmt==1?(const void*)e->g.q8:(const void*)e->g.q4; \
MU[nbb]=e->u.fmt==1?(const void*)e->u.q8:(const void*)e->u.q4; \
MD[nbb]=e->d.fmt==1?(const void*)e->d.q8:(const void*)e->d.q4; \
@@ -4273,7 +4303,8 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
xoffb[nbb]=Rtot; nrb[nbb]=cnt; Rtot+=cnt; nbb++; \
} \
if(TRY_SH){ int shf = mfmt<0 ? l->sh_gate.fmt : mfmt; \
- if(c->n_shared==1 && sI==I && l->sh_gate.fmt==shf && l->sh_up.fmt==shf && l->sh_down.fmt==shf){ \
+ if(c->n_shared==1 && sI==I && l->sh_gate.fmt==shf && l->sh_up.fmt==shf && l->sh_down.fmt==shf \
+ && mb_gs_compat(mfmt,mgs,shf,l->sh_gate.gs)){ \
if(mfmt<0){ mfmt=shf; mgs=l->sh_gate.gs; } \
MG[nbb]=shf==1?(const void*)l->sh_gate.q8:(const void*)l->sh_gate.q4; \
MU[nbb]=shf==1?(const void*)l->sh_up.q8 :(const void*)l->sh_up.q4; \
@@ -4295,13 +4326,15 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
mxg=falloc((int64_t)(nb+1)*S*D);
mrows=xalloc((size_t)(nb+1)*S*sizeof(int),"moe mrows"); mrw=xalloc((size_t)(nb+1)*S*sizeof(float),"moe mrw");
MB_BUILD(0, base==0 && !g_pre_sh);
- if(nbb>0){
+ if(nbb>0 && mgs_ok){
double t0=now_s();
if(mfmt==6) metal_stage_rot_e8(mxg,mrows,Rtot,D);
mh=coli_metal_moe_block_begin(nbb,D,I,mfmt,mgs,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw);
m->t_emm += now_s()-t0;
if(mh){ cpu_res=0; mh_shared=sh_in; }
- } else cpu_res=0;
+ } else if(!nbb) cpu_res=0; /* nbb==0: nothing in this subset. nbb>0 && !mgs_ok
+ * (F2): gs-heterogeneous fmt=4 subset -- leave
+ * cpu_res=1 so the CPU loop below redoes it. */
}
#endif
/* Expert loads run HERE, after the resident-experts GPU submit above: under METAL the
@@ -4358,12 +4391,12 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
for(int q=0;q<nmiss;q++) pipe_wait(q);
m->t_ewait += now_s()-tw; }
MB_BUILD(1, 0); /* missed experts, now loaded */
- if(nbb>0){
+ if(nbb>0 && mgs_ok){
double t0=now_s();
if(mfmt==6) metal_stage_rot_e8(mxg,mrows,Rtot,D);
if(coli_metal_moe_block(nbb,D,I,mfmt,mgs,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw,out,S)) cpu_miss=0;
m->t_emm += now_s()-t0;
- } else cpu_miss=0;
+ } else if(!nbb) cpu_miss=0; /* see the resident-subset call site above */
if(mh){ double t0=now_s();
if(coli_metal_moe_block_end(mh,out)){ if(mh_shared) shared_on_gpu=1; }
else cpu_res=1;
diff --git a/c/tests/test_moe_gs_guard.c b/c/tests/test_moe_gs_guard.c
new file mode 100644
index 00000000..6b4fd137
--- /dev/null
+++ b/c/tests/test_moe_gs_guard.c
@@ -0,0 +1,63 @@
+/* Regression for moe()'s MB_BUILD subset-consistency guard (review finding F2).
+ *
+ * The trap: coli_metal_moe_block(_begin) takes ONE fmt/qgs for an entire GPU batch of
+ * experts (+ optionally the fused shared expert). MB_BUILD already refused to fold the
+ * shared expert into a batch whose fmt disagreed (`l->sh_gate.fmt==shf`), but for fmt=4
+ * (grouped int4) format equality alone isn't enough: two fmt=4 tensors can still disagree
+ * on GROUP SIZE, and moe_submit has no per-member gs -- every expert in the batch is
+ * dequantized with the SAME qgs, so a silently-included gs mismatch would corrupt every
+ * row belonging to the odd one out. mb_gs_compat() is the guard: it decides whether a
+ * candidate (fmt,gs) is safe to fold into a batch already established at (est_fmt,est_gs).
+ *
+ * This test exercises mb_gs_compat() directly rather than moe() end-to-end: MB_BUILD is a
+ * macro #define'd (and #undef'd) entirely inside moe()'s own function body, so it isn't
+ * reachable from outside moe() even via the "#include colibri.c" seam this file uses --
+ * and moe() itself needs a fully-populated Model/Layer/ESlot/ecache/router-output
+ * apparatus no existing test constructs, which would make a true end-to-end harness
+ * heavy and fragile relative to what it proves. mb_gs_compat() is not a re-implementation
+ * of MB_BUILD's guard logic -- it IS that logic (MB_BUILD calls it directly, at both the
+ * routed-expert and the shared-expert call sites) -- so exercising it directly is a
+ * faithful, low-risk proxy for the integration behavior. See WORKER_REPORT_integ_postwave.md
+ * UNCERTAINTIES for the case against the heavier alternative.
+ *
+ * Also documents, by assertion (not just prose), the two gaps this guard deliberately
+ * does NOT cover: per-expert fmt heterogeneity (g vs g of a different expert), and
+ * per-tensor-role gs heterogeneity within one expert/shared-expert triple (g vs u vs d).
+ * Both are pre-existing, out of scope for F2 -- see the guard's own comment in colibri.c. */
+#define main coli_glm_main_unused
+#include "../colibri.c"
+#undef main
+
+#include <assert.h>
+
+int main(void){
+ /* --- nothing established yet (est_fmt<0): always compatible, any (fmt,gs) --- */
+ assert(mb_gs_compat(-1,0, 4,128)==1 && "first member of a subset is always accepted");
+ assert(mb_gs_compat(-1,0, 2,0)==1 && "first member, non-grouped fmt");
+
+ /* --- established fmt=4: SAME gs is compatible (the common, correct case) --- */
+ assert(mb_gs_compat(4,128, 4,128)==1 && "matching gs within a fmt=4 subset: ok");
+ assert(mb_gs_compat(4,64, 4,64)==1 && "matching gs (different value): ok");
+
+ /* --- established fmt=4: DIFFERENT gs is the F2 trap -- must be rejected --- */
+ assert(mb_gs_compat(4,128, 4,64)==0 && "gs mismatch within a fmt=4 subset: must reject");
+ assert(mb_gs_compat(4,64, 4,128)==0 && "gs mismatch, reversed order: must reject");
+ assert(mb_gs_compat(4,128, 4,127)==0 && "off-by-one gs mismatch: must reject (no slack)");
+
+ /* --- established fmt!=4 (no group size to disagree on): always compatible,
+ * REGARDLESS of what the candidate claims -- fmt heterogeneity itself is a
+ * separate, pre-existing gap this guard does not cover (documented, not fixed:
+ * MB_BUILD has never checked per-expert fmt agreement, fmt=4 or otherwise). --- */
+ assert(mb_gs_compat(2,0, 4,999)==1 && "est fmt=2 (ungrouped): gs never checked");
+ assert(mb_gs_compat(1,0, 4,999)==1 && "est fmt=1 (int8): gs never checked");
+
+ /* --- candidate fmt!=4 against an established fmt=4 batch: also out of scope
+ * (the candidate's OWN fmt mismatch is what would corrupt it, not gs -- and
+ * fmt agreement for the shared expert is still checked separately by MB_BUILD's
+ * TRY_SH condition before mb_gs_compat is ever consulted). --- */
+ assert(mb_gs_compat(4,128, 2,0)==1 && "candidate fmt=2 against fmt=4 batch: gs moot");
+
+ printf("OK test_moe_gs_guard: mb_gs_compat (review F2) accepts matching gs, "
+ "rejects mismatched gs, documents its fmt-heterogeneity non-scope\n");
+ return 0;
+}