Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
moe: gs-consistency guard for fmt=4 GPU batches (review F2)
monotophic95fdfda2ca57
diff --git a/c/Makefile b/c/Makefileindex 97bf41ea..c7ae0588 100644--- a/c/Makefile+++ b/c/Makefile@@ -961,6 +961,8 @@ tests/test_pipe_block$(EXE): tests/test_pipe_block.c colibri.c st.h uring.h jsontests/test_pilot_ring$(EXE): tests/test_pilot_ring.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_moe_gs_guard$(EXE): tests/test_moe_gs_guard.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)tests/test_omp_tune$(EXE): tests/test_omp_tune.c omp_tune.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)diff --git a/c/colibri.c b/c/colibri.cindex 1c402e15..83a53848 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -3823,6 +3823,18 @@ static void metal_stage_rot_e8(float *mxg, const int *mrows, int p, int D){}}#endif+/* moe()'s MB_BUILD subset builder (review F2): coli_metal_moe_block(_begin) take ONE+ * fmt/qgs for an entire GPU batch of experts (+ optionally the fused shared expert), so+ * a candidate member at (fmt,gs) is only safe to fold into a batch already established+ * at (est_fmt,est_gs) if it can't disagree about layout: fmt=4 (grouped int4) is the+ * only format with a group size to disagree ON, so this is a no-op (always compatible)+ * for est_fmt!=4 (nothing established yet, or the batch isn't grouped) or fmt!=4 (the+ * candidate itself isn't grouped -- its own fmt mismatch is a separate, pre-existing gap+ * this guard does not cover, see MB_BUILD's comment). Pulled out of MB_BUILD as its own+ * function so it's independently testable (see tests/test_moe_gs_guard.c). */+static int mb_gs_compat(int est_fmt, int est_gs, int fmt, int gs){+ return !(est_fmt==4 && fmt==4 && gs!=est_gs);+}static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int with_shared){if(g_pilot_real){ /* barriera cross-layer: prendi possesso di QUESTO layer e aspetta@@ -4253,19 +4265,37 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int* batch's rows all share s, so this is one FWHT per token in practice. The down* input is rotated on-GPU by moe_fwht inside moe_submit. */int is_miss[64]={0}; ColiMetalMoeHandle *mh=NULL;- int cpu_res=1, cpu_miss=1, mh_shared=0, nbb=0, Rtot=0, mfmt=-1, mgs=0, sh_in=0;+ int cpu_res=1, cpu_miss=1, mh_shared=0, nbb=0, Rtot=0, mfmt=-1, mgs=0, mgs_ok=1, sh_in=0;const void *MG[65],*MU[65],*MD[65]; const float *MGS[65],*MUS[65],*MDS[65];int xoffb[65],nrb[65];float *mxg=NULL; int *mrows=NULL; float *mrw=NULL;- /* subset builder: experts with is_miss==WANTMISS (+ shared expert when TRY_SH) */+ /* subset builder: experts with is_miss==WANTMISS (+ shared expert when TRY_SH).+ * mgs_ok (review F2): moe_submit takes ONE fmt/qgs for the WHOLE batch, so a+ * fmt=4 subset needs every member's group size to agree with the first expert's+ * (mgs) -- fmt equality alone (already checked below for the shared expert) isn't+ * enough once fmt=4 has a group size to disagree on. Routed experts get the same+ * guard against each other (first-expert-gs consistency): a v1-class mixed-+ * precision container could in principle mint routed experts at different gs+ * within one layer (qt_resolve_fmt derives fmt/gs per-tensor from the file, with+ * no uniformity enforced across experts at load time), even though a normal+ * single-pass conversion never would. mgs_ok=0 does NOT drop the mismatched+ * expert's rows: it leaves nbb's bookkeeping untouched and instead suppresses the+ * GPU submit call at the two call sites below, which leaves cpu_res/cpu_miss at+ * their initial 1 -- the same state a genuine GPU submission failure leaves them+ * in, so the existing CPU fallback loop (metal_done false) redoes this whole+ * nb-expert block correctly regardless of the mismatch. (Not checked, and out of+ * scope for this guard: per-expert fmt agreement across g/u/d, and gs agreement+ * across sh_gate/sh_up/sh_down or across a single expert's own g/u/d -- see+ * WORKER_REPORT UNCERTAINTIES.) */#define MB_BUILD(WANTMISS, TRY_SH) do{ \- nbb=0; Rtot=0; mfmt=-1; mgs=0; sh_in=0; \+ nbb=0; Rtot=0; mfmt=-1; mgs=0; mgs_ok=1; sh_in=0; \for(int j=0;j<nb;j++){ if(is_miss[j]!=(WANTMISS)) continue; \int eid=uniq[base+j]; ESlot *e=use[j]; int cnt=0; \for(int s=0;s<S;s++) for(int kk=0;kk<keff[s];kk++) \if(idxs[(int64_t)s*K+kk]==eid){ cnt++; break; } \if(!cnt) continue; \if(mfmt<0){ mfmt=e->g.fmt; mgs=e->g.gs; } \+ else if(!mb_gs_compat(mfmt,mgs,e->g.fmt,e->g.gs)) mgs_ok=0; \MG[nbb]=e->g.fmt==1?(const void*)e->g.q8:(const void*)e->g.q4; \MU[nbb]=e->u.fmt==1?(const void*)e->u.q8:(const void*)e->u.q4; \MD[nbb]=e->d.fmt==1?(const void*)e->d.q8:(const void*)e->d.q4; \@@ -4273,7 +4303,8 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intxoffb[nbb]=Rtot; nrb[nbb]=cnt; Rtot+=cnt; nbb++; \} \if(TRY_SH){ int shf = mfmt<0 ? l->sh_gate.fmt : mfmt; \- if(c->n_shared==1 && sI==I && l->sh_gate.fmt==shf && l->sh_up.fmt==shf && l->sh_down.fmt==shf){ \+ if(c->n_shared==1 && sI==I && l->sh_gate.fmt==shf && l->sh_up.fmt==shf && l->sh_down.fmt==shf \+ && mb_gs_compat(mfmt,mgs,shf,l->sh_gate.gs)){ \if(mfmt<0){ mfmt=shf; mgs=l->sh_gate.gs; } \MG[nbb]=shf==1?(const void*)l->sh_gate.q8:(const void*)l->sh_gate.q4; \MU[nbb]=shf==1?(const void*)l->sh_up.q8 :(const void*)l->sh_up.q4; \@@ -4295,13 +4326,15 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intmxg=falloc((int64_t)(nb+1)*S*D);mrows=xalloc((size_t)(nb+1)*S*sizeof(int),"moe mrows"); mrw=xalloc((size_t)(nb+1)*S*sizeof(float),"moe mrw");MB_BUILD(0, base==0 && !g_pre_sh);- if(nbb>0){+ if(nbb>0 && mgs_ok){double t0=now_s();if(mfmt==6) metal_stage_rot_e8(mxg,mrows,Rtot,D);mh=coli_metal_moe_block_begin(nbb,D,I,mfmt,mgs,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw);m->t_emm += now_s()-t0;if(mh){ cpu_res=0; mh_shared=sh_in; }- } else cpu_res=0;+ } else if(!nbb) cpu_res=0; /* nbb==0: nothing in this subset. nbb>0 && !mgs_ok+ * (F2): gs-heterogeneous fmt=4 subset -- leave+ * cpu_res=1 so the CPU loop below redoes it. */}#endif/* Expert loads run HERE, after the resident-experts GPU submit above: under METAL the@@ -4358,12 +4391,12 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intfor(int q=0;q<nmiss;q++) pipe_wait(q);m->t_ewait += now_s()-tw; }MB_BUILD(1, 0); /* missed experts, now loaded */- if(nbb>0){+ if(nbb>0 && mgs_ok){double t0=now_s();if(mfmt==6) metal_stage_rot_e8(mxg,mrows,Rtot,D);if(coli_metal_moe_block(nbb,D,I,mfmt,mgs,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw,out,S)) cpu_miss=0;m->t_emm += now_s()-t0;- } else cpu_miss=0;+ } else if(!nbb) cpu_miss=0; /* see the resident-subset call site above */if(mh){ double t0=now_s();if(coli_metal_moe_block_end(mh,out)){ if(mh_shared) shared_on_gpu=1; }else cpu_res=1;diff --git a/c/tests/test_moe_gs_guard.c b/c/tests/test_moe_gs_guard.cnew file mode 100644index 00000000..6b4fd137--- /dev/null+++ b/c/tests/test_moe_gs_guard.c@@ -0,0 +1,63 @@+/* Regression for moe()'s MB_BUILD subset-consistency guard (review finding F2).+ *+ * The trap: coli_metal_moe_block(_begin) takes ONE fmt/qgs for an entire GPU batch of+ * experts (+ optionally the fused shared expert). MB_BUILD already refused to fold the+ * shared expert into a batch whose fmt disagreed (`l->sh_gate.fmt==shf`), but for fmt=4+ * (grouped int4) format equality alone isn't enough: two fmt=4 tensors can still disagree+ * on GROUP SIZE, and moe_submit has no per-member gs -- every expert in the batch is+ * dequantized with the SAME qgs, so a silently-included gs mismatch would corrupt every+ * row belonging to the odd one out. mb_gs_compat() is the guard: it decides whether a+ * candidate (fmt,gs) is safe to fold into a batch already established at (est_fmt,est_gs).+ *+ * This test exercises mb_gs_compat() directly rather than moe() end-to-end: MB_BUILD is a+ * macro #define'd (and #undef'd) entirely inside moe()'s own function body, so it isn't+ * reachable from outside moe() even via the "#include colibri.c" seam this file uses --+ * and moe() itself needs a fully-populated Model/Layer/ESlot/ecache/router-output+ * apparatus no existing test constructs, which would make a true end-to-end harness+ * heavy and fragile relative to what it proves. mb_gs_compat() is not a re-implementation+ * of MB_BUILD's guard logic -- it IS that logic (MB_BUILD calls it directly, at both the+ * routed-expert and the shared-expert call sites) -- so exercising it directly is a+ * faithful, low-risk proxy for the integration behavior. See WORKER_REPORT_integ_postwave.md+ * UNCERTAINTIES for the case against the heavier alternative.+ *+ * Also documents, by assertion (not just prose), the two gaps this guard deliberately+ * does NOT cover: per-expert fmt heterogeneity (g vs g of a different expert), and+ * per-tensor-role gs heterogeneity within one expert/shared-expert triple (g vs u vs d).+ * Both are pre-existing, out of scope for F2 -- see the guard's own comment in colibri.c. */+#define main coli_glm_main_unused+#include "../colibri.c"+#undef main++#include <assert.h>++int main(void){+ /* --- nothing established yet (est_fmt<0): always compatible, any (fmt,gs) --- */+ assert(mb_gs_compat(-1,0, 4,128)==1 && "first member of a subset is always accepted");+ assert(mb_gs_compat(-1,0, 2,0)==1 && "first member, non-grouped fmt");++ /* --- established fmt=4: SAME gs is compatible (the common, correct case) --- */+ assert(mb_gs_compat(4,128, 4,128)==1 && "matching gs within a fmt=4 subset: ok");+ assert(mb_gs_compat(4,64, 4,64)==1 && "matching gs (different value): ok");++ /* --- established fmt=4: DIFFERENT gs is the F2 trap -- must be rejected --- */+ assert(mb_gs_compat(4,128, 4,64)==0 && "gs mismatch within a fmt=4 subset: must reject");+ assert(mb_gs_compat(4,64, 4,128)==0 && "gs mismatch, reversed order: must reject");+ assert(mb_gs_compat(4,128, 4,127)==0 && "off-by-one gs mismatch: must reject (no slack)");++ /* --- established fmt!=4 (no group size to disagree on): always compatible,+ * REGARDLESS of what the candidate claims -- fmt heterogeneity itself is a+ * separate, pre-existing gap this guard does not cover (documented, not fixed:+ * MB_BUILD has never checked per-expert fmt agreement, fmt=4 or otherwise). --- */+ assert(mb_gs_compat(2,0, 4,999)==1 && "est fmt=2 (ungrouped): gs never checked");+ assert(mb_gs_compat(1,0, 4,999)==1 && "est fmt=1 (int8): gs never checked");++ /* --- candidate fmt!=4 against an established fmt=4 batch: also out of scope+ * (the candidate's OWN fmt mismatch is what would corrupt it, not gs -- and+ * fmt agreement for the shared expert is still checked separately by MB_BUILD's+ * TRY_SH condition before mb_gs_compat is ever consulted). --- */+ assert(mb_gs_compat(4,128, 2,0)==1 && "candidate fmt=2 against fmt=4 batch: gs moot");++ printf("OK test_moe_gs_guard: mb_gs_compat (review F2) accepts matching gs, "+ "rejects mismatched gs, documents its fmt-heterogeneity non-scope\n");+ return 0;+}