Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
fix(degrade-zero): per-position tau gate, no renorm (issue #865)
Kritika Gargecd369c8a9e4
diff --git a/c/colibri.c b/c/colibri.cindex 85d63b0c..96b6417f 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -4145,20 +4145,17 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int}/* ---- DEGRADE_ZERO: zero-fill miss slots below the gate-weight threshold --------* When a prefetch deadline is missed, blocking on a demand-load stalls the compute- * thread. For experts whose aggregate gate weight across the batch is below- * DEGRADE_TAU, the contribution is small enough that zeroing the slot costs less- * in output quality than the I/O stall costs in latency (issue #865: tau=0.03- * zeroes 21.8% of slots for +2.9% perplexity).+ * thread. For experts whose per-position gate weight is below DEGRADE_TAU the+ * contribution is small enough that zeroing the slot costs less in output quality+ * than the I/O stall costs in latency (issue #865: tau=0.03 zeroes 21.8% of slots+ * for +2.9% perplexity). tau is compared per-position (post-norm_topk,+ * pre-routed_scale) β each position independently, matching the measured numbers.+ * No renorm: the approximation IS the dropped mass; renorm would hide it and bias+ * the output upward.* Opt-in only (DEGRADE_ZERO=1); decode-only (S<=4) for the same reason as* EXPERT_BUDGET: during prefill every dropped expert corrupts the KV cache. */if(g_degrade_zero && S<=4){- /* compute aggregate gate weight per unique expert across the batch */- float *dg_wsum=falloc(nu); for(int j=0;j<nu;j++) dg_wsum[j]=0;- for(int s=0;s<S;s++) for(int kk=0;kk<keff[s];kk++){- int e=idxs[(int64_t)s*K+kk];- for(int j=0;j<nu;j++) if(uniq[j]==e){ dg_wsum[j]+=ws[(int64_t)s*K+kk]; break; }- }- /* residency scan: only misses are eligible β cache hits are free, never drop */+ /* residency scan: hits are always kept regardless of weight */unsigned char *dg_keep=xzalloc((size_t)nu,"moe dg_keep");for(int j=0;j<nu;j++){int eid=uniq[j], resident=0;@@ -4166,8 +4163,17 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intfor(int z=0;z<m->npin[layer];z++) if(P[z].eid==eid){ resident=1; break; }if(!resident){ ESlot *Sl=m->ecache[layer]; int nn=m->ecn[layer];for(int z=0;z<nn;z++) if(Sl[z].eid==eid){ resident=1; break; } }- /* keep if: resident (hit), or gate weight meets threshold */- if(resident || dg_wsum[j]>=g_degrade_tau) dg_keep[j]=1;+ if(resident) dg_keep[j]=1;+ }+ /* per-position tau gate: a miss expert is kept if ANY position routes to it+ * with weight >= tau. Each position's weight is tested independently β this+ * is the gate the +2.9% ppl measurement was taken under. */+ for(int s=0;s<S;s++) for(int kk=0;kk<keff[s];kk++){+ float wv=ws[(int64_t)s*K+kk];+ if(wv>=g_degrade_tau){+ int e=idxs[(int64_t)s*K+kk];+ for(int j=0;j<nu;j++) if(uniq[j]==e){ dg_keep[j]=1; break; }+ }}/* rescue: no position may end up with zero routed experts.* If all of a position's experts were misses below tau, reinstate the@@ -4187,7 +4193,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intseen[be]=1;for(int j=0;j<nu;j++) if(uniq[j]==be && !dg_keep[j]){ dg_keep[j]=1; break; }}- /* apply: remove dropped experts from each position's routing list */+ /* apply: compact routing lists, no renorm β survivors keep original weights */int dg_dropped=0;for(int j=0;j<nu;j++) if(!dg_keep[j]) dg_dropped++;if(dg_dropped){@@ -4195,30 +4201,19 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intmemset(seen,0,(size_t)E);for(int j=0;j<nu;j++) if(dg_keep[j]) seen[uniq[j]]=1;for(int s=0;s<S;s++){- int w=0; float sold=0, snew=0;+ int w=0;for(int kk=0;kk<keff[s];kk++){int e=idxs[(int64_t)s*K+kk]; float wv=ws[(int64_t)s*K+kk];- sold+=wv;- if(seen[e]){ idxs[(int64_t)s*K+w]=e; ws[(int64_t)s*K+w]=wv; snew+=wv; w++; }- }- if(w<keff[s]){- keff[s]=w;- if(c->norm_topk && w>0){- float sm=0; for(int kk=0;kk<w;kk++) sm+=ws[(int64_t)s*K+kk]; sm+=1e-20f;- for(int kk=0;kk<w;kk++) ws[(int64_t)s*K+kk]/=sm;- for(int kk=0;kk<w;kk++) ws[(int64_t)s*K+kk]*=c->routed_scale;- } else if(w>0 && snew>1e-20f && sold>snew){- float sc=sold/snew;- for(int kk=0;kk<w;kk++) ws[(int64_t)s*K+kk]*=sc;- }+ if(seen[e]){ idxs[(int64_t)s*K+w]=e; ws[(int64_t)s*K+w]=wv; w++; }}+ if(w<keff[s]) keff[s]=w;}/* compact uniq[] to kept experts only */int nu2=0;for(int j=0;j<nu;j++) if(dg_keep[j]) uniq[nu2++]=uniq[j];nu=nu2;}- free(dg_wsum); free(dg_keep);+ free(dg_keep);}/* ---- FASE C/D: risolvi (pin/cache/disco) e calcola, a blocchi di 64 unici ---- */float *xg=falloc((int64_t)S*D), *gg=falloc((int64_t)S*I), *uu=falloc((int64_t)S*I), *hh=falloc((int64_t)S*D);diff --git a/c/tests/test_degrade_zero.c b/c/tests/test_degrade_zero.cindex 506e9aed..17dcc334 100644--- a/c/tests/test_degrade_zero.c+++ b/c/tests/test_degrade_zero.c@@ -5,31 +5,32 @@* No colibri.c include, no Model, no weights, no disk I/O needed.** The block under test (colibri.c, "DEGRADE_ZERO: zero-fill miss slots..."):- * 1. Computes aggregate gate weight per unique expert across the batch.- * 2. Marks cache hits as always-keep; marks cold misses below g_degrade_tau- * for zeroing.+ * 1. Marks cache hits as always-keep.+ * 2. Per-position tau gate: a miss expert is kept if ANY position routes to it+ * with weight >= tau (each position tested independently, not aggregated).* 3. Rescue rule: if all of a position's experts would be dropped, reinstates* the highest-gate-weight miss so no position has zero routed experts.- * 4. Rewrites idxs[]/ws[]/keff[] removing dropped experts, renormalises weights,- * compacts uniq[], increments g_degrade_dropped.+ * 4. Rewrites idxs[]/ws[]/keff[] removing dropped experts, NO renorm β+ * survivors keep their original weights; compacts uniq[]; increments+ * g_degrade_dropped.** Properties verified:- * P1 OFF-BY-DEFAULT β with g_degrade_zero=0, routing is byte-identical to- * the input (nothing is dropped or renormalised).- * P2 HITS-ARE-SAFE β experts already resident (simulated via a mock+ * P1 OFF-BY-DEFAULT β with g_degrade_zero=0, routing is byte-identical to+ * the input (nothing is dropped or changed).+ * P2 HITS-ARE-SAFE β experts already resident (simulated via a mock* "resident" set) are never dropped regardless of gate weight.- * P3 TAU-GATE β cold misses with aggregate gate weight >= tau are kept;+ * P3 TAU-GATE β cold misses with per-position weight >= tau are kept;* those below tau are dropped and counted in g_degrade_dropped.- * P4 RENORM-NORMTOPK β after dropping, surviving gate weights renormalise to- * sum 1 * routed_scale when norm_topk=1 (GLM-5.2 default).- * P5 RENORM-NONNORM β with norm_topk=0 the surviving weights are rescaled by- * old_sum/new_sum to preserve output magnitude.- * P6 RESCUE-RULE β when all of a position's experts are cold misses below+ * P4 NO-RENORM β surviving weights are unchanged after dropping; the+ * dropped mass is simply lost (not redistributed).+ * P5 PER-POSITION-TAU β at S>1 a shared expert is kept as long as ONE+ * position's weight meets tau; aggregate does not govern.+ * P6 RESCUE-RULE β when all of a position's experts are cold misses below* tau, the highest-weight one is reinstated; g_degrade_dropped is NOT* inflated for the rescued expert.- * P7 PREFILL-GUARD β with S>4 (prefill batch) the block is a no-op; nothing+ * P7 PREFILL-GUARD β with S>4 (prefill batch) the block is a no-op; nothing* is dropped even if g_degrade_zero=1 and all experts are below tau.- * P8 COUNTER β g_degrade_dropped accumulates correctly across calls.+ * P8 COUNTER β g_degrade_dropped accumulates correctly across calls.*/#include <stdio.h>@@ -52,8 +53,6 @@ static int g_degrade_zero = 0;static float g_degrade_tau = 0.03f;static long long g_degrade_dropped = 0;-/* ---- mirror of the Cfg fields the block reads ----------------------------- */-typedef struct { int norm_topk; float routed_scale; } Cfg;/* ---- resident-set mock: a flat array of expert ids considered "in cache" -- */#define MAX_RESIDENT 32@@ -68,28 +67,30 @@ static int is_resident(int eid) {/* ---- the drop logic extracted verbatim from colibri.c moe() --------------- ** Parameters match the local variables in moe() at the insertion point:- * idxs[S*K], ws[S*K], keff[S], uniq[nu], nu, S, K, c->norm_topk/routed_scale.+ * idxs[S*K], ws[S*K], keff[S], uniq[nu], nu, S, K.* Returns the new nu after compaction. */static int degrade_zero_apply(int *idxs, float *ws, int *keff,- int *uniq, int nu, int S, int K,- const Cfg *c)+ int *uniq, int nu, int S, int K){if (!g_degrade_zero || S > 4) return nu;- /* 1. aggregate gate weight per unique expert */- float *dg_wsum = calloc((size_t)nu, sizeof(float));+ /* 1. residency scan: hits always kept */+ unsigned char *dg_keep = calloc((size_t)nu, 1);+ for (int j = 0; j < nu; j++)+ if (is_resident(uniq[j])) dg_keep[j] = 1;++ /* 2. per-position tau gate: keep a miss expert if ANY position routes to it+ * with weight >= tau (each position tested independently, not aggregated) */for (int s = 0; s < S; s++)for (int kk = 0; kk < keff[s]; kk++) {- int e = idxs[s * K + kk];- for (int j = 0; j < nu; j++)- if (uniq[j] == e) { dg_wsum[j] += ws[s * K + kk]; break; }+ float wv = ws[s * K + kk];+ if (wv >= g_degrade_tau) {+ int e = idxs[s * K + kk];+ for (int j = 0; j < nu; j++)+ if (uniq[j] == e) { dg_keep[j] = 1; break; }+ }}- /* 2. keep = resident OR gate weight >= tau */- unsigned char *dg_keep = calloc((size_t)nu, 1);- for (int j = 0; j < nu; j++)- if (is_resident(uniq[j]) || dg_wsum[j] >= g_degrade_tau) dg_keep[j] = 1;-/* 3. rescue: no position may end up with zero routed experts *//* build seen[] from current dg_keep */int *seen = calloc((size_t)256, sizeof(int)); /* expert ids < 256 in tests */@@ -123,26 +124,15 @@ static int degrade_zero_apply(int *idxs, float *ws, int *keff,int *seen2 = calloc((size_t)256, sizeof(int));for (int j = 0; j < nu; j++) if (dg_keep[j]) seen2[uniq[j]] = 1;+ /* no renorm: survivors keep original weights β the approximation IS the+ * dropped mass; renorm would hide it and bias the output upward */for (int s = 0; s < S; s++) {- int w = 0; float sold = 0, snew = 0;+ int w = 0;for (int kk = 0; kk < keff[s]; kk++) {int e = idxs[s * K + kk]; float wv = ws[s * K + kk];- sold += wv;- if (seen2[e]) { idxs[s * K + w] = e; ws[s * K + w] = wv; snew += wv; w++; }- }- if (w < keff[s]) {- keff[s] = w;- if (c->norm_topk && w > 0) {- float sm = 0;- for (int kk = 0; kk < w; kk++) sm += ws[s * K + kk];- sm += 1e-20f;- for (int kk = 0; kk < w; kk++) ws[s * K + kk] /= sm;- for (int kk = 0; kk < w; kk++) ws[s * K + kk] *= c->routed_scale;- } else if (w > 0 && snew > 1e-20f && sold > snew) {- float sc = sold / snew;- for (int kk = 0; kk < w; kk++) ws[s * K + kk] *= sc;- }+ if (seen2[e]) { idxs[s * K + w] = e; ws[s * K + w] = wv; w++; }}+ if (w < keff[s]) keff[s] = w;}/* compact uniq[] */@@ -152,7 +142,6 @@ static int degrade_zero_apply(int *idxs, float *ws, int *keff,free(seen2);}- free(dg_wsum);free(dg_keep);return nu;}@@ -183,7 +172,6 @@ static int expert_in_routing(const int *idxs, const int *keff, int S, int K, int/* P1: g_degrade_zero=0 β block is a no-op */static void test_off_by_default(void) {printf("\nP1: off-by-default\n");- Cfg c = {1, 1.0f};/* S=1, K=2: experts 10 (w=0.8) and 11 (w=0.01, below tau) */int idxs[2] = {10, 11};float ws[2] = {0.8f, 0.01f};@@ -192,7 +180,7 @@ static void test_off_by_default(void) {g_degrade_zero = 0;g_degrade_dropped = 0;- nu = degrade_zero_apply(idxs, ws, keff, uniq, nu, 1, 2, &c);+ nu = degrade_zero_apply(idxs, ws, keff, uniq, nu, 1, 2);CHECK(nu == 2, "nu unchanged when off");CHECK(keff[0] == 2, "keff unchanged when off");@@ -202,7 +190,6 @@ static void test_off_by_default(void) {/* P2: resident experts are never dropped regardless of gate weight */static void test_hits_are_safe(void) {printf("\nP2: hits-are-safe\n");- Cfg c = {1, 1.0f};/* Expert 5 is resident with gate weight 0.001 (well below tau=0.03) */g_resident[0] = 5; g_nresident = 1;/* S=1, K=2: expert 5 (resident, w=0.001) and expert 7 (miss, w=0.8) */@@ -212,7 +199,7 @@ static void test_hits_are_safe(void) {int uniq[2] = {5, 7}; int nu = 2;g_degrade_zero = 1; g_degrade_tau = 0.03f; g_degrade_dropped = 0;- nu = degrade_zero_apply(idxs, ws, keff, uniq, nu, 1, 2, &c);+ nu = degrade_zero_apply(idxs, ws, keff, uniq, nu, 1, 2);CHECK(expert_in_uniq(uniq, nu, 5), "resident expert 5 kept despite low gate weight");CHECK(expert_in_uniq(uniq, nu, 7), "above-tau miss expert 7 kept");@@ -224,7 +211,6 @@ static void test_hits_are_safe(void) {/* P3: cold misses below tau are dropped; at/above tau are kept */static void test_tau_gate(void) {printf("\nP3: tau-gate\n");- Cfg c = {1, 1.0f};/* S=1, K=3: expert 1 (w=0.7, above tau), expert 2 (w=0.03, exactly tau),* expert 3 (w=0.02, below tau) β all cold misses */int idxs[3] = {1, 2, 3};@@ -233,7 +219,7 @@ static void test_tau_gate(void) {int uniq[3] = {1, 2, 3}; int nu = 3;g_degrade_zero = 1; g_degrade_tau = 0.03f; g_degrade_dropped = 0;- nu = degrade_zero_apply(idxs, ws, keff, uniq, nu, 1, 3, &c);+ nu = degrade_zero_apply(idxs, ws, keff, uniq, nu, 1, 3);CHECK(expert_in_uniq(uniq, nu, 1), "expert 1 (w=0.7) kept");CHECK(expert_in_uniq(uniq, nu, 2), "expert 2 (w=0.03, exactly tau) kept");@@ -242,49 +228,54 @@ static void test_tau_gate(void) {CHECK(keff[0] == 2, "keff reduced to 2");}-/* P4: renormalisation with norm_topk=1 (GLM-5.2 default) */-static void test_renorm_normtopk(void) {- printf("\nP4: renorm with norm_topk=1\n");- Cfg c = {1, 2.0f}; /* routed_scale=2.0 to verify it's applied */- /* S=1, K=2: expert 10 (w=0.6, keep), expert 11 (w=0.02, drop) */+/* P4: no-renorm β surviving weights are unchanged after a drop */+static void test_no_renorm(void) {+ printf("\nP4: no-renorm\n");+ /* S=1, K=2: expert 10 (w=0.6, keep), expert 11 (w=0.02, drop).+ * Survivor must keep its original weight 0.6 β not rescaled. */int idxs[2] = {10, 11};float ws[2] = {0.6f, 0.02f};int keff[1] = {2};int uniq[2] = {10, 11}; int nu = 2;g_degrade_zero = 1; g_degrade_tau = 0.03f; g_degrade_dropped = 0;- nu = degrade_zero_apply(idxs, ws, keff, uniq, nu, 1, 2, &c);+ nu = degrade_zero_apply(idxs, ws, keff, uniq, nu, 1, 2);CHECK(nu == 1, "uniq compacted to 1");CHECK(keff[0] == 1, "keff=1 after drop");- /* with norm_topk=1: ws[0] = (0.6/0.6) * routed_scale = 1.0 * 2.0 = 2.0 */- CHECKF(fabsf(ws[0] - 2.0f) < 1e-5f,- "renormed weight = routed_scale (got %.5f, want 2.0)", ws[0]);+ CHECKF(fabsf(ws[0] - 0.6f) < 1e-6f,+ "survivor weight unchanged (got %.6f, want 0.600000)", ws[0]);}-/* P5: renormalisation with norm_topk=0 (preserve output magnitude) */-static void test_renorm_nonnorm(void) {- printf("\nP5: renorm with norm_topk=0\n");- Cfg c = {0, 1.0f};- /* S=1, K=2: expert 20 (w=0.6, keep), expert 21 (w=0.02, drop) */- int idxs[2] = {20, 21};- float ws[2] = {0.6f, 0.02f};- int keff[1] = {2};- int uniq[2] = {20, 21}; int nu = 2;+/* P5: per-position tau gate β at S=2 a shared expert is kept when ONE+ * position's weight meets tau even if the other's does not */+static void test_per_position_tau(void) {+ printf("\nP5: per-position tau gate\n");+ /* S=2, K=1: both positions route to expert 20.+ * Position 0: w=0.01 (below tau), position 1: w=0.04 (above tau).+ * Per-position: expert 20 kept (position 1 meets tau).+ * Aggregate would also keep it (0.05 >= 0.03) β so test the case where+ * aggregate would DROP but per-position keeps: shared expert 21 with+ * pos0=0.02, pos1=0.04. Expert 22: pos0=0.01, pos1=0.01 (both below,+ * aggregate 0.02 < 0.03) β must be dropped. */+ int idxs[4] = {21, 22, /* pos 0: experts 21, 22 */+ 21, 22}; /* pos 1: experts 21, 22 */+ float ws[4] = {0.02f, 0.01f, /* pos 0 weights */+ 0.04f, 0.01f}; /* pos 1 weights */+ int keff[2] = {2, 2};+ int uniq[2] = {21, 22}; int nu = 2;g_degrade_zero = 1; g_degrade_tau = 0.03f; g_degrade_dropped = 0;- nu = degrade_zero_apply(idxs, ws, keff, uniq, nu, 1, 2, &c);+ nu = degrade_zero_apply(idxs, ws, keff, uniq, nu, 2, 2);- /* with norm_topk=0: ws[0] = 0.6 * (0.62 / 0.6) = 0.62 (old_sum/new_sum scaling) */- float expected = 0.6f * (0.62f / 0.6f);- CHECKF(fabsf(ws[0] - expected) < 1e-5f,- "rescaled weight preserves magnitude (got %.5f, want %.5f)", ws[0], expected);+ CHECK(expert_in_uniq(uniq, nu, 21), "expert 21 kept: pos 1 has w=0.04 >= tau");+ CHECK(!expert_in_uniq(uniq, nu, 22), "expert 22 dropped: all positions below tau");+ CHECK(g_degrade_dropped == 1, "counter=1");}/* P6: rescue rule β all experts are cold misses below tau */static void test_rescue_rule(void) {printf("\nP6: rescue rule\n");- Cfg c = {1, 1.0f};/* S=1, K=2: both experts are cold misses below tau.* Expert 30 has the higher gate weight β it must be rescued. */int idxs[2] = {30, 31};@@ -293,7 +284,7 @@ static void test_rescue_rule(void) {int uniq[2] = {30, 31}; int nu = 2;g_degrade_zero = 1; g_degrade_tau = 0.03f; g_degrade_dropped = 0;- nu = degrade_zero_apply(idxs, ws, keff, uniq, nu, 1, 2, &c);+ nu = degrade_zero_apply(idxs, ws, keff, uniq, nu, 1, 2);CHECK(keff[0] >= 1, "position not left with 0 routed experts");CHECK(expert_in_routing(idxs, keff, 1, 2, 30), "highest-weight expert 30 rescued");@@ -305,7 +296,6 @@ static void test_rescue_rule(void) {/* P7: prefill guard β S>4 means the block must be a complete no-op */static void test_prefill_guard(void) {printf("\nP7: prefill guard (S=8)\n");- Cfg c = {1, 1.0f};/* S=8, K=1: all experts are cold misses well below tau */int idxs[8] = {0, 1, 2, 3, 4, 5, 6, 7};float ws[8] = {0.001f, 0.001f, 0.001f, 0.001f, 0.001f, 0.001f, 0.001f, 0.001f};@@ -313,7 +303,7 @@ static void test_prefill_guard(void) {int uniq[8] = {0, 1, 2, 3, 4, 5, 6, 7}; int nu = 8;g_degrade_zero = 1; g_degrade_tau = 0.03f; g_degrade_dropped = 0;- nu = degrade_zero_apply(idxs, ws, keff, uniq, nu, 8, 1, &c);+ nu = degrade_zero_apply(idxs, ws, keff, uniq, nu, 8, 1);CHECK(nu == 8, "uniq unchanged for prefill batch");CHECK(g_degrade_dropped == 0, "counter unchanged for prefill batch");@@ -322,14 +312,13 @@ static void test_prefill_guard(void) {/* P8: counter accumulates correctly across two calls */static void test_counter_accumulates(void) {printf("\nP8: counter accumulates across calls\n");- Cfg c = {1, 1.0f};g_degrade_zero = 1; g_degrade_tau = 0.03f; g_degrade_dropped = 0;/* call 1: drop 1 expert */{int idxs[2] = {40, 41}; float ws[2] = {0.8f, 0.01f};int keff[1] = {2}; int uniq[2] = {40, 41}; int nu = 2;- degrade_zero_apply(idxs, ws, keff, uniq, nu, 1, 2, &c);+ degrade_zero_apply(idxs, ws, keff, uniq, nu, 1, 2);}CHECK(g_degrade_dropped == 1, "counter=1 after first call");@@ -337,7 +326,7 @@ static void test_counter_accumulates(void) {{int idxs[3] = {50, 51, 52}; float ws[3] = {0.8f, 0.01f, 0.01f};int keff[1] = {3}; int uniq[3] = {50, 51, 52}; int nu = 3;- degrade_zero_apply(idxs, ws, keff, uniq, nu, 1, 3, &c);+ degrade_zero_apply(idxs, ws, keff, uniq, nu, 1, 3);}CHECK(g_degrade_dropped == 3, "counter=3 after second call (cumulative)");}@@ -348,8 +337,8 @@ int main(void) {test_off_by_default();test_hits_are_safe();test_tau_gate();- test_renorm_normtopk();- test_renorm_nonnorm();+ test_no_renorm();+ test_per_position_tau();test_rescue_rule();test_prefill_guard();test_counter_accumulates();