Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
qwen36: port CACHE_ROUTE with the VRAM tier as the first residency level
Courtney Andrew Richardsonec9918149d25
diff --git a/c/Makefile b/c/Makefileindex 6cfa5cf7..9a409895 100644--- a/c/Makefile+++ b/c/Makefile@@ -1139,6 +1139,10 @@ qwen38-tier-engine-check: qwen38-tiny-fp8-generate tests/test_qwen38_tier_enginetests/test_qwen36_ctx$(EXE): tests/test_qwen36_ctx.c qwen36.c expert_ffn.h qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)$(CC) $(CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(LDFLAGS)+# CACHE_ROUTE: route_select() against a residency table, no model needed.+tests/test_qwen36_cache_route$(EXE): tests/test_qwen36_cache_route.c qwen36.c expert_ffn.h qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)+ $(CC) $(CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(LDFLAGS)+tests/test_qwen36_dense_batch$(EXE): tests/test_qwen36_dense_batch.c qwen36.c expert_ffn.h qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)$(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)diff --git a/c/qwen36.c b/c/qwen36.cindex 3a4eb071..da6eac1c 100644--- a/c/qwen36.c+++ b/c/qwen36.c@@ -636,6 +636,14 @@ typedef struct {int n, cap;} LCache;+/* CACHE_ROUTE telemetry (docs/CACHE_ROUTE.md): the lever changes which experts+ * run, so it carries its own meters. Only touched when the lever is on. */+typedef struct {+ uint64_t slots, swaps, swaps_vram; /* chosen slots; not in the true top-K; of those, VRAM-resident */+ uint64_t agree_hit, agree_tot; /* |chosen ∩ true top-K| summed, K summed */+ double kl_sum; uint64_t kl_n; /* mean KL(true top-K mass || chosen mass) */+} RouteStats;+typedef struct {Cfg c;shards S;@@ -647,6 +655,7 @@ typedef struct {float **DN_rec; /* [n_layers] recurrent state S[h]=[kdim,vdim] for DeltaNet layers (NULL for attn) */float **DN_conv; /* [n_layers] conv ring [conv_dim, convk-1] for DeltaNet layers (NULL for attn) */uint64_t clock, hits, miss;+ RouteStats route; /* CACHE_ROUTE / ROUTE_AGREE meters *//* Telemetria per la dashboard (Brain/Profile): tempo di lettura esperti* accumulato dall'avvio, e bitmap degli esperti toccati nel turno. */double t_disk;@@ -678,6 +687,13 @@ static volatile unsigned pilot_r = 0, pilot_w = 0;static Model *pilot_m = NULL;static int g_pilot = 0;static int g_wide = 1;+/* CACHE_ROUTE family, same names and defaults as the GLM engine (docs/CACHE_ROUTE.md). */+static int g_cache_route = 0;+static int g_route_j = 2;+static int g_route_m = 12;+static float g_route_p = 0.f;+static float g_route_alpha = 1.f;+static int g_route_agree = 0;static void pilot_prefetch(Model *m, int lnext, const float *x, int S);static void *pilot_worker(void *arg);@@ -2053,6 +2069,103 @@ static void moe_xf_run(Model *m, int layer, const float *x, int S, float *out, cfree(ex); free(exp); free(ridx); free(rval); free(tmp); free(scratch);}+/* ---------- CACHE_ROUTE: residency-aware top-K fill (docs/CACHE_ROUTE.md) ----------+ * The GLM engine's max-rank lever (arXiv:2412.00099) with one more level: an+ * expert already in the VRAM tier outranks one that is only in the RAM cache,+ * which outranks one on disk. The ranking is the same softmax mass the plain+ * path uses, restricted to `keep`. Selection inside the top-`win` window: the+ * true top-J first, then VRAM-resident experts in rank order, then RAM-+ * resident ones, then the true ranking. lvl(ctx, e) answers 2 / 1 / 0 and is+ * asked only for ranked candidates past J. val[] carries the raw mass, with+ * substitutes scaled by alpha; moe() renormalises afterwards exactly as it+ * does for the plain top-K. Callable without a Model so+ * tests/test_qwen36_cache_route.c can pin the selection against a residency+ * table. */+#define ROUTE_RANK_MAX 256+static void route_select(const float *pr, const uint8_t *keep, int E, int K,+ int J, int M, float P, float alpha,+ int (*lvl)(void *ctx, int e), void *ctx,+ int *idx, float *val, RouteStats *st) {+ if (K > ROUTE_RANK_MAX) K = ROUTE_RANK_MAX;+ if (J < 0) J = 0; if (J > K) J = K;+ int cap = (P > 0.f && P < 1.f) ? (M > 4*K ? M : 4*K) : (M > K ? M : K);+ if (cap > E) cap = E;+ if (cap > ROUTE_RANK_MAX) cap = ROUTE_RANK_MAX;+ int rank[ROUTE_RANK_MAX]; float rw[ROUTE_RANK_MAX]; int8_t rl[ROUTE_RANK_MAX];+ int n = 0;+ for (int r = 0; r < cap; r++) {+ int best = -1; float bv = -1e30f;+ for (int e = 0; e < E; e++) {+ if (keep && !keep[e]) continue;+ int taken = 0; for (int j = 0; j < n; j++) if (rank[j] == e) { taken = 1; break; }+ if (!taken && pr[e] > bv) { bv = pr[e]; best = e; }+ }+ if (best < 0) break;+ rank[n] = best; rw[n] = bv; n++;+ }+ int Kt = K < n ? K : n; /* the true top-K is rank[0..Kt) */+ int win = n;+ if (P > 0.f && P < 1.f) { /* cumulative-mass window: grow past K until P of the ranked mass */+ float tot = 1e-20f; for (int r = 0; r < n; r++) tot += rw[r] > 0 ? rw[r] : 0;+ float cum = 0; win = Kt;+ for (int r = 0; r < n; r++) { cum += rw[r] > 0 ? rw[r] : 0; win = r + 1; if (cum >= P * tot) break; }+ if (win < Kt) win = Kt;+ }+ for (int r = 0; r < n; r++) rl[r] = (r >= J && r < win) ? (int8_t)lvl(ctx, rank[r]) : 0;+ int chosen = 0, pos[ROUTE_RANK_MAX]; uint8_t used[ROUTE_RANK_MAX] = {0};+ for (int r = 0; r < J && r < n && chosen < K; r++) { pos[chosen++] = r; used[r] = 1; }+ for (int level = 2; level >= 1; level--)+ for (int r = J; r < win && chosen < K; r++)+ if (!used[r] && rl[r] == level) { pos[chosen++] = r; used[r] = 1; }+ for (int r = 0; r < n && chosen < K; r++)+ if (!used[r]) { pos[chosen++] = r; used[r] = 1; }+ for (int kk = 0; kk < chosen; kk++) {+ int r = pos[kk]; idx[kk] = rank[r]; val[kk] = rw[r];+ if (r >= Kt && alpha > 0.f && alpha < 1.f) val[kk] *= alpha;+ }+ for (int kk = chosen; kk < K; kk++) { idx[kk] = -1; val[kk] = 0.f; } /* fewer eligible than K: keep mask */+ if (!st) return;+ st->slots += (uint64_t)chosen; st->agree_tot += (uint64_t)chosen;+ float tsum = 1e-20f, csum = 1e-20f;+ for (int t = 0; t < Kt; t++) tsum += rw[t] > 0 ? rw[t] : 0;+ for (int kk = 0; kk < chosen; kk++) {+ csum += val[kk] > 0 ? val[kk] : 0;+ if (pos[kk] < Kt) st->agree_hit++;+ else { st->swaps++; if (rl[pos[kk]] == 2) st->swaps_vram++; }+ }+ double kl = 0; /* KL(true top-K mass || chosen mass), as the GLM meter */+ for (int t = 0; t < Kt; t++) {+ double pt = (rw[t] > 0 ? rw[t] : 0) / tsum; if (pt <= 0) continue;+ double pc = 1e-12;+ for (int kk = 0; kk < chosen; kk++) if (pos[kk] == t) { pc = (val[kk] > 0 ? val[kk] : 0) / csum; break; }+ kl += pt * log(pt / pc);+ }+ st->kl_sum += kl; st->kl_n++;+}++/* Residency levels for route_select: 2 = in the VRAM tier, 1 = in this+ * layer's RAM cache (pinned or LRU), 0 = would be read from disk. */+typedef struct { Model *m; int layer; } RouteCtx;+static int route_level(void *vctx, int e) {+ RouteCtx *rc = (RouteCtx *)vctx;+ if (qt_is_resident(rc->layer, e)) return 2;+ pthread_mutex_lock(&g_pilot_mx);+ Slot *s = slot_indexed(rc->m, rc->layer, e);+ pthread_mutex_unlock(&g_pilot_mx);+ return s ? 1 : 0;+}++static void route_footer(FILE *f, const Model *m) {+ if (g_cache_route && m->route.slots)+ fprintf(f, "CACHE_ROUTE J=%d M=%d P=%.2f alpha=%.2f | swap %.1f%% (%llu/%llu, %llu to VRAM)\n",+ g_route_j, g_route_m, g_route_p, g_route_alpha,+ 100.0*m->route.swaps/m->route.slots, (unsigned long long)m->route.swaps,+ (unsigned long long)m->route.slots, (unsigned long long)m->route.swaps_vram);+ if (m->route.agree_tot)+ fprintf(f, "route_agree %.1f%% | route_kl %.4f\n", 100.0*m->route.agree_hit/m->route.agree_tot,+ m->route.kl_n ? m->route.kl_sum/(double)m->route.kl_n : 0.0);+}+static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {Cfg *c = &m->c; int D = c->hidden, E = c->n_experts, K = c->topk, I = c->inter;float *logits = falloc((int64_t)S*E);@@ -2100,14 +2213,23 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {for (int e = 0; e < Ec; e++) keep[e] = 1;}int idx[256]; float val[256];- for (int kk = 0; kk < K; kk++) {- int best = -1; float bv = -1e30f;- for (int e = 0; e < E; e++) {- if (!keep[e]) continue;- int taken = 0; for (int j = 0; j < kk; j++) if (idx[j]==e){taken=1;break;}- if (!taken && pr[e] > bv) { bv = pr[e]; best = e; }+ if (g_cache_route) {+ RouteCtx rc = { m, layer };+ route_select(pr, keep, E, K, g_route_j, g_route_m, g_route_p, g_route_alpha,+ route_level, &rc, idx, val, &m->route);+ } else {+ for (int kk = 0; kk < K; kk++) {+ int best = -1; float bv = -1e30f;+ for (int e = 0; e < E; e++) {+ if (!keep[e]) continue;+ int taken = 0; for (int j = 0; j < kk; j++) if (idx[j]==e){taken=1;break;}+ if (!taken && pr[e] > bv) { bv = pr[e]; best = e; }+ }+ idx[kk] = best; val[kk] = bv;+ }+ if (g_route_agree) { /* plain routing: full agreement by construction */+ m->route.agree_hit += (uint64_t)K; m->route.agree_tot += (uint64_t)K; m->route.kl_n++;}- idx[kk] = best; val[kk] = bv;}if (m->resident_collecting) {for (int kk = 0; kk < K; kk++) if (idx[kk] >= 0) m->seen[(int64_t)layer * E + idx[kk]] = 1;@@ -3068,6 +3190,15 @@ int main(int argc, char **argv) {g_pilot = getenv("PILOT") ? atoi(getenv("PILOT")) : 0;g_wide = getenv("WIDE") ? atoi(getenv("WIDE")) : 1;if (g_wide < 1) g_wide = 1; if (g_wide > 4) g_wide = 4;+ g_cache_route = getenv("CACHE_ROUTE") ? atoi(getenv("CACHE_ROUTE")) : 0; /* prefer resident experts (VRAM tier, then RAM cache) inside the top-M window; changes which experts run: docs/CACHE_ROUTE.md */+ g_route_j = getenv("ROUTE_J") ? atoi(getenv("ROUTE_J")) : 2; /* true top-J always taken, resident or not, under CACHE_ROUTE=1 */+ g_route_m = getenv("ROUTE_M") ? atoi(getenv("ROUTE_M")) : 12; /* rank window inside which a resident expert may replace an unresident one */+ g_route_p = getenv("ROUTE_P") ? (float)atof(getenv("ROUTE_P")) : 0.f; /* cumulative-mass window for CACHE_ROUTE (0 = fixed M) */+ g_route_alpha = getenv("ROUTE_ALPHA") ? (float)atof(getenv("ROUTE_ALPHA")) : 1.f; /* scale substituted experts' gate mass before renorm (1 = off) */+ g_route_agree = getenv("ROUTE_AGREE") ? atoi(getenv("ROUTE_AGREE")) : g_cache_route; /* overlap% + KL vs the true top-K in the footer; auto-on under CACHE_ROUTE=1 */+ if (g_cache_route)+ fprintf(stderr, "[qwen36] CACHE_ROUTE=1 J=%d M=%d P=%.2f alpha=%.2f: VRAM tier > RAM cache > disk inside the top-M window (lossy: A/B it)\n",+ g_route_j, g_route_m, g_route_p, g_route_alpha);if (getenv("OPENAI")) g_openai = 1; /* OpenAI-compatible output */const char *mv = getenv("MODEL"); if (mv && *mv) g_model = mv;int hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0;@@ -3287,6 +3418,7 @@ int main(int argc, char **argv) {printf("TF-NLL: %.4f nats/token over %d tokens | ppl = %.2f\n", nll, scored, exp(nll));printf("Expert cache hit rate: %.1f%% (hit=%llu miss=%llu)\n", tot?100.0*m.hits/tot:0.0,(unsigned long long)m.hits, (unsigned long long)m.miss);+ route_footer(stdout, &m);printf("Speed: %.2f tok/s (%.1fs for %d tokens) | PEAK RSS: %.2f GB\n", scored/dt, dt, scored, rss_gb());free(buf); free(arena); return 0;}@@ -3352,6 +3484,7 @@ int main(int argc, char **argv) {fprintf(stderr, "\nPEAK RSS: %.2f GB\n", rss_gb());fprintf(stderr, "Expert cache hit rate: %.1f%% (hit=%llu miss=%llu)\n", tot?100.0*m.hits/tot:0.0,(unsigned long long)m.hits, (unsigned long long)m.miss);+ route_footer(stderr, &m);fprintf(stderr, "Speed: %.2f tok/s (%.1fs for %d tokens)\n", n_new/dt, dt, n_new);free(buf); free(arena);/* Oracle mode is a gate, not a report: a mismatch must fail the caller.diff --git a/c/tests/test_qwen36_cache_route.c b/c/tests/test_qwen36_cache_route.cnew file mode 100644index 00000000..dbb1d01d--- /dev/null+++ b/c/tests/test_qwen36_cache_route.c@@ -0,0 +1,106 @@+/* CACHE_ROUTE in the qwen36 engine: route_select() against a residency table.+ *+ * The lever changes which experts run, so what it must never do is spelled+ * out here: touch the true top-J, look past the top-M window, rank an expert+ * the group mask excluded, or report agreement it does not have. The two+ * residency levels (VRAM tier over RAM cache) and ROUTE_ALPHA are pinned too.+ * No Model, no weights: the function takes the softmax row and a callback. */+#define main qwen36_main_unused+#include "../qwen36.c"+#undef main++#include <stdio.h>+#include <string.h>++static int fails;+static void check(int ok, const char *what) {+ if (!ok) { fails++; printf(" FAIL: %s\n", what); }+}++/* residency table for the callback: level per expert id */+static int8_t g_lvl[64];+static int lvl_table(void *ctx, int e) { (void)ctx; return g_lvl[e]; }++/* softmax row where expert e has mass proportional to (E - e): rank == id. */+static void ranked_row(float *pr, int E) {+ float sum = 0; for (int e = 0; e < E; e++) { pr[e] = (float)(E - e); sum += pr[e]; }+ for (int e = 0; e < E; e++) pr[e] /= sum;+}++static int has(const int *idx, int K, int e) { for (int k = 0; k < K; k++) if (idx[k] == e) return 1; return 0; }++int main(void) {+ enum { E = 16, K = 4 };+ float pr[E]; uint8_t keep[E]; int idx[K]; float val[K]; RouteStats st;+ ranked_row(pr, E); memset(keep, 1, sizeof keep);++ /* nothing resident: the plain top-K, no substitution, full agreement, zero KL */+ memset(g_lvl, 0, sizeof g_lvl); memset(&st, 0, sizeof st);+ route_select(pr, keep, E, K, 2, 12, 0.f, 1.f, lvl_table, NULL, idx, val, &st);+ check(idx[0]==0 && idx[1]==1 && idx[2]==2 && idx[3]==3, "no_resident_expert_means_the_true_top_k");+ check(st.swaps == 0 && st.agree_hit == K && st.agree_tot == K, "no_substitution_reports_full_agreement");+ check(st.kl_n == 1 && st.kl_sum == 0.0, "identical_choice_has_zero_kl");+ check(val[0] == pr[0] && val[3] == pr[3], "val_carries_the_raw_mass_of_each_chosen_expert");++ /* a RAM-resident expert inside the window replaces the unresident tail */+ memset(g_lvl, 0, sizeof g_lvl); g_lvl[7] = 1; memset(&st, 0, sizeof st);+ route_select(pr, keep, E, K, 2, 12, 0.f, 1.f, lvl_table, NULL, idx, val, &st);+ check(idx[0]==0 && idx[1]==1, "true_top_j_is_taken_first");+ check(idx[2]==7 && idx[3]==2, "resident_expert_in_window_fills_before_the_unresident_ranking");+ check(st.swaps == 1 && st.swaps_vram == 0 && st.agree_hit == 3, "one_substitution_is_counted_once_and_not_as_vram");+ check(st.kl_sum > 0.0, "a_substitution_shows_up_as_positive_kl");++ /* the sacred top-J is taken even when a lower rank is resident and it is not */+ memset(g_lvl, 0, sizeof g_lvl); g_lvl[2] = 1; g_lvl[3] = 1; g_lvl[4] = 1; g_lvl[5] = 1; memset(&st, 0, sizeof st);+ route_select(pr, keep, E, K, 2, 12, 0.f, 1.f, lvl_table, NULL, idx, val, &st);+ check(idx[0]==0 && idx[1]==1 && idx[2]==2 && idx[3]==3, "unresident_top_j_is_never_displaced");+ check(st.swaps == 0, "resident_experts_already_in_the_top_k_are_not_substitutions");++ /* VRAM outranks RAM inside the window, whatever their rank order */+ memset(g_lvl, 0, sizeof g_lvl); g_lvl[5] = 1; g_lvl[9] = 2; memset(&st, 0, sizeof st);+ route_select(pr, keep, E, K, 2, 12, 0.f, 1.f, lvl_table, NULL, idx, val, &st);+ check(idx[2]==9 && idx[3]==5, "vram_resident_fills_before_ram_resident");+ check(st.swaps == 2 && st.swaps_vram == 1, "swaps_to_vram_are_counted_separately");++ /* residency outside the top-M window is ignored */+ memset(g_lvl, 0, sizeof g_lvl); g_lvl[12] = 2; g_lvl[15] = 1; memset(&st, 0, sizeof st);+ route_select(pr, keep, E, K, 2, 12, 0.f, 1.f, lvl_table, NULL, idx, val, &st);+ check(!has(idx, K, 12) && !has(idx, K, 15) && idx[2]==2 && idx[3]==3, "resident_expert_past_the_window_is_not_chosen");++ /* ROUTE_J=0: even rank 0 may be replaced; ROUTE_J=K: nothing may */+ memset(g_lvl, 0, sizeof g_lvl); g_lvl[6] = 1; g_lvl[7] = 1; g_lvl[8] = 1; g_lvl[9] = 1; memset(&st, 0, sizeof st);+ route_select(pr, keep, E, K, 0, 12, 0.f, 1.f, lvl_table, NULL, idx, val, &st);+ check(idx[0]==6 && idx[1]==7 && idx[2]==8 && idx[3]==9 && st.swaps == 4, "route_j_zero_lets_residents_take_every_slot");+ memset(&st, 0, sizeof st);+ route_select(pr, keep, E, K, K, 12, 0.f, 1.f, lvl_table, NULL, idx, val, &st);+ check(idx[0]==0 && idx[3]==3 && st.swaps == 0, "route_j_equal_to_k_disables_substitution");++ /* the group mask is respected: an excluded expert is never ranked or chosen */+ memset(g_lvl, 0, sizeof g_lvl); g_lvl[5] = 2; memset(&st, 0, sizeof st);+ keep[1] = 0; keep[5] = 0;+ route_select(pr, keep, E, K, 2, 12, 0.f, 1.f, lvl_table, NULL, idx, val, &st);+ check(!has(idx, K, 1) && !has(idx, K, 5), "masked_experts_are_never_chosen_even_when_resident");+ check(idx[0]==0 && idx[1]==2 && idx[2]==3 && idx[3]==4, "masked_ranking_shifts_the_true_top_k");+ memset(keep, 1, sizeof keep);++ /* ROUTE_ALPHA scales only the substitute's mass, before moe() renormalises */+ memset(g_lvl, 0, sizeof g_lvl); g_lvl[7] = 1; memset(&st, 0, sizeof st);+ route_select(pr, keep, E, K, 2, 12, 0.f, 0.5f, lvl_table, NULL, idx, val, &st);+ check(idx[2]==7 && val[2] == pr[7]*0.5f && val[0] == pr[0] && val[3] == pr[2], "alpha_halves_the_substitute_and_nothing_else");++ /* ROUTE_P: the window is the mass, not M. Ranks 0..4 hold 70/136 = 0.515 of the+ * mass, so P=0.5 closes the window at rank 4: rank 4 is inside, rank 5 outside. */+ memset(g_lvl, 0, sizeof g_lvl); g_lvl[4] = 1; g_lvl[5] = 1; memset(&st, 0, sizeof st);+ route_select(pr, keep, E, K, 2, 12, 0.5f, 1.f, lvl_table, NULL, idx, val, &st);+ check(has(idx, K, 4) && !has(idx, K, 5), "route_p_bounds_the_window_by_cumulative_mass");++ /* fewer eligible experts than K: the remainder is marked -1, not garbage */+ memset(g_lvl, 0, sizeof g_lvl); memset(&st, 0, sizeof st);+ memset(keep, 0, sizeof keep); keep[3] = 1; keep[9] = 1;+ route_select(pr, keep, E, K, 2, 12, 0.f, 1.f, lvl_table, NULL, idx, val, &st);+ check(idx[0]==3 && idx[1]==9 && idx[2]==-1 && idx[3]==-1 && st.slots == 2, "short_candidate_list_pads_with_minus_one");++ if (fails) { printf("test_qwen36_cache_route: %d fallimenti\n", fails); return 1; }+ printf("test_qwen36_cache_route: ok\n");+ return 0;+}diff --git a/docs/CACHE_ROUTE.md b/docs/CACHE_ROUTE.mdindex 78f47e9c..30440988 100644--- a/docs/CACHE_ROUTE.md+++ b/docs/CACHE_ROUTE.md@@ -9,6 +9,24 @@ still rank inside top-`M`.This is **routing-side** (can change which experts run). Complementary to**PILOT** (next-layer *prefetch* of weights; does not change expert IDs).+## Engines and residency levels++| engine | "resident" means | levels |+|---|---|---|+| `colibri` (GLM-5.2) | in the pin set or the RAM LRU cache | one |+| `qwen36` | in the VRAM expert tier (CUDA / Vulkan), else in the RAM LRU cache | two: VRAM first, then RAM |++In `qwen36` a slot past the sacred top-`J` is filled, inside the top-`M`+window, by the highest-ranked VRAM-resident expert, then the highest-ranked+RAM-resident one, then the plain ranking. With no tier enabled the VRAM level+is empty and the lever degrades to the single-level GLM behaviour. The knobs+and meters are the same in both engines; the `qwen36` footer additionally+splits the swap count into `N to VRAM`.++The only substitutable slots are ranks `J..K-1`. On a model that routes+top-2, the default `ROUTE_J=2` leaves nothing to substitute and the footer+reports `swap 0.0%`; lower `ROUTE_J` to trade agreement for hits.+## Flags| Env | Default | Meaning |@@ -42,6 +60,7 @@ Footer / serve `STAT` when enabled:- `route_agree` — |chosen ∩ true top-K| / K- `route_kl` — mass KL (true top-K vs chosen)- `hit N%` — expert cache hit (disk residency)+- `N to VRAM` (`qwen36` only) — of the swaps, how many landed on a tier-resident expert## A/B vs PILOT@@ -61,5 +80,10 @@ CACHE_ROUTE=1 PILOT=1 ...**Routing-only** + telemetry + this note. Does **not** require CUDA/fuse/device-tierpatches — CPU streaming + pin/LRU is enough to A/B the lever against PILOT / #119.+The `qwen36` port keeps that shape: `route_select()` is a pure function of the+softmax row, the group mask and a residency callback, pinned by+`tests/test_qwen36_cache_route.c` without a model. With the lever unset the+router runs the original top-K loop and emits byte-identical token ids.+Treat as experimental until quality gates (e.g. `./coli bench`) pass; do notdefault `CACHE_ROUTE=1`.diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex b3d36252..48f1fc22 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -15,7 +15,7 @@ what follows, but the sister engines read their own:| `colibri` | `c/colibri.c` | everything below except the three sections named for another engine || `kimi_k3` | `c/kimi_k3.c` | the `K3_*` family — see [Kimi K3 engine](#kimi-k3-engine-kimi_k3) || `inkling` | `c/inkling.c` | `INK_*`, plus `CTX_MAX`, `PIN_N`, `REP_PEN`, `GPU_DEV`, `NOGPU` — see [Inkling engine](#inkling-engine-inkling) |-| `qwen36` | `c/qwen36.c` | `QWEN_*`, `Q36_*`, and its dense/CUDA-tier controls — see [Qwen3.6 engine](#qwen36-engine-qwen36) |+| `qwen36` | `c/qwen36.c` | `QWEN_*`, `Q36_*`, its dense/CUDA-tier controls, and the `CACHE_ROUTE` family (VRAM tier over RAM cache) — see [Qwen3.6 engine](#qwen36-engine-qwen36) || `qwen38` | `c/qwen38.c` | `Q38_MAXT`, `Q38_EOS`, `Q38_NATIVE_FP8`, `Q38_NATIVE_BF16`, `Q38_PREFILL_BATCH`, `COLI_TIMERS` — see [Qwen3.8 engine](#qwen38-engine-qwen38) || `olmoe` | `c/olmoe.c` | `HOT`, `WIDE`, `SMOOTH`, `CONF_LIMIT`, `MAX_NEW`, `CHAT`, `EXPERT_DROP`, `WARMUP` — see [OLMoE engine](#olmoe-engine-olmoe) || `deepseek_v4` | `c/deepseek_v4.c` | `CTX`, the `V4_*` / `DSV4_*` families and the two `COLI_CUDA_*_BATCH` gates — see [DeepSeek V4 engine](#deepseek-v4-engine-deepseek_v4); note that the CUDA section below describes `colibri.c` knobs (`COLI_CUDA`, `CUDA_DENSE`, ...) which the V4 engine does not read — its GPU switch is `DSV4_CUDA` |@@ -103,12 +103,12 @@ Format: `VAR` — default — effect.| `COUPLE` | unset | Path to a coupling-score file driving cross-layer expert prefetch (#176). When set, `couple_load` reads it. || `COUPLE_K` | `8` | Top-K coupled experts per layer when `COUPLE` is set. || `COUPLE_D` | `1` | Coupling lookahead depth (`1` or `2`) when `COUPLE` is set. |-| `CACHE_ROUTE` | `0` (off) | Opt-in max-rank cache-aware MoE routing (pin∪LRU prefer within top-M). See [CACHE_ROUTE.md](CACHE_ROUTE.md). |+| `CACHE_ROUTE` | `0` (off) | Opt-in max-rank cache-aware MoE routing (pin∪LRU prefer within top-M). Also read by `qwen36`, where the VRAM tier outranks the RAM cache. See [CACHE_ROUTE.md](CACHE_ROUTE.md). || `ROUTE_J` | `2` | Sacred top ranks always taken when `CACHE_ROUTE=1`. || `ROUTE_M` | `12` | Max-rank window for resident preference when `CACHE_ROUTE=1`. || `ROUTE_P` | `0` | Cumulative mass window for CACHE_ROUTE (`0` = fixed M). || `ROUTE_ALPHA` | `1` | Scale gate mass of substituted experts before renorm (`1` = off). |-| `ROUTE_AGREE` | auto | Overlap% + KL vs true top-K; auto-on when `CACHE_ROUTE=1`. |+| `ROUTE_AGREE` | auto | Overlap% + KL vs true top-K; auto-on when `CACHE_ROUTE=1`. Alone it changes nothing and prints the meters (always 100% / 0). || `ROUTE_TRACE` | unset | If set to a path, logs every routing decision there (testing/analysis). || `ABSORB` | `-1` (auto: absorbed for S≤4) | MLA attention absorption mode. || `IDOT` | `1` | Integer dot-product kernel. `IDOT=0` uses exact f32 kernels (for A/B numerical checks). |diff --git a/docs/qwen36.md b/docs/qwen36.mdindex 32cfcdd4..b449af49 100644--- a/docs/qwen36.md+++ b/docs/qwen36.md@@ -104,6 +104,25 @@ overlap, not this kernel. `tests/test_expert_ffn` holds the numerics.keeps its own path: it uploads the pair-layout int4 and computes misses fromthe int8 copy.+## Cache-aware routing (`CACHE_ROUTE`, off by default)++The residual misses above are the lever's target. `CACHE_ROUTE=1` ports the+GLM engine's max-rank re-routing ([CACHE_ROUTE.md](CACHE_ROUTE.md),+arXiv:2412.00099) to this engine with two residency levels: inside the top-`M`+window, a slot past the sacred top-`J` prefers an expert already in the VRAM+tier, then one in the RAM cache, then the plain ranking. It is **lossy**: it+changes which experts run, so the semantic contract is off while it is set and+the footer prints what it cost, `route_agree` (overlap with the true top-K)+and `route_kl` (mass KL), next to the swap and hit rates. Unset, the router+is the original loop and the token ids are byte-identical; `ROUTE_AGREE=1`+alone prints the meters at 100 % / 0 without touching routing.++Qwen3.6 routes top-8 (plus the shared expert), so the default `ROUTE_J=2`+leaves six substitutable slots per token; the tiny fixture routes top-2 and+needs `ROUTE_J<2` to show any swap at all. A/B it the way the GLM doc does:+same prompt and seed, tok/s and hit rate against agreement and KL, and treat+`PPL=1` on a teacher-forced reference as the quality bar.+## Which container?The gs64 container carries one scale per 64-weight group instead of one per