Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

qwen36: port CACHE_ROUTE with the VRAM tier as the first residency level

Courtney Andrew Richardsonec9918149d25
diff --git a/c/Makefile b/c/Makefile
index 6cfa5cf7..9a409895 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1139,6 +1139,10 @@ qwen38-tier-engine-check: qwen38-tiny-fp8-generate tests/test_qwen38_tier_engine
tests/test_qwen36_ctx$(EXE): tests/test_qwen36_ctx.c qwen36.c expert_ffn.h qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
$(CC) $(CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(LDFLAGS)
+# CACHE_ROUTE: route_select() against a residency table, no model needed.
+tests/test_qwen36_cache_route$(EXE): tests/test_qwen36_cache_route.c qwen36.c expert_ffn.h qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
+ $(CC) $(CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(LDFLAGS)
+
tests/test_qwen36_dense_batch$(EXE): tests/test_qwen36_dense_batch.c qwen36.c expert_ffn.h qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
$(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)
diff --git a/c/qwen36.c b/c/qwen36.c
index 3a4eb071..da6eac1c 100644
--- a/c/qwen36.c
+++ b/c/qwen36.c
@@ -636,6 +636,14 @@ typedef struct {
int n, cap;
} LCache;
+/* CACHE_ROUTE telemetry (docs/CACHE_ROUTE.md): the lever changes which experts
+ * run, so it carries its own meters. Only touched when the lever is on. */
+typedef struct {
+ uint64_t slots, swaps, swaps_vram; /* chosen slots; not in the true top-K; of those, VRAM-resident */
+ uint64_t agree_hit, agree_tot; /* |chosen ∩ true top-K| summed, K summed */
+ double kl_sum; uint64_t kl_n; /* mean KL(true top-K mass || chosen mass) */
+} RouteStats;
+
typedef struct {
Cfg c;
shards S;
@@ -647,6 +655,7 @@ typedef struct {
float **DN_rec; /* [n_layers] recurrent state S[h]=[kdim,vdim] for DeltaNet layers (NULL for attn) */
float **DN_conv; /* [n_layers] conv ring [conv_dim, convk-1] for DeltaNet layers (NULL for attn) */
uint64_t clock, hits, miss;
+ RouteStats route; /* CACHE_ROUTE / ROUTE_AGREE meters */
/* Telemetria per la dashboard (Brain/Profile): tempo di lettura esperti
* accumulato dall'avvio, e bitmap degli esperti toccati nel turno. */
double t_disk;
@@ -678,6 +687,13 @@ static volatile unsigned pilot_r = 0, pilot_w = 0;
static Model *pilot_m = NULL;
static int g_pilot = 0;
static int g_wide = 1;
+/* CACHE_ROUTE family, same names and defaults as the GLM engine (docs/CACHE_ROUTE.md). */
+static int g_cache_route = 0;
+static int g_route_j = 2;
+static int g_route_m = 12;
+static float g_route_p = 0.f;
+static float g_route_alpha = 1.f;
+static int g_route_agree = 0;
static void pilot_prefetch(Model *m, int lnext, const float *x, int S);
static void *pilot_worker(void *arg);
@@ -2053,6 +2069,103 @@ static void moe_xf_run(Model *m, int layer, const float *x, int S, float *out, c
free(ex); free(exp); free(ridx); free(rval); free(tmp); free(scratch);
}
+/* ---------- CACHE_ROUTE: residency-aware top-K fill (docs/CACHE_ROUTE.md) ----------
+ * The GLM engine's max-rank lever (arXiv:2412.00099) with one more level: an
+ * expert already in the VRAM tier outranks one that is only in the RAM cache,
+ * which outranks one on disk. The ranking is the same softmax mass the plain
+ * path uses, restricted to `keep`. Selection inside the top-`win` window: the
+ * true top-J first, then VRAM-resident experts in rank order, then RAM-
+ * resident ones, then the true ranking. lvl(ctx, e) answers 2 / 1 / 0 and is
+ * asked only for ranked candidates past J. val[] carries the raw mass, with
+ * substitutes scaled by alpha; moe() renormalises afterwards exactly as it
+ * does for the plain top-K. Callable without a Model so
+ * tests/test_qwen36_cache_route.c can pin the selection against a residency
+ * table. */
+#define ROUTE_RANK_MAX 256
+static void route_select(const float *pr, const uint8_t *keep, int E, int K,
+ int J, int M, float P, float alpha,
+ int (*lvl)(void *ctx, int e), void *ctx,
+ int *idx, float *val, RouteStats *st) {
+ if (K > ROUTE_RANK_MAX) K = ROUTE_RANK_MAX;
+ if (J < 0) J = 0; if (J > K) J = K;
+ int cap = (P > 0.f && P < 1.f) ? (M > 4*K ? M : 4*K) : (M > K ? M : K);
+ if (cap > E) cap = E;
+ if (cap > ROUTE_RANK_MAX) cap = ROUTE_RANK_MAX;
+ int rank[ROUTE_RANK_MAX]; float rw[ROUTE_RANK_MAX]; int8_t rl[ROUTE_RANK_MAX];
+ int n = 0;
+ for (int r = 0; r < cap; r++) {
+ int best = -1; float bv = -1e30f;
+ for (int e = 0; e < E; e++) {
+ if (keep && !keep[e]) continue;
+ int taken = 0; for (int j = 0; j < n; j++) if (rank[j] == e) { taken = 1; break; }
+ if (!taken && pr[e] > bv) { bv = pr[e]; best = e; }
+ }
+ if (best < 0) break;
+ rank[n] = best; rw[n] = bv; n++;
+ }
+ int Kt = K < n ? K : n; /* the true top-K is rank[0..Kt) */
+ int win = n;
+ if (P > 0.f && P < 1.f) { /* cumulative-mass window: grow past K until P of the ranked mass */
+ float tot = 1e-20f; for (int r = 0; r < n; r++) tot += rw[r] > 0 ? rw[r] : 0;
+ float cum = 0; win = Kt;
+ for (int r = 0; r < n; r++) { cum += rw[r] > 0 ? rw[r] : 0; win = r + 1; if (cum >= P * tot) break; }
+ if (win < Kt) win = Kt;
+ }
+ for (int r = 0; r < n; r++) rl[r] = (r >= J && r < win) ? (int8_t)lvl(ctx, rank[r]) : 0;
+ int chosen = 0, pos[ROUTE_RANK_MAX]; uint8_t used[ROUTE_RANK_MAX] = {0};
+ for (int r = 0; r < J && r < n && chosen < K; r++) { pos[chosen++] = r; used[r] = 1; }
+ for (int level = 2; level >= 1; level--)
+ for (int r = J; r < win && chosen < K; r++)
+ if (!used[r] && rl[r] == level) { pos[chosen++] = r; used[r] = 1; }
+ for (int r = 0; r < n && chosen < K; r++)
+ if (!used[r]) { pos[chosen++] = r; used[r] = 1; }
+ for (int kk = 0; kk < chosen; kk++) {
+ int r = pos[kk]; idx[kk] = rank[r]; val[kk] = rw[r];
+ if (r >= Kt && alpha > 0.f && alpha < 1.f) val[kk] *= alpha;
+ }
+ for (int kk = chosen; kk < K; kk++) { idx[kk] = -1; val[kk] = 0.f; } /* fewer eligible than K: keep mask */
+ if (!st) return;
+ st->slots += (uint64_t)chosen; st->agree_tot += (uint64_t)chosen;
+ float tsum = 1e-20f, csum = 1e-20f;
+ for (int t = 0; t < Kt; t++) tsum += rw[t] > 0 ? rw[t] : 0;
+ for (int kk = 0; kk < chosen; kk++) {
+ csum += val[kk] > 0 ? val[kk] : 0;
+ if (pos[kk] < Kt) st->agree_hit++;
+ else { st->swaps++; if (rl[pos[kk]] == 2) st->swaps_vram++; }
+ }
+ double kl = 0; /* KL(true top-K mass || chosen mass), as the GLM meter */
+ for (int t = 0; t < Kt; t++) {
+ double pt = (rw[t] > 0 ? rw[t] : 0) / tsum; if (pt <= 0) continue;
+ double pc = 1e-12;
+ for (int kk = 0; kk < chosen; kk++) if (pos[kk] == t) { pc = (val[kk] > 0 ? val[kk] : 0) / csum; break; }
+ kl += pt * log(pt / pc);
+ }
+ st->kl_sum += kl; st->kl_n++;
+}
+
+/* Residency levels for route_select: 2 = in the VRAM tier, 1 = in this
+ * layer's RAM cache (pinned or LRU), 0 = would be read from disk. */
+typedef struct { Model *m; int layer; } RouteCtx;
+static int route_level(void *vctx, int e) {
+ RouteCtx *rc = (RouteCtx *)vctx;
+ if (qt_is_resident(rc->layer, e)) return 2;
+ pthread_mutex_lock(&g_pilot_mx);
+ Slot *s = slot_indexed(rc->m, rc->layer, e);
+ pthread_mutex_unlock(&g_pilot_mx);
+ return s ? 1 : 0;
+}
+
+static void route_footer(FILE *f, const Model *m) {
+ if (g_cache_route && m->route.slots)
+ fprintf(f, "CACHE_ROUTE J=%d M=%d P=%.2f alpha=%.2f | swap %.1f%% (%llu/%llu, %llu to VRAM)\n",
+ g_route_j, g_route_m, g_route_p, g_route_alpha,
+ 100.0*m->route.swaps/m->route.slots, (unsigned long long)m->route.swaps,
+ (unsigned long long)m->route.slots, (unsigned long long)m->route.swaps_vram);
+ if (m->route.agree_tot)
+ fprintf(f, "route_agree %.1f%% | route_kl %.4f\n", 100.0*m->route.agree_hit/m->route.agree_tot,
+ m->route.kl_n ? m->route.kl_sum/(double)m->route.kl_n : 0.0);
+}
+
static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
Cfg *c = &m->c; int D = c->hidden, E = c->n_experts, K = c->topk, I = c->inter;
float *logits = falloc((int64_t)S*E);
@@ -2100,14 +2213,23 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
for (int e = 0; e < Ec; e++) keep[e] = 1;
}
int idx[256]; float val[256];
- for (int kk = 0; kk < K; kk++) {
- int best = -1; float bv = -1e30f;
- for (int e = 0; e < E; e++) {
- if (!keep[e]) continue;
- int taken = 0; for (int j = 0; j < kk; j++) if (idx[j]==e){taken=1;break;}
- if (!taken && pr[e] > bv) { bv = pr[e]; best = e; }
+ if (g_cache_route) {
+ RouteCtx rc = { m, layer };
+ route_select(pr, keep, E, K, g_route_j, g_route_m, g_route_p, g_route_alpha,
+ route_level, &rc, idx, val, &m->route);
+ } else {
+ for (int kk = 0; kk < K; kk++) {
+ int best = -1; float bv = -1e30f;
+ for (int e = 0; e < E; e++) {
+ if (!keep[e]) continue;
+ int taken = 0; for (int j = 0; j < kk; j++) if (idx[j]==e){taken=1;break;}
+ if (!taken && pr[e] > bv) { bv = pr[e]; best = e; }
+ }
+ idx[kk] = best; val[kk] = bv;
+ }
+ if (g_route_agree) { /* plain routing: full agreement by construction */
+ m->route.agree_hit += (uint64_t)K; m->route.agree_tot += (uint64_t)K; m->route.kl_n++;
}
- idx[kk] = best; val[kk] = bv;
}
if (m->resident_collecting) {
for (int kk = 0; kk < K; kk++) if (idx[kk] >= 0) m->seen[(int64_t)layer * E + idx[kk]] = 1;
@@ -3068,6 +3190,15 @@ int main(int argc, char **argv) {
g_pilot = getenv("PILOT") ? atoi(getenv("PILOT")) : 0;
g_wide = getenv("WIDE") ? atoi(getenv("WIDE")) : 1;
if (g_wide < 1) g_wide = 1; if (g_wide > 4) g_wide = 4;
+ g_cache_route = getenv("CACHE_ROUTE") ? atoi(getenv("CACHE_ROUTE")) : 0; /* prefer resident experts (VRAM tier, then RAM cache) inside the top-M window; changes which experts run: docs/CACHE_ROUTE.md */
+ g_route_j = getenv("ROUTE_J") ? atoi(getenv("ROUTE_J")) : 2; /* true top-J always taken, resident or not, under CACHE_ROUTE=1 */
+ g_route_m = getenv("ROUTE_M") ? atoi(getenv("ROUTE_M")) : 12; /* rank window inside which a resident expert may replace an unresident one */
+ g_route_p = getenv("ROUTE_P") ? (float)atof(getenv("ROUTE_P")) : 0.f; /* cumulative-mass window for CACHE_ROUTE (0 = fixed M) */
+ g_route_alpha = getenv("ROUTE_ALPHA") ? (float)atof(getenv("ROUTE_ALPHA")) : 1.f; /* scale substituted experts' gate mass before renorm (1 = off) */
+ g_route_agree = getenv("ROUTE_AGREE") ? atoi(getenv("ROUTE_AGREE")) : g_cache_route; /* overlap% + KL vs the true top-K in the footer; auto-on under CACHE_ROUTE=1 */
+ if (g_cache_route)
+ fprintf(stderr, "[qwen36] CACHE_ROUTE=1 J=%d M=%d P=%.2f alpha=%.2f: VRAM tier > RAM cache > disk inside the top-M window (lossy: A/B it)\n",
+ g_route_j, g_route_m, g_route_p, g_route_alpha);
if (getenv("OPENAI")) g_openai = 1; /* OpenAI-compatible output */
const char *mv = getenv("MODEL"); if (mv && *mv) g_model = mv;
int hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0;
@@ -3287,6 +3418,7 @@ int main(int argc, char **argv) {
printf("TF-NLL: %.4f nats/token over %d tokens | ppl = %.2f\n", nll, scored, exp(nll));
printf("Expert cache hit rate: %.1f%% (hit=%llu miss=%llu)\n", tot?100.0*m.hits/tot:0.0,
(unsigned long long)m.hits, (unsigned long long)m.miss);
+ route_footer(stdout, &m);
printf("Speed: %.2f tok/s (%.1fs for %d tokens) | PEAK RSS: %.2f GB\n", scored/dt, dt, scored, rss_gb());
free(buf); free(arena); return 0;
}
@@ -3352,6 +3484,7 @@ int main(int argc, char **argv) {
fprintf(stderr, "\nPEAK RSS: %.2f GB\n", rss_gb());
fprintf(stderr, "Expert cache hit rate: %.1f%% (hit=%llu miss=%llu)\n", tot?100.0*m.hits/tot:0.0,
(unsigned long long)m.hits, (unsigned long long)m.miss);
+ route_footer(stderr, &m);
fprintf(stderr, "Speed: %.2f tok/s (%.1fs for %d tokens)\n", n_new/dt, dt, n_new);
free(buf); free(arena);
/* Oracle mode is a gate, not a report: a mismatch must fail the caller.
diff --git a/c/tests/test_qwen36_cache_route.c b/c/tests/test_qwen36_cache_route.c
new file mode 100644
index 00000000..dbb1d01d
--- /dev/null
+++ b/c/tests/test_qwen36_cache_route.c
@@ -0,0 +1,106 @@
+/* CACHE_ROUTE in the qwen36 engine: route_select() against a residency table.
+ *
+ * The lever changes which experts run, so what it must never do is spelled
+ * out here: touch the true top-J, look past the top-M window, rank an expert
+ * the group mask excluded, or report agreement it does not have. The two
+ * residency levels (VRAM tier over RAM cache) and ROUTE_ALPHA are pinned too.
+ * No Model, no weights: the function takes the softmax row and a callback. */
+#define main qwen36_main_unused
+#include "../qwen36.c"
+#undef main
+
+#include <stdio.h>
+#include <string.h>
+
+static int fails;
+static void check(int ok, const char *what) {
+ if (!ok) { fails++; printf(" FAIL: %s\n", what); }
+}
+
+/* residency table for the callback: level per expert id */
+static int8_t g_lvl[64];
+static int lvl_table(void *ctx, int e) { (void)ctx; return g_lvl[e]; }
+
+/* softmax row where expert e has mass proportional to (E - e): rank == id. */
+static void ranked_row(float *pr, int E) {
+ float sum = 0; for (int e = 0; e < E; e++) { pr[e] = (float)(E - e); sum += pr[e]; }
+ for (int e = 0; e < E; e++) pr[e] /= sum;
+}
+
+static int has(const int *idx, int K, int e) { for (int k = 0; k < K; k++) if (idx[k] == e) return 1; return 0; }
+
+int main(void) {
+ enum { E = 16, K = 4 };
+ float pr[E]; uint8_t keep[E]; int idx[K]; float val[K]; RouteStats st;
+ ranked_row(pr, E); memset(keep, 1, sizeof keep);
+
+ /* nothing resident: the plain top-K, no substitution, full agreement, zero KL */
+ memset(g_lvl, 0, sizeof g_lvl); memset(&st, 0, sizeof st);
+ route_select(pr, keep, E, K, 2, 12, 0.f, 1.f, lvl_table, NULL, idx, val, &st);
+ check(idx[0]==0 && idx[1]==1 && idx[2]==2 && idx[3]==3, "no_resident_expert_means_the_true_top_k");
+ check(st.swaps == 0 && st.agree_hit == K && st.agree_tot == K, "no_substitution_reports_full_agreement");
+ check(st.kl_n == 1 && st.kl_sum == 0.0, "identical_choice_has_zero_kl");
+ check(val[0] == pr[0] && val[3] == pr[3], "val_carries_the_raw_mass_of_each_chosen_expert");
+
+ /* a RAM-resident expert inside the window replaces the unresident tail */
+ memset(g_lvl, 0, sizeof g_lvl); g_lvl[7] = 1; memset(&st, 0, sizeof st);
+ route_select(pr, keep, E, K, 2, 12, 0.f, 1.f, lvl_table, NULL, idx, val, &st);
+ check(idx[0]==0 && idx[1]==1, "true_top_j_is_taken_first");
+ check(idx[2]==7 && idx[3]==2, "resident_expert_in_window_fills_before_the_unresident_ranking");
+ check(st.swaps == 1 && st.swaps_vram == 0 && st.agree_hit == 3, "one_substitution_is_counted_once_and_not_as_vram");
+ check(st.kl_sum > 0.0, "a_substitution_shows_up_as_positive_kl");
+
+ /* the sacred top-J is taken even when a lower rank is resident and it is not */
+ memset(g_lvl, 0, sizeof g_lvl); g_lvl[2] = 1; g_lvl[3] = 1; g_lvl[4] = 1; g_lvl[5] = 1; memset(&st, 0, sizeof st);
+ route_select(pr, keep, E, K, 2, 12, 0.f, 1.f, lvl_table, NULL, idx, val, &st);
+ check(idx[0]==0 && idx[1]==1 && idx[2]==2 && idx[3]==3, "unresident_top_j_is_never_displaced");
+ check(st.swaps == 0, "resident_experts_already_in_the_top_k_are_not_substitutions");
+
+ /* VRAM outranks RAM inside the window, whatever their rank order */
+ memset(g_lvl, 0, sizeof g_lvl); g_lvl[5] = 1; g_lvl[9] = 2; memset(&st, 0, sizeof st);
+ route_select(pr, keep, E, K, 2, 12, 0.f, 1.f, lvl_table, NULL, idx, val, &st);
+ check(idx[2]==9 && idx[3]==5, "vram_resident_fills_before_ram_resident");
+ check(st.swaps == 2 && st.swaps_vram == 1, "swaps_to_vram_are_counted_separately");
+
+ /* residency outside the top-M window is ignored */
+ memset(g_lvl, 0, sizeof g_lvl); g_lvl[12] = 2; g_lvl[15] = 1; memset(&st, 0, sizeof st);
+ route_select(pr, keep, E, K, 2, 12, 0.f, 1.f, lvl_table, NULL, idx, val, &st);
+ check(!has(idx, K, 12) && !has(idx, K, 15) && idx[2]==2 && idx[3]==3, "resident_expert_past_the_window_is_not_chosen");
+
+ /* ROUTE_J=0: even rank 0 may be replaced; ROUTE_J=K: nothing may */
+ memset(g_lvl, 0, sizeof g_lvl); g_lvl[6] = 1; g_lvl[7] = 1; g_lvl[8] = 1; g_lvl[9] = 1; memset(&st, 0, sizeof st);
+ route_select(pr, keep, E, K, 0, 12, 0.f, 1.f, lvl_table, NULL, idx, val, &st);
+ check(idx[0]==6 && idx[1]==7 && idx[2]==8 && idx[3]==9 && st.swaps == 4, "route_j_zero_lets_residents_take_every_slot");
+ memset(&st, 0, sizeof st);
+ route_select(pr, keep, E, K, K, 12, 0.f, 1.f, lvl_table, NULL, idx, val, &st);
+ check(idx[0]==0 && idx[3]==3 && st.swaps == 0, "route_j_equal_to_k_disables_substitution");
+
+ /* the group mask is respected: an excluded expert is never ranked or chosen */
+ memset(g_lvl, 0, sizeof g_lvl); g_lvl[5] = 2; memset(&st, 0, sizeof st);
+ keep[1] = 0; keep[5] = 0;
+ route_select(pr, keep, E, K, 2, 12, 0.f, 1.f, lvl_table, NULL, idx, val, &st);
+ check(!has(idx, K, 1) && !has(idx, K, 5), "masked_experts_are_never_chosen_even_when_resident");
+ check(idx[0]==0 && idx[1]==2 && idx[2]==3 && idx[3]==4, "masked_ranking_shifts_the_true_top_k");
+ memset(keep, 1, sizeof keep);
+
+ /* ROUTE_ALPHA scales only the substitute's mass, before moe() renormalises */
+ memset(g_lvl, 0, sizeof g_lvl); g_lvl[7] = 1; memset(&st, 0, sizeof st);
+ route_select(pr, keep, E, K, 2, 12, 0.f, 0.5f, lvl_table, NULL, idx, val, &st);
+ check(idx[2]==7 && val[2] == pr[7]*0.5f && val[0] == pr[0] && val[3] == pr[2], "alpha_halves_the_substitute_and_nothing_else");
+
+ /* ROUTE_P: the window is the mass, not M. Ranks 0..4 hold 70/136 = 0.515 of the
+ * mass, so P=0.5 closes the window at rank 4: rank 4 is inside, rank 5 outside. */
+ memset(g_lvl, 0, sizeof g_lvl); g_lvl[4] = 1; g_lvl[5] = 1; memset(&st, 0, sizeof st);
+ route_select(pr, keep, E, K, 2, 12, 0.5f, 1.f, lvl_table, NULL, idx, val, &st);
+ check(has(idx, K, 4) && !has(idx, K, 5), "route_p_bounds_the_window_by_cumulative_mass");
+
+ /* fewer eligible experts than K: the remainder is marked -1, not garbage */
+ memset(g_lvl, 0, sizeof g_lvl); memset(&st, 0, sizeof st);
+ memset(keep, 0, sizeof keep); keep[3] = 1; keep[9] = 1;
+ route_select(pr, keep, E, K, 2, 12, 0.f, 1.f, lvl_table, NULL, idx, val, &st);
+ check(idx[0]==3 && idx[1]==9 && idx[2]==-1 && idx[3]==-1 && st.slots == 2, "short_candidate_list_pads_with_minus_one");
+
+ if (fails) { printf("test_qwen36_cache_route: %d fallimenti\n", fails); return 1; }
+ printf("test_qwen36_cache_route: ok\n");
+ return 0;
+}
diff --git a/docs/CACHE_ROUTE.md b/docs/CACHE_ROUTE.md
index 78f47e9c..30440988 100644
--- a/docs/CACHE_ROUTE.md
+++ b/docs/CACHE_ROUTE.md
@@ -9,6 +9,24 @@ still rank inside top-`M`.
This is **routing-side** (can change which experts run). Complementary to
**PILOT** (next-layer *prefetch* of weights; does not change expert IDs).
+## Engines and residency levels
+
+| engine | "resident" means | levels |
+|---|---|---|
+| `colibri` (GLM-5.2) | in the pin set or the RAM LRU cache | one |
+| `qwen36` | in the VRAM expert tier (CUDA / Vulkan), else in the RAM LRU cache | two: VRAM first, then RAM |
+
+In `qwen36` a slot past the sacred top-`J` is filled, inside the top-`M`
+window, by the highest-ranked VRAM-resident expert, then the highest-ranked
+RAM-resident one, then the plain ranking. With no tier enabled the VRAM level
+is empty and the lever degrades to the single-level GLM behaviour. The knobs
+and meters are the same in both engines; the `qwen36` footer additionally
+splits the swap count into `N to VRAM`.
+
+The only substitutable slots are ranks `J..K-1`. On a model that routes
+top-2, the default `ROUTE_J=2` leaves nothing to substitute and the footer
+reports `swap 0.0%`; lower `ROUTE_J` to trade agreement for hits.
+
## Flags
| Env | Default | Meaning |
@@ -42,6 +60,7 @@ Footer / serve `STAT` when enabled:
- `route_agree` — |chosen ∩ true top-K| / K
- `route_kl` — mass KL (true top-K vs chosen)
- `hit N%` — expert cache hit (disk residency)
+- `N to VRAM` (`qwen36` only) — of the swaps, how many landed on a tier-resident expert
## A/B vs PILOT
@@ -61,5 +80,10 @@ CACHE_ROUTE=1 PILOT=1 ...
**Routing-only** + telemetry + this note. Does **not** require CUDA/fuse/device-tier
patches — CPU streaming + pin/LRU is enough to A/B the lever against PILOT / #119.
+The `qwen36` port keeps that shape: `route_select()` is a pure function of the
+softmax row, the group mask and a residency callback, pinned by
+`tests/test_qwen36_cache_route.c` without a model. With the lever unset the
+router runs the original top-K loop and emits byte-identical token ids.
+
Treat as experimental until quality gates (e.g. `./coli bench`) pass; do not
default `CACHE_ROUTE=1`.
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index b3d36252..48f1fc22 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -15,7 +15,7 @@ what follows, but the sister engines read their own:
| `colibri` | `c/colibri.c` | everything below except the three sections named for another engine |
| `kimi_k3` | `c/kimi_k3.c` | the `K3_*` family — see [Kimi K3 engine](#kimi-k3-engine-kimi_k3) |
| `inkling` | `c/inkling.c` | `INK_*`, plus `CTX_MAX`, `PIN_N`, `REP_PEN`, `GPU_DEV`, `NOGPU` — see [Inkling engine](#inkling-engine-inkling) |
-| `qwen36` | `c/qwen36.c` | `QWEN_*`, `Q36_*`, and its dense/CUDA-tier controls — see [Qwen3.6 engine](#qwen36-engine-qwen36) |
+| `qwen36` | `c/qwen36.c` | `QWEN_*`, `Q36_*`, its dense/CUDA-tier controls, and the `CACHE_ROUTE` family (VRAM tier over RAM cache) — see [Qwen3.6 engine](#qwen36-engine-qwen36) |
| `qwen38` | `c/qwen38.c` | `Q38_MAXT`, `Q38_EOS`, `Q38_NATIVE_FP8`, `Q38_NATIVE_BF16`, `Q38_PREFILL_BATCH`, `COLI_TIMERS` — see [Qwen3.8 engine](#qwen38-engine-qwen38) |
| `olmoe` | `c/olmoe.c` | `HOT`, `WIDE`, `SMOOTH`, `CONF_LIMIT`, `MAX_NEW`, `CHAT`, `EXPERT_DROP`, `WARMUP` — see [OLMoE engine](#olmoe-engine-olmoe) |
| `deepseek_v4` | `c/deepseek_v4.c` | `CTX`, the `V4_*` / `DSV4_*` families and the two `COLI_CUDA_*_BATCH` gates — see [DeepSeek V4 engine](#deepseek-v4-engine-deepseek_v4); note that the CUDA section below describes `colibri.c` knobs (`COLI_CUDA`, `CUDA_DENSE`, ...) which the V4 engine does not read — its GPU switch is `DSV4_CUDA` |
@@ -103,12 +103,12 @@ Format: `VAR` — default — effect.
| `COUPLE` | unset | Path to a coupling-score file driving cross-layer expert prefetch (#176). When set, `couple_load` reads it. |
| `COUPLE_K` | `8` | Top-K coupled experts per layer when `COUPLE` is set. |
| `COUPLE_D` | `1` | Coupling lookahead depth (`1` or `2`) when `COUPLE` is set. |
-| `CACHE_ROUTE` | `0` (off) | Opt-in max-rank cache-aware MoE routing (pin∪LRU prefer within top-M). See [CACHE_ROUTE.md](CACHE_ROUTE.md). |
+| `CACHE_ROUTE` | `0` (off) | Opt-in max-rank cache-aware MoE routing (pin∪LRU prefer within top-M). Also read by `qwen36`, where the VRAM tier outranks the RAM cache. See [CACHE_ROUTE.md](CACHE_ROUTE.md). |
| `ROUTE_J` | `2` | Sacred top ranks always taken when `CACHE_ROUTE=1`. |
| `ROUTE_M` | `12` | Max-rank window for resident preference when `CACHE_ROUTE=1`. |
| `ROUTE_P` | `0` | Cumulative mass window for CACHE_ROUTE (`0` = fixed M). |
| `ROUTE_ALPHA` | `1` | Scale gate mass of substituted experts before renorm (`1` = off). |
-| `ROUTE_AGREE` | auto | Overlap% + KL vs true top-K; auto-on when `CACHE_ROUTE=1`. |
+| `ROUTE_AGREE` | auto | Overlap% + KL vs true top-K; auto-on when `CACHE_ROUTE=1`. Alone it changes nothing and prints the meters (always 100% / 0). |
| `ROUTE_TRACE` | unset | If set to a path, logs every routing decision there (testing/analysis). |
| `ABSORB` | `-1` (auto: absorbed for S≤4) | MLA attention absorption mode. |
| `IDOT` | `1` | Integer dot-product kernel. `IDOT=0` uses exact f32 kernels (for A/B numerical checks). |
diff --git a/docs/qwen36.md b/docs/qwen36.md
index 32cfcdd4..b449af49 100644
--- a/docs/qwen36.md
+++ b/docs/qwen36.md
@@ -104,6 +104,25 @@ overlap, not this kernel. `tests/test_expert_ffn` holds the numerics.
keeps its own path: it uploads the pair-layout int4 and computes misses from
the int8 copy.
+## Cache-aware routing (`CACHE_ROUTE`, off by default)
+
+The residual misses above are the lever's target. `CACHE_ROUTE=1` ports the
+GLM engine's max-rank re-routing ([CACHE_ROUTE.md](CACHE_ROUTE.md),
+arXiv:2412.00099) to this engine with two residency levels: inside the top-`M`
+window, a slot past the sacred top-`J` prefers an expert already in the VRAM
+tier, then one in the RAM cache, then the plain ranking. It is **lossy**: it
+changes which experts run, so the semantic contract is off while it is set and
+the footer prints what it cost, `route_agree` (overlap with the true top-K)
+and `route_kl` (mass KL), next to the swap and hit rates. Unset, the router
+is the original loop and the token ids are byte-identical; `ROUTE_AGREE=1`
+alone prints the meters at 100 % / 0 without touching routing.
+
+Qwen3.6 routes top-8 (plus the shared expert), so the default `ROUTE_J=2`
+leaves six substitutable slots per token; the tiny fixture routes top-2 and
+needs `ROUTE_J<2` to show any swap at all. A/B it the way the GLM doc does:
+same prompt and seed, tok/s and hit rate against agreement and KL, and treat
+`PPL=1` on a teacher-forced reference as the quality bar.
+
## Which container?
The gs64 container carries one scale per 64-weight group instead of one per