Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

qwen36: offer the shared expert to the placer on request only (Q36_OFFER_SHEXP=1)

kreuzzelg006c540b52d4
diff --git a/c/qwen36.c b/c/qwen36.c
index 9e253b51..d4b940a7 100644
--- a/c/qwen36.c
+++ b/c/qwen36.c
@@ -3483,11 +3483,20 @@ static void trunk_offer_dense(Model *m){
size_t bq = qdw_bytes(&l->q), bk = qdw_bytes(&l->k), bv = qdw_bytes(&l->v), bo = qdw_bytes(&l->o);
if (bq && bk && bv && bo) qt_trunk_offer("attnproj", i, bq + bk + bv + bo);
}
- for (int i = 0; i < c->n_layers; i++) {
- Layer *l = &m->L[i];
- size_t bg = qdw_bytes(&l->sh_g), bu = qdw_bytes(&l->sh_u), bd = qdw_bytes(&l->sh_d);
- if (bg && bu && bd) qt_trunk_offer("shexp", i, bg + bu + bd);
- }
+ /* The shared expert is offered only on request (Q36_OFFER_SHEXP=1). On the
+ * card it is 120 synchronous small GEMVs per token that sit between
+ * qt_issue and qt_take, where on the CPU it hides behind the expert group:
+ * measured on the 35B, 3070, shared 3.7-4.0 ms/token on the CPU against
+ * 6.3 on the same card and 11-12.7 with layers on a slower second card
+ * (docs/qwen36-cuda-tier.md). A hand-written COLI_PLACE naming shexp is
+ * still obeyed when the offer is made. */
+ { const char *so = getenv("Q36_OFFER_SHEXP");
+ if (so && *so == '1')
+ for (int i = 0; i < c->n_layers; i++) {
+ Layer *l = &m->L[i];
+ size_t bg = qdw_bytes(&l->sh_g), bu = qdw_bytes(&l->sh_u), bd = qdw_bytes(&l->sh_d);
+ if (bg && bu && bd) qt_trunk_offer("shexp", i, bg + bu + bd);
+ } }
}
/* After qt_init decided: upload what was placed, keep the handles in the
* Layer. Every matrix falls back on its own, so a failed upload costs one
diff --git a/c/tests/test_qwen36_trunk_dense.c b/c/tests/test_qwen36_trunk_dense.c
index 0640990a..2c68c5d8 100644
--- a/c/tests/test_qwen36_trunk_dense.c
+++ b/c/tests/test_qwen36_trunk_dense.c
@@ -102,6 +102,7 @@ int main(void) {
setenv("COLI_CUDA", "1", 1); setenv("COLI_GPUS", "0", 1);
setenv("QT_NO_WARMSTART", "1", 1); setenv("HEAT_FILE", "", 1);
setenv("COLI_PLACE", "", 1); /* "" == unset == auto */
+ setenv("Q36_OFFER_SHEXP", "1", 1); /* the shared expert is offered on request only; this test is about the offer */
setenv("CUDA_EXPERT_GB", "1", 1); /* room for everything */
unsetenv("COLI_DENSE_I8");
/* the GEMV from VRAM is compared with the CPU's f32-activation kernel, the