Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(qwen36): CUDA VRAM expert tier — heat-based placement across GPUs via the shared CUDA backend
kreuzzelg34deee396624
diff --git a/c/Makefile b/c/Makefileindex 7196ccac..f5505c9d 100644--- a/c/Makefile+++ b/c/Makefile@@ -897,19 +897,34 @@ olmoe$(EXE): olmoe.c st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unico$(CC) $(NOCUDA_CFLAGS) olmoe.c -o olmoe$(EXE) $(NOCUDA_LDFLAGS)# Qwen3.6-35B-A3B engine (hybrid Gated Attention + Gated DeltaNet + streaming-# MoE). CPU-only in this target; the optional CUDA expert tier is a separate-# follow-up (see docs/qwen36-phase01.md).-# NOCUDA_*: this file contains no CUDA, so building it with -DCOLI_CUDA and-# linking -lcudart only made `make qwen36 CUDA=1` depend on a toolkit it-# never calls. Same shape as olmoe. (The CUDA expert tier is #713, which-# adds its own sources and takes the CUDA flags back.)-qwen36$(EXE): qwen36.c st.h json.h compat.h- $(CC) $(NOCUDA_CFLAGS) qwen36.c -o qwen36$(EXE) $(NOCUDA_LDFLAGS)+# MoE). With CUDA=1 the optional VRAM expert tier (qwen36_tier.c) is compiled+# in and reuses the shared CUDA backend; without it the engine is CPU-only+# (qwen36_tier.h provides inline stubs).+#+# The flags follow the same switch. Without CUDA=1 this target keeps the+# NOCUDA_* flags it got when the engine had no CUDA at all: compiling+# -DCOLI_CUDA and linking -lcudart for a build that calls neither only made+# `make qwen36` depend on a toolkit it never touches. With CUDA=1 the tier is+# real CUDA and takes the normal flags back.+ifeq ($(CUDA),1)+QWEN36_TIER_SRC = qwen36_tier.c+QWEN36_CFLAGS = $(CFLAGS)+QWEN36_LDFLAGS = $(LDFLAGS)+else+QWEN36_TIER_SRC =+QWEN36_CFLAGS = $(NOCUDA_CFLAGS)+QWEN36_LDFLAGS = $(NOCUDA_LDFLAGS)+endif+qwen36$(EXE): qwen36.c qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)+ $(CC) $(QWEN36_CFLAGS) qwen36.c $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o qwen36$(EXE) $(QWEN36_LDFLAGS)# Context-size gates: KV layout, growth across requests, and the attention# capacity. Includes qwen36.c directly, so no model file is needed.-tests/test_qwen36_ctx$(EXE): tests/test_qwen36_ctx.c qwen36.c st.h json.h compat.h- $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+# Same tier sources as the engine: the test includes qwen36.c, so with CUDA=1+# it needs qwen36_tier.c and the backend object too (without CUDA, the header's+# inline stubs cover it and both are empty).+tests/test_qwen36_ctx$(EXE): tests/test_qwen36_ctx.c qwen36.c qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)+ $(CC) $(CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(LDFLAGS)inkling$(EXE): inkling.c st.h json.h compat.h $(INK_CUDA_OBJ) $(METAL_OBJ)$(CC) $(CFLAGS) inkling.c $(INK_CUDA_OBJ) $(METAL_OBJ) -o inkling$(EXE) $(LDFLAGS)diff --git a/c/qwen36.c b/c/qwen36.cindex 8213c228..6689fd98 100644--- a/c/qwen36.c+++ b/c/qwen36.c@@ -59,6 +59,7 @@ static int qwen36_max_ctx(void) {#endif#include "st.h"#include "json.h" /* tokenizer.json parsing (reuse minimal parser) */+#include "qwen36_tier.h" /* optional transparent Vulkan compute backend for MoE experts */#ifdef _WIN32#include <windows.h>@@ -583,7 +584,7 @@ typedef struct {} Layer;/* ---------- LRU expert cache (int8 weights + per-row float scales) ---------- */-typedef struct { int eid; int pinned; int is_int4; int8_t *g, *u, *d; float *gs, *us, *ds; uint64_t used; } Slot;+typedef struct { int eid; int pinned; int is_int4; int8_t *g, *u, *d; uint8_t *g4, *u4, *d4; float *gs, *us, *ds; uint64_t used; } Slot;typedef struct { Slot *slots; int n, cap; } LCache;typedef struct {@@ -651,6 +652,7 @@ static double g_tm_dec[6], g_tm_pre[6]; /* 0=deltanet 1=attention 2=moe_totalstatic long g_tm_dec_tokens = 0, g_tm_pre_tokens = 0;static double tm_now(void){ struct timespec ts; clock_gettime(CLOCK_MONOTONIC,&ts); return ts.tv_sec*1e3 + ts.tv_nsec/1e6; }static int tm_on(void){ if(g_timers<0){ const char *e=getenv("COLI_TIMERS"); g_timers = (e && *e=='1'); } return g_timers; }+double g_qt_iss=0, g_qt_cpu=0, g_qt_tak=0; /* QTIER-Phasen (Decode) */double g_dn_sub[4]; /* DN: proj, conv+split, l2n+rec, norm+out */double g_tm_step=0; /* step() total (decode) */static double g_tm_win_moe=0; static int g_tm_win_n=0;@@ -682,6 +684,9 @@ static void tm_report(void){if(g_dn_sub[0]+g_dn_sub[1]+g_dn_sub[2]+g_dn_sub[3]>0)fprintf(stderr,"[timers] dn-sub: proj %.1f | conv %.1f | l2n+rec %.1f | norm+out %.1f ms/token\n",g_dn_sub[0]/g_tm_dec_tokens,g_dn_sub[1]/g_tm_dec_tokens,g_dn_sub[2]/g_tm_dec_tokens,g_dn_sub[3]/g_tm_dec_tokens);+ if(g_qt_iss+g_qt_cpu+g_qt_tak>0)+ fprintf(stderr,"[timers] qtier: issue %.2f | cpu-miss %.2f | take %.2f ms/token\n",+ g_qt_iss/g_tm_dec_tokens, g_qt_cpu/g_tm_dec_tokens, g_qt_tak/g_tm_dec_tokens);fprintf(stderr,"[timers] prefill: %ld tokens dn=%.0f attn=%.0f moe=%.0f(sh=%.0f rt=%.0f) head=%.0f ms\n",g_tm_pre_tokens,g_tm_pre[0],g_tm_pre[1],g_tm_pre[2],g_tm_pre[3],g_tm_pre[4],g_tm_pre[5]);}@@ -1197,6 +1202,7 @@ static void slot_ensure_allocated(Model *m, Slot *s) {s->ds = s_block + 2*scale_count_gu(c);s->pinned = 0;s->is_int4 = 0;+ s->g4 = s->u4 = s->d4 = NULL; /* packed int4 (allocated on int4 load if GPU int4 active) */}static void load_expert_merged(Model *m, int layer, int eid, Slot *s) {@@ -1234,14 +1240,29 @@ static void load_expert_merged(Model *m, int layer, int eid, Slot *s) {s->g[i] = v;}s->is_int4 = 1;- /* The packed int4 bytes are NOT kept here. This engine only ever reads- * the unpacked int8 copy above, so retaining them doubled expert-cache- * RSS on the recommended gs64 container for nothing. They are the upload- * source for the CUDA expert tier and come back with it (#713), which is- * where they are actually read. */+ /* Free any previous occupant first (LRU slot reuse). */+ free(s->g4); free(s->u4); free(s->d4); s->g4 = s->u4 = s->d4 = NULL;+ /* Keep the packed int4 bytes alongside the unpacked int8 copy only when+ * the CUDA tier is actually running: they are its upload source, and+ * they let slot_ensure_int8() rematerialize an evicted expert whose+ * int8 copy the warmstart freed. Without the tier nothing ever reads+ * them, and keeping them would add ~50% to expert-cache RSS on the+ * recommended gs64 container -- so qt_ready() gates the allocation.+ * Under CUDA=0 that is an inline `return 0` and this costs nothing. */+ if (qt_ready()) {+ int64_t gp = ng / 2, up = ng / 2, dp = nd / 2; /* gate/up/down packed sizes */+ s->g4 = (uint8_t *)malloc((size_t)gp);+ s->u4 = (uint8_t *)malloc((size_t)up);+ s->d4 = (uint8_t *)malloc((size_t)dp);+ if (!s->g4 || !s->u4 || !s->d4) { fprintf(stderr, "OOM int4-packed %s\n", nm); exit(1); }+ memcpy(s->g4, raw, (size_t)gp);+ memcpy(s->u4, raw + gp, (size_t)up);+ memcpy(s->d4, raw + gp + up, (size_t)dp);+ }free(raw);} else {s->is_int4 = 0;+ free(s->g4); free(s->u4); free(s->d4); s->g4 = s->u4 = s->d4 = NULL;st_read_raw(&m->S, nm, s->g, 1);}st_read_f32(&m->S, qsnm, s->gs, 0);@@ -1262,6 +1283,31 @@ static int container_is_int4(Model *m) {return (tw->nbytes == want_w / 2) ? 1 : 0;}+/* Rematerialize a slot's int8 block from its packed int4 copy on demand+ * (~0.5 ms, no container access). Needed after the warmstart freed the int8+ * copies of VRAM-resident experts and one of them got LFRU-evicted. */+static void slot_ensure_int8(Model *m, Slot *s) {+ if (s->g || !s->g4) return;+ Cfg *c = &m->c;+ int64_t ng = (int64_t)c->inter * c->hidden, nd = (int64_t)c->hidden * c->inter;+ int8_t *w = malloc((size_t)(ng + ng + nd));+ if (!w) { fprintf(stderr, "OOM slot_ensure_int8\n"); exit(1); }+ const uint8_t *src4[3] = { s->g4, s->u4, s->d4 };+ int64_t lens[3] = { ng, ng, nd };+ int8_t *dst = w;+ for (int t = 0; t < 3; t++) {+ const uint8_t *p = src4[t];+ for (int64_t i = 0; i < lens[t]; i += 2) {+ uint8_t b = p[i >> 1];+ int8_t lo = (int8_t)(b & 0xF); if (lo & 8) lo -= 16;+ int8_t hi = (int8_t)((b >> 4) & 0xF); if (hi & 8) hi -= 16;+ dst[i] = lo; dst[i + 1] = hi;+ }+ dst += lens[t];+ }+ s->g = w; s->u = w + ng; s->d = w + ng + ng;+}+static void expert_get(Model *m, int layer, int eid, Slot **out) {LCache *lc = &m->cache[layer];pthread_mutex_lock(&g_pilot_mx);@@ -1572,9 +1618,59 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {for (int kk = 0; kk < K; kk++) if (idx[kk] >= 0) freq_l[idx[kk]]++;}const float *xs = x + (int64_t)s*D;- {+ int shared_done = 0;+ if (qt_ready()) {+ /* CUDA expert tier: run the resident experts as async groups on+ * all devices, compute the misses on the CPU (overlapped), then+ * collect the GPU results. */+ for (int kk = 0; kk < K; kk++) {+ Slot *e; expert_get(m, layer, idx[kk], &e);+ if (e->g4) qt_note(layer, idx[kk], e->g4, e->u4, e->d4, e->gs, e->us, e->ds);+ }+ double _q0 = tm_on()? tm_now():0;+ uint32_t qmask = qt_issue(layer, idx, K, xs);+ double _q1 = tm_on()? tm_now():0;+ for (int kk = 0; kk < K; kk++) {+ if (qmask & (1u<<kk)) continue;+ Slot *e; expert_get(m, layer, idx[kk], &e);+ slot_ensure_int8(m, e);+ matmul_qe(g, xs, e->g, e->gs, D, I);+ matmul_qe(u, xs, e->u, e->us, D, I);+ for (int i = 0; i < I; i++) { float gv = g[i]; g[i] = (gv / (1.f + expf(-gv))) * u[i]; }+ matmul_qe(hh, g, e->d, e->ds, I, D);+ float w = val[kk]; float *os = out + (int64_t)s*D;+ for (int d = 0; d < D; d++) os[d] += w * hh[d];+ }+ /* Compute the shared expert NOW so it overlaps with the GPU+ * groups; the common block below is skipped. */+ {+ double _ts2 = tm_on() ? tm_now() : 0.0;+ int Ish = c->shared_inter;+ matmul_d(sh, xs, l->sh_g, 1, D, Ish);+ matmul_d(shu, xs, l->sh_u, 1, D, Ish);+ for (int i = 0; i < Ish; i++) { float sv = sh[i]; sh[i] = (sv / (1.f + expf(-sv))) * shu[i]; }+ matmul_d(shd, sh, l->sh_d, 1, Ish, D);+ float sgate = 1.f;+ if (l->sh_gate) {+ float sg = 0.f; const float *wg = l->sh_gate;+ for (int i = 0; i < D; i++) sg += xs[i] * wg[i];+ sgate = 1.f / (1.f + expf(-sg));+ }+ float *os = out + (int64_t)s*D;+ for (int d = 0; d < D; d++) os[d] += sgate * shd[d];+ if (tm_on()) tm_add(S, 3, tm_now()-_ts2);+ }+ shared_done = 1;+ double _q2 = tm_on()? tm_now():0;+ qt_take(qmask, val, K, out + (int64_t)s*D);+ if (tm_on() && S==1) {+ extern double g_qt_iss, g_qt_cpu, g_qt_tak;+ g_qt_iss += _q1-_q0; g_qt_cpu += _q2-_q1; g_qt_tak += tm_now()-_q2;+ }+ } else {for (int kk = 0; kk < K; kk++) {Slot *e; expert_get(m, layer, idx[kk], &e);+ slot_ensure_int8(m, e);matmul_qe(g, xs, e->g, e->gs, D, I);matmul_qe(u, xs, e->u, e->us, D, I);for (int i = 0; i < I; i++) { float gv = g[i]; g[i] = (gv / (1.f + expf(-gv))) * u[i]; }@@ -1585,6 +1681,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {}}/* shared expert (SwiGLU), sigmoid-gated by shared_expert_gate */+ if (shared_done) { if (0) goto shared_skip_dummy; shared_skip_dummy: continue; }double _ts = tm_on() ? tm_now() : 0.0;int Ish = c->shared_inter;matmul_d(sh, xs, l->sh_g, 1, D, Ish);@@ -1919,9 +2016,14 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S) {if (idx[b] >= 0 && (idx[a] < 0 || idx[a] > idx[b])) { int t = idx[a]; idx[a] = idx[b]; idx[b] = t; }for (int kk = 0; kk < cand; kk++) {int eid = idx[kk]; if (eid < 0) continue;- int found = 0; pthread_mutex_lock(&g_pilot_mx); LCache *lc = &m->cache[lnext];- for (int z = 0; z < lc->n; z++) if (lc->slots[z].eid == eid) { found = 1; break; }+ int found = 0, fz = -1; pthread_mutex_lock(&g_pilot_mx); LCache *lc = &m->cache[lnext];+ for (int z = 0; z < lc->n; z++) if (lc->slots[z].eid == eid) { found = 1; fz = z; break; }pthread_mutex_unlock(&g_pilot_mx);+ /* Lookahead: RAM-resident layer-L+1 candidates go to VRAM asynchronously */+ if (found && fz >= 0 && qt_ready()) {+ Slot *ps = &lc->slots[fz];+ if (ps->g4) qt_note(lnext, eid, ps->g4, ps->u4, ps->d4, ps->gs, ps->us, ps->ds);+ }if (!found) {int gidx = lnext*E + eid;pthread_mutex_lock(&g_pilot_mx); int already_queued = m->is_queued[gidx];@@ -2307,7 +2409,55 @@ int main(int argc, char **argv) {g_qdw_n, now_s()-tq, freed/1073741824.0);}- /* coli serve mode: speak the gateway wire protocol instead of argv generation */+ /* Optional CUDA VRAM expert tier (COLI_CUDA=1): hot experts live in+ * DEVICE_LOCAL memory across the configured GPUs, misses fall back to the+ * CPU int8 path. See qwen36_tier.h. */+ if (qt_init(m.c.n_layers, m.c.n_experts, m.c.hidden, m.c.inter, cap, m.c.topk, m.c.expert_gs)) {+ fprintf(stderr, "[gpu] MoE experts -> CUDA VRAM tier\n");+ atexit(qt_shutdown);+ /* Warmstart: fill the VRAM budget BEFORE the first token (heat order+ * when HEAT_FILE exists, natural order otherwise), loading all RAM+ * slots along the way. */+ const char *nws = getenv("QT_NO_WARMSTART");+ if (!(nws && *nws=='1')) {+ /* Plan the set (heat order), then load+stage IN PARALLEL. The+ * load path is thread-safe: expert_get locks the layer cache+ * (g_pilot_mx), st_read_raw uses pread; entries are unique. */+ double t0 = now_s();+ int cap_total = m.c.n_layers * m.c.n_experts;+ int *wpl = malloc((size_t)cap_total*sizeof(int));+ int *wpe = malloc((size_t)cap_total*sizeof(int));+ int wn = qt_plan_fill(wpl, wpe, cap_total);+ /* Load ALL experts into RAM, not just the planned (VRAM) set:+ * otherwise the first touch of a CPU-fallback expert triggers a+ * ~12 ms container read in the middle of decode (measured: 139+ * ms/token on a single-GPU run). Planned ones also go to VRAM. */+ uint8_t *planned = calloc((size_t)cap_total, 1);+ for (int i = 0; i < wn; i++) planned[wpl[i]*m.c.n_experts + wpe[i]] = 1;+ int keep8 = getenv("COLI_KEEP_INT8") != NULL;+ #pragma omp parallel for schedule(dynamic, 16)+ for (int gi = 0; gi < cap_total; gi++) {+ int l = gi / m.c.n_experts, eidw = gi % m.c.n_experts;+ Slot *e; expert_get(&m, l, eidw, &e);+ if (planned[gi] && e->g4) {+ qt_note_planned(l, eidw, e->g4, e->u4, e->d4, e->gs, e->us, e->ds);+ /* The staging copy is done; free the int8 copy RIGHT AWAY+ * so it never shows up in peak RSS. On LFRU eviction+ * slot_ensure_int8() rematerializes from g4 (no container+ * access). */+ if (!keep8 && e->g) { free(e->g); e->g = e->u = e->d = NULL; }+ }+ }+ qt_fill_wait();+ free(wpl); free(wpe); free(planned);+ fprintf(stderr, "[qtier] warmstart (parallel): all %d experts in RAM (int8 only for non-residents), %d in VRAM -- %.1f s\n",+ cap_total, wn, now_s()-t0);+ }+ }++ /* coli serve mode: speak the gateway wire protocol instead of argv+ * generation. AFTER the tier init: serve sessions ride the VRAM experts+ * exactly like argv runs, and serve_loop never returns. */if (getenv("SERVE") && getenv("SERVE")[0] == '1') {if (!g_tok) { fprintf(stderr, "[serve] tokenizer.json required (put in SNAP or set TOK)\n"); return 1; }serve_loop(&m);@@ -2383,6 +2533,7 @@ int main(int argc, char **argv) {double tot = m.hits + m.miss;if (g_ttft >= 0) fprintf(stderr, "TTFT: %.2f s (time to first token)\n", g_ttft);tm_report();+ qt_stats();fprintf(stderr, "\nPEAK RSS: %.2f GB\n", rss_gb());fprintf(stderr, "Expert cache hit rate: %.1f%% (hit=%llu miss=%llu)\n", tot?100.0*m.hits/tot:0.0,(unsigned long long)m.hits, (unsigned long long)m.miss);diff --git a/c/qwen36_tier.c b/c/qwen36_tier.cnew file mode 100644index 00000000..7639c844--- /dev/null+++ b/c/qwen36_tier.c@@ -0,0 +1,446 @@+/* qwen36_tier.c -- CUDA VRAM expert tier for the qwen36 engine. See header. */+#ifdef COLI_CUDA+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <pthread.h>+#include "qwen36_tier.h"+#include "backend_cuda.h"++#define QT_MAX_DEV 8+#define QT_QCAP 48 /* upload queue depth (staging ~1.6 MB/entry) */++typedef struct {+ ColiCudaTensor *tg, *tu, *td;+ uint32_t heat;+ uint8_t resident, queued, planned;+ /* raw RAM pointers (slots are never evicted when cap==n_experts) -- lets+ * warmstart, lookahead and LFRU swaps run without an engine callback */+ const uint8_t *g4,*u4,*d4; const float *gs,*us,*ds;+} QSlot;++static struct {+ int on, nl, ne, D, Ih, topk, ndev;+ int egs; size_t sc_gu, sc_d; /* expert group size + per-matrix scale counts (gs64) */+ int dev[QT_MAX_DEV];+ size_t budget[QT_MAX_DEV], used[QT_MAX_DEV];+ size_t exp_bytes; /* estimated VRAM bytes per expert */+ QSlot *slot; /* [nl*ne] */+ pthread_mutex_t mx;+ pthread_t th;+ int th_stop;+ /* upload ring with staging copies */+ struct { int layer, eid; uint8_t *w; float *s; int v_layer, v_eid; } q[QT_QCAP];+ int qh, qt_, qn;+ pthread_cond_t cv;+ /* statistics */+ uint64_t hits[QT_MAX_DEV], miss, uploads, q_full_skips;+ /* issue state of the (single) decode thread */+ int is_cnt[QT_MAX_DEV];+ int is_k[QT_MAX_DEV][32];+ float *is_x; /* count*D input replicas per device */+ /* M3 */+ int *fill_order; int fill_cur; /* warmstart order (heat desc) */+ int issue_open; /* guard: no tensor_free while a group is in flight */+ pthread_cond_t cv_take; /* signals qt_take done + queue space */+ uint64_t tick, swaps, pf_hits, pf_notes;+ uint32_t *heat0; /* heat table loaded from HEAT_FILE */+} G;++static QSlot *qs(int layer, int eid){ return &G.slot[(size_t)layer*G.ne + eid]; }+static int home(int eid){ return eid % G.ndev; }++/* Staging: packed int4 (g|u|d) two's-complement -> offset-binary (XOR 0x88,+ * the upload format of backend_cuda fmt=2) + copy the scales (gs|us|ds). */+static void stage(uint8_t *dw, float *dsc,+ const uint8_t *g4,const uint8_t *u4,const uint8_t *d4,+ const float *gs,const float *us,const float *ds){+ size_t mb = (size_t)G.D*G.Ih/2;+ const uint64_t X=0x8888888888888888ull;+ const uint64_t *sg=(const uint64_t*)g4,*su=(const uint64_t*)u4,*sd=(const uint64_t*)d4;+ uint64_t *w0=(uint64_t*)dw,*w1=(uint64_t*)(dw+mb),*w2=(uint64_t*)(dw+2*mb);+ for(size_t i=0;i<mb/8;i++){ w0[i]=sg[i]^X; w1[i]=su[i]^X; w2[i]=sd[i]^X; }+ memcpy(dsc, gs, G.sc_gu*sizeof(float));+ memcpy(dsc+G.sc_gu, us, G.sc_gu*sizeof(float));+ memcpy(dsc+2*G.sc_gu, ds, G.sc_d *sizeof(float));+}++static void *uploader(void *arg){+ (void)arg;+ for(;;){+ pthread_mutex_lock(&G.mx);+ while(G.qn==0 && !G.th_stop) pthread_cond_wait(&G.cv,&G.mx);+ if(G.th_stop && G.qn==0){ pthread_mutex_unlock(&G.mx); return NULL; }+ int layer=G.q[G.qh].layer, eid=G.q[G.qh].eid;+ int vl=G.q[G.qh].v_layer, ve=G.q[G.qh].v_eid;+ uint8_t *w=G.q[G.qh].w; float *sc=G.q[G.qh].s;+ G.qh=(G.qh+1)%QT_QCAP; G.qn--;+ pthread_cond_broadcast(&G.cv_take); /* queue space available */+ if(ve>=0){+ /* LFRU swap: free the victim only when no group is in flight */+ while(G.issue_open && !G.th_stop) pthread_cond_wait(&G.cv_take,&G.mx);+ QSlot *v=qs(vl,ve);+ ColiCudaTensor *a=v->tg,*b=v->tu,*ct=v->td;+ v->tg=v->tu=v->td=NULL;+ pthread_mutex_unlock(&G.mx);+ if(a)coli_cuda_tensor_free(a); if(b)coli_cuda_tensor_free(b); if(ct)coli_cuda_tensor_free(ct);+ } else pthread_mutex_unlock(&G.mx);++ int dv = G.dev[home(eid)];+ size_t mb=(size_t)G.D*G.Ih/2;+ ColiCudaTensor *tg=NULL,*tu=NULL,*td=NULL;+ int ok;+ if(G.egs){+ ok = coli_cuda_tensor_upload_g(&tg, w, sc, 4, G.D, G.Ih, dv, G.egs)+ && coli_cuda_tensor_upload_g(&tu, w+mb, sc+G.sc_gu, 4, G.D, G.Ih, dv, G.egs)+ && coli_cuda_tensor_upload_g(&td, w+2*mb, sc+2*G.sc_gu, 4, G.Ih, G.D, dv, G.egs);+ } else {+ ok = coli_cuda_tensor_upload(&tg, w, sc, 2, G.D, G.Ih, dv)+ && coli_cuda_tensor_upload(&tu, w+mb, sc+G.Ih, 2, G.D, G.Ih, dv)+ && coli_cuda_tensor_upload(&td, w+2*mb, sc+2*G.Ih, 2, G.Ih, G.D, dv);+ }+ free(w); free(sc);+ pthread_mutex_lock(&G.mx);+ QSlot *s=qs(layer,eid);+ if(ok){ s->tg=tg; s->tu=tu; s->td=td; s->resident=1; G.uploads++; }+ else { int hd=home(eid); G.used[hd]-=G.exp_bytes;+ G.budget[hd]=G.used[hd]; /* device genuinely full: stop trying */ }+ s->queued=0;+ pthread_mutex_unlock(&G.mx);+ }+}++int qt_init(int nl, int ne, int D, int Ih, int cap, int topk, int expert_gs){+ const char *e=getenv("COLI_CUDA");+ if(!(e && *e=='1')) return 0;+ if(cap != ne){+ fprintf(stderr,"[qtier] cap=%d != n_experts=%d -> tier disabled (needs full RAM residency)\n",cap,ne);+ return 0;+ }+ if(topk>32){ fprintf(stderr,"[qtier] topk>32 unsupported\n"); return 0; }+ memset(&G,0,sizeof G);+ G.nl=nl; G.ne=ne; G.D=D; G.Ih=Ih; G.topk=topk;++ /* devices: COLI_GPUS="0,1" (default: 0,1 when present, else 0) */+ const char *gl=getenv("COLI_GPUS");+ char buf[128]; snprintf(buf,sizeof buf,"%s", gl?gl:"0,1");+ for(char *t=strtok(buf,","); t && G.ndev<QT_MAX_DEV; t=strtok(NULL,","))+ G.dev[G.ndev++]=atoi(t);+ if(!coli_cuda_init(G.dev,G.ndev)){ fprintf(stderr,"[qtier] coli_cuda_init failed -> CPU path\n"); return 0; }+ int have=coli_cuda_device_count();+ if(have<G.ndev){ G.ndev=have; }+ if(G.ndev<1){ fprintf(stderr,"[qtier] no CUDA devices -> CPU path\n"); return 0; }++ /* per-device budget: CUDA_EXPERT_GB, or auto = free minus 1 GB headroom.+ * Scale counts follow the container: per-row (expert_gs=0) or grouped+ * (gs64: [O, ceil(I/gs)] per projection). */+ G.egs = expert_gs;+ G.sc_gu = expert_gs ? (size_t)Ih * ((D + expert_gs - 1)/expert_gs) : (size_t)Ih;+ G.sc_d = expert_gs ? (size_t)D * ((Ih + expert_gs - 1)/expert_gs) : (size_t)D;+ G.exp_bytes = 3ull*D*Ih/2 + (2*G.sc_gu+G.sc_d)*sizeof(float) + 4096; /* + allocation slack */+ const char *bg=getenv("CUDA_EXPERT_GB");+ for(int i=0;i<G.ndev;i++){+ size_t freeb=0,totb=0; coli_cuda_mem_info(G.dev[i],&freeb,&totb);+ size_t b = (bg && strcmp(bg,"auto") && atof(bg)>0)+ ? (size_t)(atof(bg)*1024.0*1024.0*1024.0)+ : (freeb>(1ull<<30) ? freeb-(1ull<<30) : 0);+ G.budget[i]=b;+ fprintf(stderr,"[qtier] dev %d: %.1f GB free, budget %.1f GB (~%zu experts)\n",+ G.dev[i], freeb/1073741824.0, b/1073741824.0, b/G.exp_bytes);+ }+ G.slot=calloc((size_t)nl*ne,sizeof(QSlot));+ G.is_x=malloc((size_t)32*D*sizeof(float));+ if(!G.slot||!G.is_x) return 0;+ /* load learned heat (HEAT_FILE): warmstart order + initial values */+ const char *hf=getenv("HEAT_FILE");+ if(hf){+ FILE *f=fopen(hf,"rb");+ if(f){+ uint32_t hdr[3]={0,0,0};+ if(fread(hdr,4,3,f)==3 && hdr[0]==0x51544831u && hdr[1]==(uint32_t)nl && hdr[2]==(uint32_t)ne){+ G.heat0=malloc((size_t)nl*ne*4);+ if(G.heat0 && fread(G.heat0,4,(size_t)nl*ne,f)==(size_t)nl*ne){+ for(size_t i=0;i<(size_t)nl*ne;i++) G.slot[i].heat=G.heat0[i]>>1; /* decay */+ fprintf(stderr,"[qtier] HEAT_FILE loaded: %s\n",hf);+ } else { free(G.heat0); G.heat0=NULL; }+ }+ fclose(f);+ }+ }+ pthread_mutex_init(&G.mx,NULL); pthread_cond_init(&G.cv,NULL); pthread_cond_init(&G.cv_take,NULL);+ if(pthread_create(&G.th,NULL,uploader,NULL)!=0) return 0;+ G.on=1;+ fprintf(stderr,"[qtier] CUDA VRAM expert tier active: %d device(s), %.2f MB/expert\n",+ G.ndev, G.exp_bytes/1048576.0);+ return 1;+}++int qt_ready(void){ return G.on; }++/* Is (layer,eid) currently VRAM-resident? (used to free RAM-side int8 copies) */+int qt_is_resident(int layer,int eid){+ if(!G.on) return 0;+ pthread_mutex_lock(&G.mx);+ int r = qs(layer,eid)->resident;+ pthread_mutex_unlock(&G.mx);+ return r;+}++/* internal, G.mx held: enqueue one upload. victim=-1: plain upload (budget is+ * reserved here); victim>=0: LFRU swap (budget neutral). */+static int enqueue_locked(int layer,int eid,int v_layer,int v_eid,int reserved){+ QSlot *s=qs(layer,eid);+ if(s->resident||s->queued||!s->g4) return 0;+ if(G.qn>=QT_QCAP){ G.q_full_skips++; return 0; }+ int hd=home(eid);+ if(!reserved && v_eid<0 && G.used[hd]+G.exp_bytes>G.budget[hd]) return 0;+ size_t mb=(size_t)G.D*G.Ih/2;+ uint8_t *w=malloc(3*mb); float *sc=malloc((2*G.sc_gu+G.sc_d)*sizeof(float));+ if(!w||!sc){ free(w); free(sc); return 0; }+ if(!reserved && v_eid<0) G.used[hd]+=G.exp_bytes;+ s->queued=1;+ stage(w,sc,s->g4,s->u4,s->d4,s->gs,s->us,s->ds);+ G.q[G.qt_].layer=layer; G.q[G.qt_].eid=eid; G.q[G.qt_].w=w; G.q[G.qt_].s=sc;+ G.q[G.qt_].v_layer=v_layer; G.q[G.qt_].v_eid=v_eid;+ G.qt_=(G.qt_+1)%QT_QCAP; G.qn++;+ pthread_cond_signal(&G.cv);+ return 1;+}++void qt_note(int layer,int eid,+ const uint8_t *g4,const uint8_t *u4,const uint8_t *d4,+ const float *gs,const float *us,const float *ds){+ if(!G.on || !g4) return;+ QSlot *s=qs(layer,eid);+ pthread_mutex_lock(&G.mx);+ if(!s->g4){ s->g4=g4; s->u4=u4; s->d4=d4; s->gs=gs; s->us=us; s->ds=ds; }+ if(s->heat<0xFFFFFFFFu) s->heat++;+ enqueue_locked(layer,eid,-1,-1,0);+ pthread_mutex_unlock(&G.mx);+}++/* blocking variant for the warmstart (waits for queue space). */+void qt_note_block(int layer,int eid,+ const uint8_t *g4,const uint8_t *u4,const uint8_t *d4,+ const float *gs,const float *us,const float *ds){+ if(!G.on || !g4) return;+ QSlot *s=qs(layer,eid);+ pthread_mutex_lock(&G.mx);+ if(!s->g4){ s->g4=g4; s->u4=u4; s->d4=d4; s->gs=gs; s->us=us; s->ds=ds; }+ while(G.qn>=QT_QCAP && !G.th_stop) pthread_cond_wait(&G.cv_take,&G.mx);+ enqueue_locked(layer,eid,-1,-1,0);+ pthread_mutex_unlock(&G.mx);+}++/* warmstart order -- heat descending (HEAT_FILE) or natural order.+ * Returns 0 once all budgets are full or the list is exhausted. */+static const uint32_t *g_sort_heat;+static int cmp_heat_desc(const void *a,const void *b){+ uint32_t ha=g_sort_heat[*(const int*)a], hb=g_sort_heat[*(const int*)b];+ return ha<hb ? 1 : ha>hb ? -1 : 0;+}+int qt_fill_next(int *layer,int *eid){+ if(!G.on) return 0;+ size_t n=(size_t)G.nl*G.ne;+ pthread_mutex_lock(&G.mx);+ if(!G.fill_order){+ G.fill_order=malloc(n*sizeof(int));+ for(size_t i=0;i<n;i++) G.fill_order[i]=(int)i;+ if(G.heat0){ g_sort_heat=G.heat0; qsort(G.fill_order,n,sizeof(int),cmp_heat_desc); }+ G.fill_cur=0;+ }+ while((size_t)G.fill_cur<n){+ int gi=G.fill_order[G.fill_cur];+ int l=gi/G.ne, e=gi%G.ne, hd=home(e);+ QSlot *s=qs(l,e);+ int full=1; for(int i=0;i<G.ndev;i++) if(G.used[i]+G.exp_bytes<=G.budget[i]) full=0;+ if(full){ pthread_mutex_unlock(&G.mx); return 0; }+ G.fill_cur++;+ if(s->resident||s->queued) continue;+ if(G.used[hd]+G.exp_bytes>G.budget[hd]) continue; /* dieses Device voll */+ *layer=l; *eid=e;+ pthread_mutex_unlock(&G.mx);+ return 1;+ }+ pthread_mutex_unlock(&G.mx);+ return 0;+}++/* Plan the whole warmstart set in one pass -- same heat order and budget+ * reservation as qt_fill_next, but without loading. The experts are then+ * loaded by any number of threads and handed over via qt_note_planned. */+int qt_plan_fill(int *layers,int *eids,int max){+ if(!G.on) return 0;+ size_t n=(size_t)G.nl*G.ne;+ int cnt=0;+ pthread_mutex_lock(&G.mx);+ if(!G.fill_order){+ G.fill_order=malloc(n*sizeof(int));+ for(size_t i=0;i<n;i++) G.fill_order[i]=(int)i;+ if(G.heat0){ g_sort_heat=G.heat0; qsort(G.fill_order,n,sizeof(int),cmp_heat_desc); }+ G.fill_cur=0;+ }+ while((size_t)G.fill_cur<n && cnt<max){+ int full=1; for(int i=0;i<G.ndev;i++) if(G.used[i]+G.exp_bytes<=G.budget[i]) full=0;+ if(full) break;+ int gi=G.fill_order[G.fill_cur++];+ int l=gi/G.ne, e=gi%G.ne, hd=home(e);+ QSlot *s=qs(l,e);+ if(s->resident||s->queued||s->planned) continue;+ if(G.used[hd]+G.exp_bytes>G.budget[hd]) continue;+ G.used[hd]+=G.exp_bytes; /* reserve */+ s->planned=1;+ layers[cnt]=l; eids[cnt]=e; cnt++;+ }+ pthread_mutex_unlock(&G.mx);+ return cnt;+}++/* Thread-safe (callable from multiple loader threads): stage + enqueue one+ * expert reserved by qt_plan_fill; blocks only while the queue is full. */+void qt_note_planned(int layer,int eid,+ const uint8_t *g4,const uint8_t *u4,const uint8_t *d4,+ const float *gs,const float *us,const float *ds){+ if(!G.on || !g4) return;+ QSlot *s=qs(layer,eid);+ pthread_mutex_lock(&G.mx);+ if(!s->g4){ s->g4=g4; s->u4=u4; s->d4=d4; s->gs=gs; s->us=us; s->ds=ds; }+ while(G.qn>=QT_QCAP && !G.th_stop) pthread_cond_wait(&G.cv_take,&G.mx);+ if(!enqueue_locked(layer,eid,-1,-1,1)){+ /* not enqueueable (e.g. already resident): return the reservation */+ if(s->planned) G.used[home(eid)]-=G.exp_bytes;+ }+ s->planned=0;+ pthread_mutex_unlock(&G.mx);+}++/* waits until the upload queue is drained (end of warmstart). */+void qt_fill_wait(void){+ if(!G.on) return;+ pthread_mutex_lock(&G.mx);+ while(G.qn>0 && !G.th_stop) pthread_cond_wait(&G.cv_take,&G.mx);+ pthread_mutex_unlock(&G.mx);+}++/* LFRU swap check (every 16 ticks = tokens): per device, coldest resident vs+ * hottest non-resident, with the tier.h hysteresis. */+static void qt_lfru_tick_locked(void){+ if(++G.tick % 16) return;+ size_t n=(size_t)G.nl*G.ne;+ for(int di=0;di<G.ndev;di++){+ int cold=-1, hot=-1; uint32_t ch=0, hh=0;+ for(size_t i=0;i<n;i++){+ QSlot *s=&G.slot[i];+ int e=(int)(i%G.ne);+ if(home(e)!=di) continue;+ if(s->resident && !s->queued){ if(cold<0||s->heat<ch){ cold=(int)i; ch=s->heat; } }+ else if(!s->resident && !s->queued && s->g4){ if(hot<0||s->heat>hh){ hot=(int)i; hh=s->heat; } }+ }+ if(cold<0||hot<0) continue;+ if(hh<=ch+(ch>>2)+4) continue; /* hysteresis as in tier.h */+ QSlot *v=&G.slot[cold];+ v->resident=0; /* CPU fallback from now on */+ if(enqueue_locked(hot/G.ne,hot%G.ne,cold/G.ne,cold%G.ne,0)) G.swaps++;+ else v->resident=1; /* queue full: revert */+ }+}++uint32_t qt_issue(int layer,const int *eids,int K,const float *x){+ if(!G.on||K>32) return 0;+ uint32_t mask=0;+ ColiCudaTensor *tg[QT_MAX_DEV][32],*tu[QT_MAX_DEV][32],*td[QT_MAX_DEV][32];+ static int rows[32]={0};+ if(!rows[0]) for(int i=0;i<32;i++) rows[i]=1;+ for(int i=0;i<G.ndev;i++) G.is_cnt[i]=0;++ pthread_mutex_lock(&G.mx);+ if(layer==0) qt_lfru_tick_locked();+ G.issue_open=1;+ for(int k=0;k<K;k++){+ QSlot *s=qs(layer,eids[k]);+ if(s->resident){+ int di=home(eids[k]); int c=G.is_cnt[di];+ tg[di][c]=s->tg; tu[di][c]=s->tu; td[di][c]=s->td;+ G.is_k[di][c]=k; G.is_cnt[di]=c+1;+ mask|=1u<<k; G.hits[di]++;+ } else G.miss++;+ }+ pthread_mutex_unlock(&G.mx);++ for(int di=0;di<G.ndev;di++){+ int c=G.is_cnt[di];+ if(!c) continue;+ float *xr=G.is_x + (size_t)di*8*G.D; /* per-device input block */+ for(int j=0;j<c;j++) memcpy(xr+(size_t)j*G.D, x, (size_t)G.D*sizeof(float));+ if(!coli_cuda_expert_group_issue(tg[di],tu[di],td[di],rows,c,xr)){+ /* issue failed -> hand these k back to the CPU */+ for(int j=0;j<c;j++) mask &= ~(1u<<G.is_k[di][j]);+ G.is_cnt[di]=0;+ }+ }+ return mask;+}++void qt_take(uint32_t mask,const float *val,int K,float *out){+ (void)K;+ if(!G.on) return;+ if(mask) for(int di=0;di<G.ndev;di++){+ int c=G.is_cnt[di];+ if(!c) continue;+ const float *y=coli_cuda_expert_group_take(G.dev[di]);+ if(!y) continue;+ for(int j=0;j<c;j++){+ float w=val[G.is_k[di][j]];+ const float *row=y+(size_t)j*G.D;+ for(int d=0;d<G.D;d++) out[d]+=w*row[d];+ }+ G.is_cnt[di]=0;+ }+ pthread_mutex_lock(&G.mx);+ G.issue_open=0;+ pthread_cond_broadcast(&G.cv_take);+ pthread_mutex_unlock(&G.mx);+}++void qt_stats(void){+ if(!G.on) return;+ uint64_t hits=0; size_t res=0;+ for(size_t i=0;i<(size_t)G.nl*G.ne;i++) res += G.slot[i].resident;+ fprintf(stderr,"[qtier] resident %zu/%d experts | uploads %llu | miss(CPU) %llu | q_skips %llu\n",+ res, G.nl*G.ne, (unsigned long long)G.uploads,+ (unsigned long long)G.miss, (unsigned long long)G.q_full_skips);+ for(int i=0;i<G.ndev;i++){+ size_t tc=0,tb=0; coli_cuda_stats(G.dev[i],&tc,&tb);+ hits+=G.hits[i];+ fprintf(stderr,"[qtier] dev %d: hits %llu | %zu tensors, %.2f GB VRAM used (budget %.2f GB)\n",+ G.dev[i], (unsigned long long)G.hits[i], tc, tb/1073741824.0, G.budget[i]/1073741824.0);+ }+ double tot=(double)(hits+G.miss);+ fprintf(stderr,"[qtier] VRAM hit rate: %.1f %% | LFRU swaps %llu\n",+ tot>0? 100.0*hits/tot : 0.0, (unsigned long long)G.swaps);+ { uint64_t calls=0,ex=0,rows=0; double h2d=0,kms=0,d2h=0;+ coli_cuda_group_stats(&calls,&ex,&rows,&h2d,&kms,&d2h);+ if(calls) fprintf(stderr,"[qtier] group_stats: %llu calls, %llu experts | h2d %.0f ms, kernel %.0f ms, d2h %.0f ms\n",+ (unsigned long long)calls,(unsigned long long)ex,h2d,kms,d2h); }+}++void qt_shutdown(void){+ if(!G.on) return;+ const char *hf=getenv("HEAT_FILE");+ if(hf){+ FILE *f=fopen(hf,"wb");+ if(f){+ uint32_t hdr[3]={0x51544831u,(uint32_t)G.nl,(uint32_t)G.ne};+ fwrite(hdr,4,3,f);+ for(size_t i=0;i<(size_t)G.nl*G.ne;i++) fwrite(&G.slot[i].heat,4,1,f);+ fclose(f);+ fprintf(stderr,"[qtier] HEAT_FILE saved: %s\n",hf);+ }+ }+ pthread_mutex_lock(&G.mx); G.th_stop=1; pthread_cond_signal(&G.cv); pthread_mutex_unlock(&G.mx);+ pthread_join(G.th,NULL);+ G.on=0;+ coli_cuda_shutdown();+}++#endif /* COLI_CUDA */diff --git a/c/qwen36_tier.h b/c/qwen36_tier.hnew file mode 100644index 00000000..e424a168--- /dev/null+++ b/c/qwen36_tier.h@@ -0,0 +1,81 @@+/* qwen36_tier.h -- optional CUDA VRAM expert tier for the qwen36 engine.+ *+ * Applies colibri's placement concept ("route -> place -> overlap -> learn")+ * one level up from the GLM disk tier: experts live in RAM, the *hot* ones+ * are promoted into DEVICE_LOCAL VRAM across one or more GPUs and computed+ * there via the existing CUDA backend (backend_cuda.cu, expert-group API).+ *+ * - Every expert has one home device (eid % n_gpus), no duplicates.+ * - Routing heat decides who earns VRAM (LFRU semantics from tier.h, with+ * hysteresis); a warmstart pre-fills the budget before the first token,+ * ordered by a persisted heat table (HEAT_FILE) when available.+ * - Uploads run on a background thread through staging copies; decode never+ * blocks on placement. A VRAM miss falls back to the CPU int8 path and+ * overlaps with the in-flight GPU groups.+ *+ * Enable with COLI_CUDA=1 [COLI_GPUS=0,1] [CUDA_EXPERT_GB=<G>|auto]+ * [HEAT_FILE=<path>] [QT_NO_WARMSTART=1]. Compiled only when the build sets+ * -DCOLI_CUDA (CUDA=1); otherwise the inline stubs below keep the engine+ * CPU-only with zero overhead. */+#ifndef QWEN36_TIER_H+#define QWEN36_TIER_H+#include <stdint.h>++#ifdef COLI_CUDA++/* Init after model load. Returns 1 when the tier is active.+ * cap_experts_per_layer must equal n_experts (full RAM residency): the tier+ * stores raw pointers into the expert slots, which must never be evicted. */+int qt_init(int n_layers, int n_experts, int hidden, int inter,+ int cap_experts_per_layer, int topk, int expert_gs);+int qt_ready(void);+int qt_is_resident(int layer, int eid);+void qt_shutdown(void);++/* Call once per routed expert per token (pointers to the RAM slot: packed+ * int4 + per-row scales). Updates heat and may enqueue a background upload. */+void qt_note(int layer, int eid,+ const uint8_t *g4, const uint8_t *u4, const uint8_t *d4,+ const float *gs, const float *us, const float *ds);++/* Launch the GPU groups for the resident subset of the K selected experts+ * (async, all devices in parallel). Returns a bitmask of the k handled by+ * the GPU. Compute the misses on the CPU, then call qt_take(). */+uint32_t qt_issue(int layer, const int *eids, int K, const float *x);++/* Collect the GPU results and accumulate val[k]*y_k into out[hidden]. */+void qt_take(uint32_t mask, const float *val, int K, float *out);++/* Warmstart: plan the full fill set (heat order, budget reserved), then any+ * number of loader threads may call qt_note_planned per planned expert. */+int qt_plan_fill(int *layers, int *eids, int max);+void qt_note_planned(int layer, int eid,+ const uint8_t *g4, const uint8_t *u4, const uint8_t *d4,+ const float *gs, const float *us, const float *ds);+int qt_fill_next(int *layer, int *eid);+void qt_note_block(int layer, int eid,+ const uint8_t *g4, const uint8_t *u4, const uint8_t *d4,+ const float *gs, const float *us, const float *ds);+void qt_fill_wait(void); /* blocks until the upload queue is drained */++/* One telemetry block on stderr: residency, hits/misses, uploads per device. */+void qt_stats(void);++#else /* !COLI_CUDA: inline stubs, engine stays CPU-only */++static inline int qt_init(int a,int b,int c,int d,int e,int f,int g){(void)a;(void)b;(void)c;(void)d;(void)e;(void)f;(void)g;return 0;}+static inline int qt_ready(void){return 0;}+static inline int qt_is_resident(int a,int b){(void)a;(void)b;return 0;}+static inline void qt_shutdown(void){}+static inline void qt_note(int a,int b,const uint8_t*c,const uint8_t*d,const uint8_t*e,const float*f,const float*g,const float*h){(void)a;(void)b;(void)c;(void)d;(void)e;(void)f;(void)g;(void)h;}+static inline uint32_t qt_issue(int a,const int*b,int c,const float*d){(void)a;(void)b;(void)c;(void)d;return 0;}+static inline void qt_take(uint32_t a,const float*b,int c,float*d){(void)a;(void)b;(void)c;(void)d;}+static inline int qt_plan_fill(int*a,int*b,int c){(void)a;(void)b;(void)c;return 0;}+static inline void qt_note_planned(int a,int b,const uint8_t*c,const uint8_t*d,const uint8_t*e,const float*f,const float*g,const float*h){(void)a;(void)b;(void)c;(void)d;(void)e;(void)f;(void)g;(void)h;}+static inline int qt_fill_next(int*a,int*b){(void)a;(void)b;return 0;}+static inline void qt_note_block(int a,int b,const uint8_t*c,const uint8_t*d,const uint8_t*e,const float*f,const float*g,const float*h){(void)a;(void)b;(void)c;(void)d;(void)e;(void)f;(void)g;(void)h;}+static inline void qt_fill_wait(void){}+static inline void qt_stats(void){}++#endif /* COLI_CUDA */+#endif /* QWEN36_TIER_H */diff --git a/docs/qwen36-cuda-tier.md b/docs/qwen36-cuda-tier.mdnew file mode 100644index 00000000..21684a6c--- /dev/null+++ b/docs/qwen36-cuda-tier.md@@ -0,0 +1,50 @@+# qwen36: CUDA VRAM expert tier++Applies colibri's placement concept ("route -> place -> overlap -> learn") to+Qwen3.6-35B-A3B one level up from the GLM disk tier: all 10,240 experts live+in RAM, the **hot** ones are promoted into DEVICE_LOCAL VRAM across one or+more GPUs and computed there through the existing shared CUDA backend+(`backend_cuda.cu` expert-group API — no new backend).++## How it works++- **Home device:** expert `eid` lives on GPU `eid % n_gpus`; no duplicates.+- **Placement:** routing heat decides who earns VRAM (LFRU semantics from+ `tier.h`, 25%+4 hysteresis). A parallel **warmstart** fills the per-device+ budget before the first token — ordered by a persisted heat table+ (`HEAT_FILE`) when present, so a second run starts fully placed.+- **Decode:** per (token, layer) the resident experts are issued as async+ groups on all devices (`coli_cuda_expert_group_issue/take`); VRAM misses+ fall back to the CPU int8 path and overlap with the in-flight groups, as+ does the shared expert. Placement never changes routing or precision.+- **Memory:** the warmstart frees the RAM int8 copies of VRAM-resident+ experts (rematerialized from the packed int4 copy on LFRU eviction; no+ container access). Peak RSS for the 35B int4 container: ~29 GB with two+ 8 GB GPUs.++## Usage++```bash+make -C c qwen36 CUDA=1 CUDA_ARCH=native # NVCC=/usr/bin/nvcc on distro CUDA+COLI_CUDA=1 COLI_GPUS=0,1 HEAT_FILE=heat.bin CUDA_EXPERT_GB=auto \+OMP_NUM_THREADS=<physical cores> OMP_WAIT_POLICY=ACTIVE OMP_PROC_BIND=close \+SNAP=<container> N_NEW=200 ./c/qwen36 256 4 prompt.txt+```++`cap` (argv[1]) must equal `n_experts` (full RAM residency). int4 containers+only (the int8 container keeps the CPU path). `COLI_TIMERS=1` prints+per-phase timings and tier telemetry.++## Measured (Threadripper 3945WX 12C, RTX 3070 8 GB + Quadro RTX 4000 8 GB, Qwen3.6-35B-A3B int4, 200-token decode)++| | 1 GPU (8 GB) | 2 GPUs (16 GB) |+|---|---|---|+| decode tok/s (cold / warm heat) | 9.2 / 9.9 | 10.6 / **11.3** |+| VRAM-resident experts | 4,391 (43 %) | 8,532 (83 %) |+| VRAM hit rate (cold / warm) | 44 % / 95 % | 85 % / 100 % |+| peak RSS | 40 GB | **29 GB** |+| reference: Ollama q4_K_M, same box | 7.5 | 10.5 |++CPU-only baseline of this engine before the tier: 0.35 tok/s.+Numerics: logits cosine vs the f32 CPU reference 0.9992 (dense int8 on),+bit-identical GPU-vs-CPU on the same container (cosine 1.0000001).