Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

qwen36 tier: fp8 streaming mode for experts that do not fit in RAM

kreuzzelg979025c26472
diff --git a/c/Makefile b/c/Makefile
index 86ddde1d..c3a22582 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1691,6 +1691,13 @@ tests/test_qwen36_tier_invariants$(EXE): tests/test_qwen36_tier_invariants.c tes
tests/test_qwen36_tier_autoplace$(EXE): tests/test_qwen36_tier_autoplace.c tests/qwen36_fake_cuda.h qwen36_tier.c qwen36_tier.h backend_cuda.h
$(CC) $(CFLAGS) -DCOLI_CUDA $< -o $@ $(LDFLAGS)
+# Same fake backend: the fp8 streaming mode a model whose experts do not fit
+# in RAM needs (Qwen3.8) -- cap < n_experts accepted, fmt=8 uploads with
+# 128x128 block scales, bytes staged unchanged, no pointer kept into the
+# engine's recycled slot, promotion decided when the bytes pass by.
+tests/test_qwen36_tier_fp8$(EXE): tests/test_qwen36_tier_fp8.c tests/qwen36_fake_cuda.h qwen36_tier.c qwen36_tier.h backend_cuda.h
+ $(CC) $(CFLAGS) -DCOLI_CUDA $< -o $@ $(LDFLAGS)
+
tests/test_qwen36_tier_int8_engine$(EXE): tests/test_qwen36_tier_int8_engine.c tests/qwen36_fake_cuda.h qwen36.c qwen36_tier.c qwen36_tier.h backend_cuda.h cli_args.h st.h json.h compat.h
$(CC) $(CFLAGS) -DCOLI_CUDA $< -o $@ $(LDFLAGS)
diff --git a/c/qwen36_tier.c b/c/qwen36_tier.c
index 6ca20420..2f4fde7d 100644
--- a/c/qwen36_tier.c
+++ b/c/qwen36_tier.c
@@ -63,8 +63,8 @@ static int home(int eid){ return eid % G.ndev; }
static void stage(uint8_t *dw, float *dsc,
const uint8_t *g4,const uint8_t *u4,const uint8_t *d4,
const float *gs,const float *us,const float *ds){
- size_t mb = (size_t)G.D*G.Ih/(G.wfmt==1?1:2);
- if(G.wfmt==1){
+ size_t mb = (size_t)G.D*G.Ih/(G.wfmt==4?2:1);
+ if(G.wfmt==1 || G.wfmt==8){
/* int8: il formato del backend e' gia' quello in RAM, si copia e basta.
* Niente XOR: quello serve a portare i nibble int4 da complemento a due
* a binario sfalsato, e su byte interi sarebbe corruzione. */
@@ -117,10 +117,17 @@ static void *uploader(void *arg){
int dv = G.dev[home(eid)];
/* passo fra le tre matrici nello staging: int4 impacchettato = mezzo
* byte per elemento, int8 = uno. */
- size_t mb=(size_t)G.D*G.Ih/(G.wfmt==1?1:2);
+ size_t mb=(size_t)G.D*G.Ih/(G.wfmt==4?2:1);
ColiCudaTensor *tg=NULL,*tu=NULL,*td=NULL;
int ok;
- if(G.wfmt==1){
+ if(G.wfmt==8){
+ /* e4m3 bytes as they came from the checkpoint, block scales
+ * [ceil(O/128), ceil(I/128)] per matrix -- the layout #817's
+ * kernels and tensor_upload(fmt=8) already agree on */
+ ok = coli_cuda_tensor_upload(&tg, w, sc, 8, G.D, G.Ih, dv)
+ && coli_cuda_tensor_upload(&tu, w+mb, sc+G.sc_gu, 8, G.D, G.Ih, dv)
+ && coli_cuda_tensor_upload(&td, w+2*mb, sc+2*G.sc_gu, 8, G.Ih, G.D, dv);
+ } else if(G.wfmt==1){
/* int8, scale per riga: qt_init ha gia' rifiutato il caso raggruppato,
* che questo formato non sa esprimere. */
ok = coli_cuda_tensor_upload(&tg, w, sc, 1, G.D, G.Ih, dv)
@@ -373,11 +380,32 @@ static void auto_place(int nl, int ne, int topk, const size_t *capacity, const u
}
+/* ---- fp8 streaming mode (Qwen3.8) -----------------------------------------
+ * qwen36 keeps every expert in RAM and lets the tier retain raw pointers into
+ * slots that are never recycled; that is what `cap == n_experts` guards. A
+ * model whose experts do not fit in RAM (Qwen3.8: 24 576 x 4.7 MiB) streams
+ * them through an LRU whose slots ARE recycled, so a retained pointer would
+ * dangle by the next token. In this mode the tier owns what it uploads: the
+ * bytes are copied into the staging buffer inside the qt_note call, while the
+ * engine's slot is still live, and the pointers are dropped right after. A
+ * promotion can therefore only happen when the bytes pass by -- the LFRU
+ * decision moves from the periodic tick into qt_note, which asks: is this
+ * expert, now in hand, hotter than the coldest resident on its device? */
+static int G_fp8_stream;
+static const float *G_fp8_lut;
+
+int qt_init_fp8(int nl, int ne, int D, int Ih, int cap, int topk, const float *e4m3_lut){
+ G_fp8_stream = 1; G_fp8_lut = e4m3_lut;
+ int ok = qt_init(nl, ne, D, Ih, cap, topk, 0, 0);
+ if(!ok) G_fp8_stream = 0;
+ return ok;
+}
+
int qt_init(int nl, int ne, int D, int Ih, int cap, int topk, int expert_gs,
int expert_is_int4){
const char *e=getenv("COLI_CUDA");
if(!(e && *e=='1')) return 0;
- if(cap != ne){
+ if(cap != ne && !G_fp8_stream){
fprintf(stderr,"[qtier] cap=%d != n_experts=%d -> tier disabled (needs full RAM residency)\n",cap,ne);
return 0;
}
@@ -422,19 +450,32 @@ int qt_init(int nl, int ne, int D, int Ih, int cap, int topk, int expert_gs,
int have=coli_cuda_device_count();
if(have<G.ndev){ G.ndev=have; }
if(G.ndev<1){ fprintf(stderr,"[qtier] no CUDA devices -> CPU path\n"); return 0; }
+ if(G_fp8_stream){
+ if(!G_fp8_lut || !coli_cuda_fp8_set_lut(G_fp8_lut)){
+ fprintf(stderr,"[qtier] fmt=8 decode table not published -> CPU path\n");
+ return 0;
+ }
+ }
/* Weight format and bytes per expert come first now: the automatic
* placement below needs them to price the experts a trunk item displaces. */
- G.wfmt = expert_is_int4 ? 4 : 1;
+ G.wfmt = G_fp8_stream ? 8 : (expert_is_int4 ? 4 : 1);
if(G.wfmt==1 && expert_gs>0){
fprintf(stderr,"[qtier] int8 experts with grouped scales (gs=%d) cannot be "
"expressed on the GPU (fmt=1 is per-row only) -> CPU path\n", expert_gs);
return 0;
}
G.egs = expert_gs;
- G.sc_gu = expert_gs ? (size_t)Ih * ((D + expert_gs - 1)/expert_gs) : (size_t)Ih;
- G.sc_d = expert_gs ? (size_t)D * ((Ih + expert_gs - 1)/expert_gs) : (size_t)D;
- G.exp_bytes = (G.wfmt==1 ? 3ull*D*Ih : 3ull*D*Ih/2)
+ if(G.wfmt==8){
+ /* one f32 scale per 128x128 block of [O,I]: gate/up are [Ih,D], down is
+ * [D,Ih] -- the same count either way, kept as two fields for symmetry */
+ size_t nbD=(size_t)(D+127)/128, nbI=(size_t)(Ih+127)/128;
+ G.sc_gu = nbI*nbD; G.sc_d = nbD*nbI;
+ } else {
+ G.sc_gu = expert_gs ? (size_t)Ih * ((D + expert_gs - 1)/expert_gs) : (size_t)Ih;
+ G.sc_d = expert_gs ? (size_t)D * ((Ih + expert_gs - 1)/expert_gs) : (size_t)D;
+ }
+ G.exp_bytes = (G.wfmt==4 ? 3ull*D*Ih/2 : 3ull*D*Ih)
+ (2*G.sc_gu+G.sc_d)*sizeof(float) + 4096; /* + allocation slack */
/* Per-device allowance for tier + trunk: CUDA_EXPERT_GB when numeric,
@@ -633,7 +674,7 @@ static int enqueue_locked(int layer,int eid,int v_layer,int v_eid,int reserved){
if(G.qn>=QT_QCAP){ G.q_full_skips++; return 0; }
int hd=home(eid);
if(!reserved && v_eid<0 && G.used[hd]+G.exp_bytes>G.budget[hd]) return 0;
- size_t mb=(size_t)G.D*G.Ih/(G.wfmt==1?1:2); /* buffer di staging: int8 = 1 byte/elemento */
+ size_t mb=(size_t)G.D*G.Ih/(G.wfmt==4?2:1); /* buffer di staging: int8/fp8 = 1 byte/elemento */
uint8_t *w=malloc(3*mb); float *sc=malloc((2*G.sc_gu+G.sc_d)*sizeof(float));
if(!w||!sc){ free(w); free(sc); return 0; }
if(!reserved && v_eid<0) G.used[hd]+=G.exp_bytes;
@@ -646,12 +687,50 @@ static int enqueue_locked(int layer,int eid,int v_layer,int v_eid,int reserved){
return 1;
}
+/* streaming mode: the bytes in hand are valid only during this call, so set
+ * the pointers for the enqueue (which stages a copy under the lock) and drop
+ * them again before returning. Nothing downstream may read them later. */
+static void stream_point(QSlot *s,const uint8_t *g4,const uint8_t *u4,const uint8_t *d4,
+ const float *gs,const float *us,const float *ds){
+ s->g4=g4; s->u4=u4; s->d4=d4; s->gs=gs; s->us=us; s->ds=ds;
+}
+static void stream_forget(QSlot *s){ s->g4=s->u4=s->d4=NULL; s->gs=s->us=s->ds=NULL; }
+
+/* The LFRU decision at the moment the bytes pass by: if this expert is not
+ * resident and its device has no room, evict the coldest resident there when
+ * the admission rule says the newcomer is worth it. Budget-neutral swap. */
+static void stream_promote_locked(int layer,int eid){
+ QSlot *s=qs(layer,eid);
+ if(s->resident||s->queued) return;
+ int hd=home(eid);
+ if(G.used[hd]+G.exp_bytes<=G.budget[hd]){ enqueue_locked(layer,eid,-1,-1,0); return; }
+ size_t n=(size_t)G.nl*G.ne; int cold=-1; uint32_t ch=0;
+ for(size_t i=0;i<n;i++){
+ QSlot *c=&G.slot[i];
+ if(home((int)(i%G.ne))!=hd || !c->resident || c->queued) continue;
+ if(cold<0||c->heat<ch){ cold=(int)i; ch=c->heat; }
+ }
+ if(cold<0 || !tier_should_promote(s->heat,ch)) return;
+ QSlot *v=&G.slot[cold];
+ v->resident=0;
+ if(enqueue_locked(layer,eid,cold/G.ne,cold%G.ne,0)) G.swaps++;
+ else v->resident=1;
+}
+
void qt_note(int layer,int eid,
const uint8_t *g4,const uint8_t *u4,const uint8_t *d4,
const float *gs,const float *us,const float *ds){
if(!G.on || !g4) return;
QSlot *s=qs(layer,eid);
pthread_mutex_lock(&G.mx);
+ if(G_fp8_stream){
+ if(s->heat<0xFFFFFFFFu) s->heat++;
+ stream_point(s,g4,u4,d4,gs,us,ds);
+ stream_promote_locked(layer,eid);
+ stream_forget(s);
+ pthread_mutex_unlock(&G.mx);
+ return;
+ }
if(!s->g4){ s->g4=g4; s->u4=u4; s->d4=d4; s->gs=gs; s->us=us; s->ds=ds; }
if(s->heat<0xFFFFFFFFu) s->heat++;
enqueue_locked(layer,eid,-1,-1,0);
@@ -665,9 +744,11 @@ void qt_note_block(int layer,int eid,
if(!G.on || !g4) return;
QSlot *s=qs(layer,eid);
pthread_mutex_lock(&G.mx);
- if(!s->g4){ s->g4=g4; s->u4=u4; s->d4=d4; s->gs=gs; s->us=us; s->ds=ds; }
+ if(G_fp8_stream) stream_point(s,g4,u4,d4,gs,us,ds);
+ else if(!s->g4){ s->g4=g4; s->u4=u4; s->d4=d4; s->gs=gs; s->us=us; s->ds=ds; }
while(G.qn>=QT_QCAP && !G.th_stop) pthread_cond_wait(&G.cv_take,&G.mx);
enqueue_locked(layer,eid,-1,-1,0);
+ if(G_fp8_stream) stream_forget(s);
pthread_mutex_unlock(&G.mx);
}
@@ -756,13 +837,15 @@ void qt_note_planned(int layer,int eid,
pthread_mutex_unlock(&G.mx);
return;
}
- if(!s->g4){ s->g4=g4; s->u4=u4; s->d4=d4; s->gs=gs; s->us=us; s->ds=ds; }
+ if(G_fp8_stream) stream_point(s,g4,u4,d4,gs,us,ds);
+ else if(!s->g4){ s->g4=g4; s->u4=u4; s->d4=d4; s->gs=gs; s->us=us; s->ds=ds; }
while(G.qn>=QT_QCAP && !G.th_stop) pthread_cond_wait(&G.cv_take,&G.mx);
if(!enqueue_locked(layer,eid,-1,-1,1)){
/* not enqueueable (e.g. already resident): return the reservation */
if(s->planned) G.used[home(eid)]-=G.exp_bytes;
}
s->planned=0;
+ if(G_fp8_stream) stream_forget(s);
pthread_mutex_unlock(&G.mx);
}
@@ -902,6 +985,7 @@ void qt_shutdown(void){
pthread_mutex_lock(&G.mx); G.th_stop=1; pthread_cond_signal(&G.cv); pthread_cond_broadcast(&G.cv_take); pthread_mutex_unlock(&G.mx);
pthread_join(G.th,NULL);
G.on=0;
+ G_fp8_stream=0;
coli_cuda_shutdown();
}
diff --git a/c/qwen36_tier.h b/c/qwen36_tier.h
index b71bb730..f99311be 100644
--- a/c/qwen36_tier.h
+++ b/c/qwen36_tier.h
@@ -73,6 +73,13 @@ int qt_dnproj_init(int layer, const int8_t *q, const float *sc,
int I, int O, int device);
int qt_dnproj_matmul(int layer, float *y, const float *x, int I, int O);
+/* fp8 streaming mode (Qwen3.8): experts arrive as e4m3 bytes with 128x128
+ * block scales and do NOT all fit in RAM. cap may be smaller than n_experts;
+ * the tier copies what it uploads inside the qt_note call and keeps no
+ * pointer into the engine's slot. e4m3_lut is quant.h's E4M3_LUT, published
+ * to the backend so fmt=8 uploads are accepted. */
+int qt_init_fp8(int n_layers, int n_experts, int hidden, int inter,
+ int cap_experts_per_layer, int topk, const float *e4m3_lut);
int qt_init(int n_layers, int n_experts, int hidden, int inter,
int cap_experts_per_layer, int topk, int expert_gs,
int expert_is_int4);
@@ -112,6 +119,7 @@ void qt_stats(void);
#else /* !COLI_CUDA: inline stubs, engine stays CPU-only */
static inline int qt_init(int a,int b,int c,int d,int e,int f,int g,int h){(void)h;(void)a;(void)b;(void)c;(void)d;(void)e;(void)f;(void)g;return 0;}
+static inline int qt_init_fp8(int a,int b,int c,int d,int e,int f,const float*g){(void)a;(void)b;(void)c;(void)d;(void)e;(void)f;(void)g;return 0;}
static inline int qt_lmhead_init(const int8_t*a,const float*b,int c,int d){(void)a;(void)b;(void)c;(void)d;return 0;}
static inline int qt_lmhead_matmul(float*a,const float*b,int c,int d){(void)a;(void)b;(void)c;(void)d;return 0;}
#define QT_PLACE_CPU (-1)
diff --git a/c/tests/qwen36_fake_cuda.h b/c/tests/qwen36_fake_cuda.h
index 196ef53b..62197c4f 100644
--- a/c/tests/qwen36_fake_cuda.h
+++ b/c/tests/qwen36_fake_cuda.h
@@ -52,7 +52,7 @@ static int upload_common(ColiCudaTensor **t, const void *w, int fmt,
*t = n;
fake_uploads++;
last_fmt = fmt;
- last_bytes = (size_t)I * O / (fmt == 1 ? 1 : 2);
+ last_bytes = (size_t)I * O / ((fmt == 1 || fmt == 8) ? 1 : 2);
if (fake_uploads == 1) {
captured_len = last_bytes < sizeof captured ? last_bytes : sizeof captured;
memcpy(captured, w, captured_len);
@@ -71,6 +71,8 @@ void coli_cuda_tensor_free(ColiCudaTensor *t) { free(t); }
int coli_cuda_available_device_count(void) { return fake_ndev; }
int coli_cuda_device_count(void) { return fake_ndev; }
int coli_cuda_init(const int *d, int n) { (void)d; (void)n; return 1; }
+static int fake_lut_published;
+int coli_cuda_fp8_set_lut(const float *lut) { fake_lut_published = lut != NULL; return lut != NULL; }
void coli_cuda_shutdown(void) {}
int coli_cuda_mem_info(int device, size_t *freeb, size_t *total) {
(void)device;
diff --git a/c/tests/test_qwen36_tier_fp8.c b/c/tests/test_qwen36_tier_fp8.c
new file mode 100644
index 00000000..b17aa7fe
--- /dev/null
+++ b/c/tests/test_qwen36_tier_fp8.c
@@ -0,0 +1,145 @@
+/* The tier's fp8 streaming mode (Qwen3.8), on the fake backend.
+ *
+ * What changes against the qwen36 modes and is pinned here:
+ * - cap < n_experts is accepted: the experts do not all live in RAM;
+ * - bytes per expert count one byte per element plus three 128x128 block
+ * scale tables, and the upload is fmt=8 with exactly those scales;
+ * - staging copies the e4m3 bytes unchanged (no 0x88 XOR: on whole bytes it
+ * would be corruption) and the block scales in order gate|up|down;
+ * - the tier keeps NO pointer into the engine's slot after qt_note returns,
+ * because that slot is recycled by the next token;
+ * - promotion happens when the bytes pass by: a non-resident expert noted
+ * on a full device evicts the coldest resident there if it is hotter, a
+ * colder one is refused, and the swap is budget-neutral;
+ * - the e4m3 decode table is published to the backend before any upload.
+ *
+ * No GPU, no toolkit; the fake backend records what the tier sends. */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "qwen36_fake_cuda.h"
+
+#include "../qwen36_tier.c"
+
+static int fails;
+static void check(int ok, const char *what) {
+ if (!ok) { printf(" FAIL: %s\n", what); fails++; }
+}
+
+/* toy Qwen3.8 geometry: hidden 256, inter 128 -> gate/up [128,256] = 2x2 blocks,
+ * down [256,128] = 2x1 -- more than one block per matrix on purpose */
+enum { NL = 2, NE = 16, D = 256, IH = 128, CAP = 4, TOPK = 2 };
+#define NBD ((D + 127) / 128)
+#define NBI ((IH + 127) / 128)
+#define NSC (NBI * NBD) /* scales per matrix */
+
+static unsigned char slab[NE][3 * D * IH]; /* gate|up|down e4m3 bytes, one per expert */
+static float scales[NE][3 * NSC];
+static float lut[256];
+
+static void wait_idle(void) {
+ for (int i = 0; i < 1000; i++) {
+ pthread_mutex_lock(&G.mx);
+ int pending = G.qn;
+ for (int l = 0; l < G.nl; l++) for (int e = 0; e < G.ne; e++) pending += qs(l, e)->queued;
+ pthread_mutex_unlock(&G.mx);
+ if (!pending) break;
+ struct timespec ts = {0, 2000000}; nanosleep(&ts, NULL);
+ }
+}
+#define NOTE(fn, l, e) fn((l), (e), slab[e], slab[e] + D * IH, slab[e] + 2 * D * IH, \
+ scales[e], scales[e] + NSC, scales[e] + 2 * NSC)
+
+int main(void) {
+ for (int e = 0; e < NE; e++) {
+ for (size_t i = 0; i < sizeof slab[e]; i++) slab[e][i] = (unsigned char)((i * 7 + e * 13) & 0xFF);
+ for (int i = 0; i < 3 * NSC; i++) scales[e][i] = 0.001f * (e + 1) + 0.0001f * i;
+ }
+ for (int i = 0; i < 256; i++) lut[i] = (float)i;
+ setenv("COLI_CUDA", "1", 1);
+ setenv("COLI_GPUS", "0", 1);
+ setenv("QT_NO_WARMSTART", "1", 1);
+ setenv("COLI_PLACE", "off", 1);
+ setenv("HEAT_FILE", "", 1);
+
+ /* ---- 1. the mode starts with cap < n_experts, publishes the LUT, sizes fmt 8 ---- */
+ printf(" 1. init\n");
+ /* allowance for exactly 3 experts on the one device */
+ size_t exp_bytes = 3ull * D * IH + 3ull * NSC * sizeof(float) + 4096;
+ char gb[64]; snprintf(gb, sizeof gb, "%.15f", (double)(3 * exp_bytes + exp_bytes / 2) / 1073741824.0);
+ setenv("CUDA_EXPERT_GB", gb, 1);
+ fake_ndev = 1; fake_uploads = 0; fake_lut_published = 0;
+ check(qt_init_fp8(NL, NE, D, IH, CAP, TOPK, lut), "fp8 streaming mode must start with cap 4 < 16 experts");
+ check(G.wfmt == 8, "weight format is 8");
+ check(fake_lut_published, "e4m3 LUT published to the backend before any upload");
+ check(G.exp_bytes == exp_bytes, "exp_bytes = 3*D*IH bytes + 3 block-scale tables + slack");
+ check(G.sc_gu == NSC && G.sc_d == NSC, "one scale per 128x128 block per matrix");
+
+ /* ---- 2. a note uploads fmt 8, bytes intact, scales in order, pointers dropped ---- */
+ printf(" 2. note -> upload\n");
+ NOTE(qt_note, 0, 5);
+ wait_idle();
+ check(qt_is_resident(0, 5), "noted expert becomes resident");
+ check(fake_uploads == 3, "three tensors uploaded for one expert");
+ check(last_fmt == 8, "upload format is 8");
+ check(last_bytes == (size_t)D * IH, "fmt 8 carries one byte per element");
+ int intact = 1;
+ for (size_t i = 0; i < captured_len; i++) if (captured[i] != slab[5][i]) { intact = 0; break; }
+ check(intact, "e4m3 bytes staged unchanged (no XOR)");
+ pthread_mutex_lock(&G.mx);
+ check(qs(0, 5)->g4 == NULL && qs(0, 5)->gs == NULL, "no pointer into the engine slot survives qt_note");
+ pthread_mutex_unlock(&G.mx);
+
+ /* ---- 3. fill the budget, then promotion-at-note ---- */
+ printf(" 3. fill, then swap at note\n");
+ NOTE(qt_note, 0, 6); NOTE(qt_note, 0, 7);
+ wait_idle();
+ check(qt_is_resident(0, 6) && qt_is_resident(0, 7), "budget holds three experts");
+ pthread_mutex_lock(&G.mx);
+ check(G.used[0] == 3 * G.exp_bytes, "used == 3 x exp_bytes");
+ pthread_mutex_unlock(&G.mx);
+ /* a fourth, cold expert (heat 1) must not displace anyone (residents have heat 1 too) */
+ NOTE(qt_note, 0, 8);
+ wait_idle();
+ check(!qt_is_resident(0, 8), "an expert no hotter than the coldest resident is refused");
+ check(qt_is_resident(0, 5) && qt_is_resident(0, 6) && qt_is_resident(0, 7), "residents untouched by a refused newcomer");
+ /* make expert 9 hot: note it many times; each note is a chance to promote */
+ for (int i = 0; i < 64 && !qt_is_resident(0, 9); i++) { NOTE(qt_note, 0, 9); wait_idle(); }
+ check(qt_is_resident(0, 9), "a hot expert noted repeatedly gets promoted when the device is full");
+ int residents = 0; for (int e = 0; e < NE; e++) residents += qt_is_resident(0, e);
+ check(residents == 3, "the promotion was a swap: still three residents");
+ pthread_mutex_lock(&G.mx);
+ check(G.used[0] == 3 * G.exp_bytes, "swap is budget-neutral");
+ check(G.swaps >= 1, "swap counted");
+ int stale_ptr = 0; for (int e = 0; e < NE; e++) stale_ptr += (qs(0, e)->g4 != NULL);
+ pthread_mutex_unlock(&G.mx);
+ check(stale_ptr == 0, "no slot retains a pointer in streaming mode");
+
+ /* ---- 4. issue on residents uses the fake issue path ---- */
+ printf(" 4. issue mask\n");
+ fake_issue_hook = NULL; /* fake issue returns 0 -> mask cleared, but hits counted */
+ float x[D]; for (int i = 0; i < D; i++) x[i] = 1.0f;
+ int eids[TOPK] = { 9, 8 };
+ uint32_t mask = qt_issue(0, eids, TOPK, x);
+ check(mask == 0, "fake issue refuses -> everything handed back to the CPU");
+ float out[D]; memset(out, 0, sizeof out); float val[TOPK] = {1, 1};
+ qt_take(mask, val, TOPK, out);
+ pthread_mutex_lock(&G.mx);
+ check(G.hits[0] == 1 && G.miss == 1, "one resident routed (hit), one not (miss)");
+ pthread_mutex_unlock(&G.mx);
+ qt_shutdown();
+ check(!G_fp8_stream, "shutdown leaves streaming mode");
+
+ /* ---- 5. no LUT -> no fp8 tier ---- */
+ printf(" 5. refusals\n");
+ fake_lut_published = 0;
+ check(!qt_init_fp8(NL, NE, D, IH, CAP, TOPK, NULL), "without a decode table the fp8 tier stays off");
+ check(!G_fp8_stream, "a refused init does not leave the flag set");
+ /* and the classic modes still insist on cap == n_experts */
+ check(!qt_init(NL, NE, D, IH, CAP, TOPK, 0, 1), "int4 mode still refuses cap < n_experts");
+
+ if (fails) { printf("test_qwen36_tier_fp8: %d failure(s)\n", fails); return 1; }
+ printf("test_qwen36_tier_fp8: ok\n");
+ return 0;
+}