Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
Merge branch 'dev' into sse41-tier-upstream-next-v2 (+25 commits)
Jaapjan Tinbergendd1cf4e1678d
diff --cc c/Makefileindex b3bd50f6,55a8bfeb..dd78531c--- a/c/Makefile+++ b/c/Makefile@@@ -1197,11 -1200,14 +1200,19 @@@ tests/test_qwen36_tok_truncated$(EXE):tests/test_qwen36_encode_oom$(EXE): tests/test_qwen36_encode_oom.c qwen36.c expert_ffn.h qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)$(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)+# slot_ensure_int8's wiring onto #1271's unpack_int4_to_int8: regression check+# that its output matches the old separate scalar nibble-unpack loop it replaced.+tests/test_qwen36_slot_int8$(EXE): tests/test_qwen36_slot_int8.c qwen36.c qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)+ $(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)++ # the dense trunk's integer path: quantizer contract, int4 planar packing, dispatch.+ tests/test_qwen36_dense_idot$(EXE): tests/test_qwen36_dense_idot.c qwen36.c expert_ffn.h qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)+ $(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)++ # #1653: added tokens are split out before the regex pre-tokenizer, HF-style.+ tests/test_qwen36_tokenizer$(EXE): tests/test_qwen36_tokenizer.c qwen36.c expert_ffn.h qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)+ $(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)+# Reproducible local timing evidence; intentionally not a noisy CI perf gate.tests/bench_qwen36_dense_batch$(EXE): tests/bench_qwen36_dense_batch.c qwen36.c qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)$(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)diff --cc c/qwen36.cindex aceb5331,49ced244..86e20043--- a/c/qwen36.c+++ b/c/qwen36.c@@@ -629,28 -662,21 +662,40 @@@ typedef structint expert_gs; /* expert scale group size along input dim; 0 = per-row */} Cfg;+/* ---- Dense int8: a dense matrix that is quantized to int8 during load+ * (load_tq, below matmul_d) instead of loaded as f32 and quantized in a+ * separate pass afterward -- so the f32 staging buffer for THIS matrix alone+ * is what's briefly resident, not every dense matrix in the model at once.+ * `w` is the f32 copy: kept (and `q`/`sc` left NULL) when COLI_DENSE_I8=0,+ * the reference/parity path; freed once `q`/`sc` are populated otherwise+ * (COLI_KEEP_F32=1 keeps it alongside them, for debugging). matmul_d+ * dispatches on q!=NULL directly -- no pointer-keyed scan. */- typedef struct { const float *w; int8_t *q; float *sc; int I, O; } QW;- static void qw_free(QW *w) { free((void*)w->w); free(w->q); free(w->sc); w->w = NULL; w->q = NULL; w->sc = NULL; }++/* q/sc: int8 rows with one scale per row (the classic copy, what the VRAM++ * tier uploads). q4/sg: the same matrix as int4 planar blocks of 64 with one++ * scale per group (COLI_DENSE_BITS=4), the layout the K1b grouped kernel++ * reads; ng = I/64 groups per row. */++typedef struct { const float *w; int8_t *q; float *sc; int I, O; uint8_t *q4; float *sg; int ng; } QW;++static void qw_free(QW *w) {++ free((void*)w->w); free(w->q); free(w->sc); free(w->q4); free(w->sg);++ w->w = NULL; w->q = NULL; w->sc = NULL; w->q4 = NULL; w->sg = NULL; w->ng = 0;++}+/* ---------- per-layer dense weights ---------- */typedef struct {- float *in_ln, *post_ln, *q, *k, *v, *o, *qn, *kn, *gate, *gate_bias;- float *sh_g, *sh_u, *sh_d, *sh_gate; /* shared expert (dense f32) + shared_expert_gate */+ float *in_ln, *post_ln, *qn, *kn, *gate_bias;+ QW q, k, v, o, gate;+ QW sh_g, sh_u, sh_d; float *sh_gate; /* shared expert (dense, int8-during-load) + shared_expert_gate *//* Gated DeltaNet (linear_attention) dense weights (f16->f32 via st_read_f32). */- float *dn_qkv, *dn_z, *dn_b, *dn_a; /* in_proj_qkv/z/b/a */+ QW dn_qkv, dn_z; float *dn_b, *dn_a; /* in_proj_qkv/z (int8-during-load), b/a (not dense-matmul'd) */float *dn_conv; /* conv1d.weight [conv_dim, convk] (groups=conv_dim) */float *dn_dtbias, *dn_alog; /* dt_bias[vh], A_log[vh] */float *dn_norm; /* RMSNormGated weight [vdim] */- float *dn_out; /* out_proj [hidden, value_dim] */+ QW dn_out; /* out_proj [hidden, value_dim] */+ /* VRAM copies the tier placed (qt_dense handle + 1, 0 = stays on the CPU):+ * the DeltaNet out_proj, the attention q/k/v/o and the shared expert's+ * three matrices. Offered per layer as "dnout", "attnproj", "shexp";+ * see trunk_offer_dense / trunk_place_dense. */+ int qth_dnout, qth_q, qth_k, qth_v, qth_o, qth_shg, qth_shu, qth_shd;} Layer;/* ---------- LRU expert cache (int8 weights + per-row float scales) ---------- */@@@ -1010,9 -1041,134 +1061,128 @@@ static void matmul_qe(float *y, const f}/* ---- Dense int8: per-row quantized copies of the large f32 matrices.- * matmul_d dispatches via pointer lookup to matmul_q; COLI_DENSE_I8=0 falls- * back to f32 (reference path for parity tests). ~4x less memory traffic. */-#define QDW_MAX 1024-/* q/sc: int8 rows with one scale per row (the classic copy, what the VRAM- * tier uploads). q4/sg: the same matrix as int4 planar blocks of 64 with one- * scale per group (COLI_DENSE_BITS=4), the layout the K1b grouped kernel- * reads; ng = I/64 groups per row. */-static struct { const float *w; int8_t *q; float *sc; int I, O; uint8_t *q4; float *sg; int ng; } g_qdw[QDW_MAX];-static int g_qdw_n = 0;+ * matmul_d dispatches directly off QW.q (no pointer-keyed scan -- see QW,+ * above Layer); COLI_DENSE_I8=0 falls back to f32 (QW.w, reference path for+ * parity tests). ~4x less memory traffic. */+ /* ---- Dense trunk, integer dot products ------------------------------------+ *+ * Every dense GEMV of a token (DeltaNet projections and out_proj, attention+ * q/k/v/o, the shared expert, lm_head) used to multiply int8 weights by f32+ * activations: each weight byte converted to f32 and fed to an FMA, eight+ * weights per instruction. Measured on lm_head (248320 x 2048 int8, 508 MB)+ * that runs at 29 GB/s on a 16-core AVX-512 host whose memory bus does 80:+ * the kernel, not the bus, was the limit, and the dense part of a token is+ * 1.9 GB of int8 on the 35B, three times the routed experts.+ *+ * The activation is now quantized to int8 once per call (one scale,+ * amax/127, the qrow_i8 contract the expert IDOT already uses) and the dot+ * is integer: 32 weights per instruction on AVX2 (maddubs), 64 on AVX-512+ * VNNI, exact int32 sums scaled once per output. Not bit-identical to the+ * f32 path (the activation is rounded): measured on the 35B, +1.0%+ * perplexity on 4 x 512 tokens, lm_head 12.6 to 10.2 ms/token, decode+ * 6.71 to 7.35 tok/s alone and 8.23 with the experts' int8 activations.+ * COLI_DENSE_IDOT=0 restores the f32-activation kernel.+ *+ * COLI_DENSE_BITS=4 additionally stores the dense matrices as int4 in blocks+ * of 64 with one scale per block, the K1b planar layout, halving the bytes+ * the token reads; lm_head alone goes from 508 to 254 MB. It implies the+ * integer dot (that layout has no f32 kernel). Same gate: measured. */+ static int dense_idot_on(void){ static int v=-1; if(v<0){ const char *e=getenv("COLI_DENSE_IDOT"); v=!(e&&*e=='0'); } return v; }+ static int dense_bits(void){ static int v=-1; if(v<0){ const char *e=getenv("COLI_DENSE_BITS"); v=(e&&atoi(e)==4)?4:8; } return v; }++ /* Activation -> int8 with one scale, plus the int32 sum of every block of 64+ * (the K1b kernel subtracts 8*sum per group because its nibbles are unsigned).+ * Vectorized: the scalar lrintf loop was measured at ~7 us for 4096 values,+ * which times ~150 GEMVs per token is a millisecond thrown away. Rounding is+ * to nearest even in both paths, so the vector path equals qrow_i8 bit for bit. */+ static float dense_act_i8(const float *x, int I, int8_t *xq, int32_t *xsg){+ float amax = 0.f;+ int i = 0;+ #ifdef __AVX2__+ {+ __m256 am = _mm256_setzero_ps();+ const __m256 sign = _mm256_set1_ps(-0.0f);+ for (; i + 8 <= I; i += 8) am = _mm256_max_ps(am, _mm256_andnot_ps(sign, _mm256_loadu_ps(x + i)));+ float tmp[8]; _mm256_storeu_ps(tmp, am);+ for (int k = 0; k < 8; k++) if (tmp[k] > amax) amax = tmp[k];+ }+ #endif+ for (; i < I; i++) { float a = fabsf(x[i]); if (a > amax) amax = a; }+ float s = amax / 127.f; if (s < 1e-12f) s = 1e-12f;+ float inv = 1.f / s;+ i = 0;+ #ifdef __AVX2__+ {+ const __m256 vinv = _mm256_set1_ps(inv);+ for (; i + 32 <= I; i += 32) {+ __m256i a = _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_loadu_ps(x + i), vinv));+ __m256i b = _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_loadu_ps(x + i + 8), vinv));+ __m256i c = _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_loadu_ps(x + i + 16), vinv));+ __m256i d = _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_loadu_ps(x + i + 24), vinv));+ /* packs interleave 128-bit lanes: fix the order with one permute */+ __m256i ab = _mm256_packs_epi32(a, b), cd = _mm256_packs_epi32(c, d);+ __m256i abcd = _mm256_packs_epi16(ab, cd);+ abcd = _mm256_permutevar8x32_epi32(abcd, _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7));+ _mm256_storeu_si256((__m256i *)(xq + i), abcd);+ }+ }+ #endif+ for (; i < I; i++) xq[i] = (int8_t)lrintf(x[i] * inv);+ if (xsg) {+ int ng = I / 64;+ for (int g = 0; g < ng; g++) {+ int32_t sum = 0;+ for (int k = 0; k < 64; k++) sum += xq[g * 64 + k];+ xsg[g] = sum;+ }+ }+ return s;+ }++ /* f32 rows -> int4 in blocks of 64 with one f32 scale per block, packed as the+ * K1b planar layout (unsigned nibbles v+8, block b: lo nibbles = elements+ * b*64..b*64+31, hi = b*64+32..b*64+63). The quantizer is the symmetric+ * absmax/7 the expert containers use. */+ static void pack_int4_g64_planar(const float *w, uint8_t *q4, float *sg, int O, int I){+ int rb = I / 2, ng = I / 64;+ #pragma omp parallel for schedule(static)+ for (int o = 0; o < O; o++) {+ const float *wr = w + (int64_t)o * I;+ uint8_t *row = q4 + (int64_t)o * rb;+ float *sr = sg + (int64_t)o * ng;+ for (int g = 0; g < ng; g++) {+ const float *blk = wr + g * 64;+ float amax = 0.f;+ for (int k = 0; k < 64; k++) { float a = fabsf(blk[k]); if (a > amax) amax = a; }+ /* absmax/7 is where the search starts; the scale is then refined+ * by least squares against the rounded codes (s = w.q / q.q) and+ * the candidate with the smallest squared error wins. Three+ * rounds: measured on the 35B this recovers a third of the+ * perplexity absmax alone loses at 4 bits. */+ float best_s = amax / 7.f; if (best_s < 1e-8f) best_s = 1e-8f;+ int best_q[64]; double best_err = 1e30;+ float s = best_s;+ for (int round = 0; round < 4; round++) {+ int q[64]; double err = 0, wq = 0, qq = 0;+ float inv = 1.f / s;+ for (int k = 0; k < 64; k++) {+ int v = (int)lrintf(blk[k] * inv); if (v > 7) v = 7; if (v < -8) v = -8;+ q[k] = v; double d = (double)blk[k] - (double)v * s; err += d * d;+ wq += (double)blk[k] * v; qq += (double)v * v;+ }+ if (err < best_err) { best_err = err; best_s = s; memcpy(best_q, q, sizeof q); }+ if (qq <= 0) break;+ float ns = (float)(wq / qq); /* least-squares scale for these codes */+ if (ns <= 0.f || ns == s) break;+ s = ns;+ }+ sr[g] = best_s;+ uint8_t *dst = row + g * 32;+ for (int k = 0; k < 32; k++)+ dst[k] = (uint8_t)((best_q[k] + 8) | ((best_q[k + 32] + 8) << 4));+ }+ }+ }#ifdef COLI_QWEN_BATCH_TESTstatic uint64_t g_qwen_matmul_d_calls;#endif@@@ -1022,15 -1178,30 +1192,36 @@@ static int dense_i8_on(void){ static in* the time. */static int kv_prefix_off(void){ const char *e=getenv("COLI_KV_PREFIX"); return e && *e=='0'; }static int dense_batch_on(void){ const char *e=getenv("QWEN_DENSE_BATCH"); return !(e&&*e=='0'); }+ /* COLI_DENSE_INT4 names which components take the int4 copy when+ * COLI_DENSE_BITS=4: a comma list of lmhead, dnproj, dnout, attn, shexp,+ * router; unset means all of them. The parts of the trunk pay 4 bits+ * differently (measured: the attention projections and the DeltaNet input+ * projections cost the most perplexity, lm_head the least per byte saved),+ * so the default is chosen per component from the numbers, not for the+ * whole trunk at once. */+ static int dense_int4_wanted(const char *tag){+ if (dense_bits() != 4 || !tag) return 0;+ const char *e = getenv("COLI_DENSE_INT4");+ if (!e || !*e) return 1;+ size_t n = strlen(tag);+ for (const char *p = e; *p; ) {+ while (*p == ',' || *p == ' ') p++;+ const char *q = p; while (*q && *q != ',' && *q != ' ') q++;+ if ((size_t)(q - p) == n && !strncmp(p, tag, n)) return 1;+ p = q;+ }+ return 0;+ }-static void qdw_register_as(const float *W, int I, int O, const char *tag){- if (!W || !dense_i8_on() || g_qdw_n >= QDW_MAX) return;+/* Per-row max-abs / round-clamp int8 quantization of an in-memory f32 matrix- * W [O][I] row-major -- same math load_tq used to run post-hoc via qdw_register,- * factored out so it can also run on a caller-owned buffer directly (tests- * that synthesize weights in memory, without a shard file to load from). Does- * not touch out->w -- the caller sets that (or leaves it, e.g. load_tq frees- * it right after). */- static void qw_quantize(const float *W, int I, int O, QW *out) {++ * W [O][I] row-major -- same math load_tq runs during streamed load, factored++ * out so it can also run on a caller-owned buffer directly (tests that++ * synthesize weights in memory, without a shard file to load from). Does not++ * touch out->w -- the caller sets that (or leaves it, e.g. load_tq frees it++ * right after). `tag` selects the int4 planar copy per dense_int4_wanted++ * (NULL/COLI_DENSE_BITS!=4: skipped, out->q4 stays NULL). */++static void qw_quantize(const float *W, int I, int O, const char *tag, QW *out) {int8_t *q = malloc((size_t)O*I); float *sc = malloc((size_t)O*sizeof(float));- if (!q || !sc) { free(q); free(sc); return; }+ if (!q || !sc) { fprintf(stderr, "OOM qw_quantize\n"); exit(1); }#pragma omp parallel for schedule(static)for (int o = 0; o < O; o++) {const float *r = W + (int64_t)o*I; float am = 0.f;@@@ -1039,21 -1210,71 +1230,63 @@@int8_t *d = q + (int64_t)o*I;for (int i = 0; i < I; i++) { int v = (int)lrintf(r[i]*inv); if (v>127) v=127; if (v<-127) v=-127; d[i] = (int8_t)v; }}- g_qdw[g_qdw_n].w=W; g_qdw[g_qdw_n].q=q; g_qdw[g_qdw_n].sc=sc; g_qdw[g_qdw_n].I=I; g_qdw[g_qdw_n].O=O;- g_qdw[g_qdw_n].q4=NULL; g_qdw[g_qdw_n].sg=NULL; g_qdw[g_qdw_n].ng=0;- if (dense_int4_wanted(tag) && I%64==0) {- uint8_t *q4=malloc((size_t)O*(I/2)); float *sg=malloc((size_t)O*(I/64)*sizeof(float));+ out->q = q; out->sc = sc; out->I = I; out->O = O;++ out->q4 = NULL; out->sg = NULL; out->ng = 0;++ if (dense_int4_wanted(tag) && I % 64 == 0) {++ uint8_t *q4 = malloc((size_t)O*(I/2)); float *sg = malloc((size_t)O*(I/64)*sizeof(float));+ if (q4 && sg) {+ pack_int4_g64_planar(W, q4, sg, O, I);- g_qdw[g_qdw_n].q4=q4; g_qdw[g_qdw_n].sg=sg; g_qdw[g_qdw_n].ng=I/64;++ out->q4 = q4; out->sg = sg; out->ng = I/64;+ } else { free(q4); free(sg); }+ }- g_qdw_n++;}-static void qdw_register(const float *W, int I, int O){ qdw_register_as(W, I, O, NULL); }-static void matmul_d(float *y, const float *x, const float *W, int S, int I, int O){+static void matmul_d(float *y, const float *x, const QW *w, int S, int I, int O){#ifdef COLI_QWEN_BATCH_TESTg_qwen_matmul_d_calls++;#endif- for (int i = 0; i < g_qdw_n; i++) if (g_qdw[i].w == W && g_qdw[i].I == I) {- if (g_qdw[i].q4 || dense_idot_on()) {+ if (w->q) {++ if (w->q4 || dense_idot_on()) {+ /* integer dot: the activation rows to int8 once, then the K1b+ * grouped kernel (int4 planar) or the per-row int8 kernel */+ int ng = I / 64;+ int8_t *xq = malloc((size_t)S * I);+ float *sx = malloc((size_t)S * sizeof(float));- int32_t *xsg = g_qdw[i].q4 ? malloc((size_t)S * ng * sizeof(int32_t)) : NULL;- if (xq && sx && (!g_qdw[i].q4 || xsg)) {++ int32_t *xsg = w->q4 ? malloc((size_t)S * ng * sizeof(int32_t)) : NULL;++ if (xq && sx && (!w->q4 || xsg)) {+ for (int s = 0; s < S; s++)+ sx[s] = dense_act_i8(x + (int64_t)s * I, I, xq + (int64_t)s * I, xsg ? xsg + (int64_t)s * ng : NULL);- if (g_qdw[i].q4) matmul_i4p_grouped_idot(y, xq, sx, xsg, g_qdw[i].q4, g_qdw[i].sg, S, I, O, 64);- else matmul_q_idot(y, xq, sx, g_qdw[i].q, g_qdw[i].sc, S, I, O);++ if (w->q4) matmul_i4p_grouped_idot(y, xq, sx, xsg, w->q4, w->sg, S, I, O, 64);++ else matmul_q_idot(y, xq, sx, w->q, w->sc, S, I, O);+ free(xq); free(sx); free(xsg);+ return;+ }+ free(xq); free(sx); free(xsg); /* out of memory: the f32 path below */+ }if (S > 1 && dense_batch_on())- matmul_q_batch(y, x, g_qdw[i].q, g_qdw[i].sc, S, I, O);+ matmul_q_batch(y, x, w->q, w->sc, S, I, O);else- for (int s = 0; s < S; s++) matmul_q(y+(int64_t)s*O, x+(int64_t)s*I, g_qdw[i].q, g_qdw[i].sc, I, O);+ for (int s = 0; s < S; s++) matmul_q(y+(int64_t)s*O, x+(int64_t)s*I, w->q, w->sc, I, O);return;}- matmul(y, x, W, S, I, O);+ matmul(y, x, w->w, S, I, O);}+ /* A dense matrix the tier placed in VRAM (handle+1 kept in the Layer, 0 = CPU):+ * one GEMV from the device, or 0 and the caller runs matmul_d as before. The+ * tier turns a failing handle off itself, so the fallback is permanent. */+ static inline int qtd(int hp1, float *y, const float *x, int I, int O){+ return hp1 > 0 && qt_dense_matmul(hp1 - 1, y, x, I, O);+ }-/* Bytes of W's dense-i8 copy (int8 rows + per-row scales), 0 when there is++/* Bytes of w's dense-i8 copy (int8 rows + per-row scales), 0 when there is+ * none (COLI_DENSE_I8=0): nothing to offer, the CPU path stands. */-static size_t qdw_bytes(const float *W){- for (int i = 0; i < g_qdw_n; i++)- if (g_qdw[i].w == W) return (size_t)g_qdw[i].I * g_qdw[i].O + (size_t)g_qdw[i].O * sizeof(float);- return 0;++static size_t qdw_bytes(const QW *w){++ return w->q ? (size_t)w->I * w->O + (size_t)w->O * sizeof(float) : 0;+ }-/* Upload W's dense-i8 copy to `dev`; handle+1, or 0 when it stays on the CPU. */-static int qdw_place(const float *W, int dev){- if (dev == QT_PLACE_CPU || !W) return 0;- for (int i = 0; i < g_qdw_n; i++)- if (g_qdw[i].w == W) {- int h = qt_dense_init(g_qdw[i].q, g_qdw[i].sc, g_qdw[i].I, g_qdw[i].O, dev);- return h >= 0 ? h + 1 : 0;- }- return 0;++/* Upload w's dense-i8 copy to `dev`; handle+1, or 0 when it stays on the CPU. */++static int qdw_place(const QW *w, int dev){++ if (dev == QT_PLACE_CPU || !w->q) return 0;++ int h = qt_dense_init(w->q, w->sc, w->I, w->O, dev);++ return h >= 0 ? h + 1 : 0;+ }/* rmsnorm over a row of length D (in-place capable: out may == x).* Qwen3_5MoeRMSNorm: out = (x * rsqrt(mean(x^2)+eps)) * (1.0 + weight). */@@@ -1270,24 -1491,6 +1503,27 @@@ static float *load_t_n(Model *m, constreturn p;}- /* Dense matrix load, quantized to int8 DURING loading rather than in a- * separate pass over the whole model afterward (see QW, above Layer):- * reads `name` (I*O elements, same size discipline as load_t_n), and when- * `quantize` && COLI_DENSE_I8 is on, quantizes it via qw_quantize and frees- * the f32 staging buffer right away -- so at most one dense matrix's f32 copy- * is ever resident at a time, not the whole model's. `quantize` is false for- * loaders that never ran through the old post-hoc qdw_register pass either- * (the Segment/Edge adapters build partial or auxiliary models straight off++/* Dense matrix load, quantized to int8 (+ int4 planar per `tag`, see++ * dense_int4_wanted) DURING loading rather than in a separate pass over the++ * whole model afterward (see QW, above Layer): reads `name` (I*O elements,++ * same size discipline as load_t_n), and when `quantize` && COLI_DENSE_I8 is++ * on, quantizes it via qw_quantize and frees the f32 staging buffer right++ * away -- so at most one dense matrix's f32 copy is ever resident at a time,++ * not the whole model's. `quantize` is false for loaders that never ran++ * through the old post-hoc qdw_register pass either (the Segment/Edge++ * adapters build partial or auxiliary models straight off+ * model_init_range/load_t_n, never main()'s dense-i8 block) -- passing it- * through keeps their f32-only behavior exactly as it was. */- static void load_tq(Model *m, const char *name, int I, int O, int quantize, QW *out) {++ * through keeps their f32-only behavior exactly as it was; `tag` is unused++ * on that path. */++static void load_tq(Model *m, const char *name, int I, int O, int quantize, const char *tag, QW *out) {+ float *p = load_t_n(m, name, (int64_t)I * O);+ out->w = p; out->q = NULL; out->sc = NULL; out->I = I; out->O = O;++ out->q4 = NULL; out->sg = NULL; out->ng = 0;+ if (!quantize || !dense_i8_on()) return;- qw_quantize(p, I, O, out);++ qw_quantize(p, I, O, tag, out);+ if (getenv("COLI_KEEP_F32")) out->w = p; else { free(p); out->w = NULL; }+}+static void model_init_range(Model *m, const char *snap, int cap, int bits,int layer_begin, int layer_end,int load_boundaries, int allocate_state) {@@@ -1313,17 -1516,9 +1549,17 @@@exit(1);}double t0 = now_s();+ /* Quantize during load only for the full-model path (main()'s static Model,+ * load_boundaries=1): the Segment/Edge adapters build partial or auxiliary+ * models straight off this same loop and never ran the old post-hoc+ * qdw_register pass either, so gating on load_boundaries keeps their+ * f32-only numerics exactly as they were. */+ int quantize_dense = load_boundaries && dense_i8_on();+ int qcount = 0; double qfreed = 0;if (load_boundaries) {- m->embed = load_t_n(m, "model.embed_tokens.weight", (int64_t)c->vocab * c->hidden);- m->lm_head = load_t_n(m, "lm_head.weight", (int64_t)c->vocab * c->hidden);+ m->embed = load_t_n(m, "model.embed_tokens.weight", (int64_t)c->vocab * c->hidden);- load_tq(m, "lm_head.weight", c->hidden, c->vocab, quantize_dense, &m->lm_head);++ load_tq(m, "lm_head.weight", c->hidden, c->vocab, quantize_dense, "lmhead", &m->lm_head);+ if (m->lm_head.q) { qcount++; qfreed += (double)c->hidden * c->vocab * sizeof(float); }m->final_norm = load_t_n(m, "model.norm.weight", c->hidden);}m->L = calloc((size_t)c->n_layers, sizeof(Layer));@@@ -1342,10 -1535,8 +1578,10 @@@#define LD(field, suffix, want) snprintf(nm,sizeof(nm),"model.layers.%d." suffix,ai); l->field = load_t_n(m,nm,(want))LD(in_ln, "input_layernorm.weight", c->hidden);LD(post_ln,"post_attention_layernorm.weight", c->hidden);- LD(gate, "mlp.gate.weight", (int64_t)c->n_experts * c->hidden);#undef LD+ snprintf(nm,sizeof(nm),"model.layers.%d.mlp.gate.weight", ai);- load_tq(m, nm, c->hidden, c->n_experts, quantize_dense, &l->gate);++ load_tq(m, nm, c->hidden, c->n_experts, quantize_dense, "router", &l->gate);+ QCOUNT(l->gate);/* q/k norms are per-head [head_dim]; only on attention layers, load if present */if (c->has_qk_norm) {snprintf(nm,sizeof(nm),"model.layers.%d.self_attn.q_norm.weight", ai);@@@ -1357,51 -1548,42 +1593,51 @@@snprintf(nm,sizeof(nm),"model.layers.%d.mlp.gate.e_score_correction_bias", ai);if (st_has(&m->S, nm)) { l->gate_bias = falloc(c->n_experts); st_read_f32(&m->S, nm, l->gate_bias, 0); }else l->gate_bias = NULL;- /* shared expert (dense f32) */- #define LD2(field, suffix, want) snprintf(nm,sizeof(nm),"model.layers.%d.mlp.shared_expert." suffix,ai); l->field = load_t_n(m,nm,(want))- LD2(sh_g, "gate_proj.weight", (int64_t)c->shared_inter * c->hidden);- LD2(sh_u, "up_proj.weight", (int64_t)c->shared_inter * c->hidden);- LD2(sh_d, "down_proj.weight", (int64_t)c->hidden * c->shared_inter);- #undef LD2+ /* shared expert (dense, int8-during-load) */+ snprintf(nm,sizeof(nm),"model.layers.%d.mlp.shared_expert.gate_proj.weight", ai);- load_tq(m, nm, c->hidden, c->shared_inter, quantize_dense, &l->sh_g); QCOUNT(l->sh_g);++ load_tq(m, nm, c->hidden, c->shared_inter, quantize_dense, "shexp", &l->sh_g); QCOUNT(l->sh_g);+ snprintf(nm,sizeof(nm),"model.layers.%d.mlp.shared_expert.up_proj.weight", ai);- load_tq(m, nm, c->hidden, c->shared_inter, quantize_dense, &l->sh_u); QCOUNT(l->sh_u);++ load_tq(m, nm, c->hidden, c->shared_inter, quantize_dense, "shexp", &l->sh_u); QCOUNT(l->sh_u);+ snprintf(nm,sizeof(nm),"model.layers.%d.mlp.shared_expert.down_proj.weight", ai);- load_tq(m, nm, c->shared_inter, c->hidden, quantize_dense, &l->sh_d); QCOUNT(l->sh_d);++ load_tq(m, nm, c->shared_inter, c->hidden, quantize_dense, "shexp", &l->sh_d); QCOUNT(l->sh_d);/* shared_expert_gate: Linear(hidden -> 1), sigmoid-gated shared expert */snprintf(nm,sizeof(nm),"model.layers.%d.mlp.shared_expert_gate.weight", ai);l->sh_gate = st_has(&m->S, nm) ? load_t_n(m, nm, c->hidden) : NULL;if (c->is_attn[i]) {- /* Gated Attention (full_attention) layer */- #define LD3(field, suffix, want) snprintf(nm,sizeof(nm),"model.layers.%d.self_attn." suffix,ai); l->field = load_t_n(m,nm,(want))- LD3(q, "q_proj.weight", (int64_t)c->q_heads * c->q_head_dim * c->hidden);- LD3(k, "k_proj.weight", (int64_t)c->kv_heads * c->k_head_dim * c->hidden);- LD3(v, "v_proj.weight", (int64_t)c->kv_heads * c->v_head_dim * c->hidden);- LD3(o, "o_proj.weight", (int64_t)c->hidden * c->o_in);- #undef LD3- l->dn_qkv=l->dn_z=l->dn_b=l->dn_a=l->dn_conv=NULL;- l->dn_dtbias=l->dn_alog=l->dn_norm=l->dn_out=NULL;+ /* Gated Attention (full_attention) layer, dense projections int8-during-load */+ snprintf(nm,sizeof(nm),"model.layers.%d.self_attn.q_proj.weight", ai);- load_tq(m, nm, c->hidden, q_out, quantize_dense, &l->q); QCOUNT(l->q);++ load_tq(m, nm, c->hidden, q_out, quantize_dense, "attn", &l->q); QCOUNT(l->q);+ snprintf(nm,sizeof(nm),"model.layers.%d.self_attn.k_proj.weight", ai);- load_tq(m, nm, c->hidden, kv_out, quantize_dense, &l->k); QCOUNT(l->k);++ load_tq(m, nm, c->hidden, kv_out, quantize_dense, "attn", &l->k); QCOUNT(l->k);+ snprintf(nm,sizeof(nm),"model.layers.%d.self_attn.v_proj.weight", ai);- load_tq(m, nm, c->hidden, kv_out, quantize_dense, &l->v); QCOUNT(l->v);++ load_tq(m, nm, c->hidden, kv_out, quantize_dense, "attn", &l->v); QCOUNT(l->v);+ snprintf(nm,sizeof(nm),"model.layers.%d.self_attn.o_proj.weight", ai);- load_tq(m, nm, c->o_in, c->hidden, quantize_dense, &l->o); QCOUNT(l->o);++ load_tq(m, nm, c->o_in, c->hidden, quantize_dense, "attn", &l->o); QCOUNT(l->o);+ l->dn_qkv=l->dn_z=(QW){0}; l->dn_b=l->dn_a=l->dn_conv=NULL;+ l->dn_dtbias=l->dn_alog=l->dn_norm=NULL; l->dn_out=(QW){0};} else {/* Gated DeltaNet (linear_attention) layer */- l->q=l->k=l->v=l->o=NULL;+ l->q=l->k=l->v=l->o=(QW){0};#define LD4(field, suffix, want) snprintf(nm,sizeof(nm),"model.layers.%d.linear_attn." suffix,ai); l->field = load_t_n(m,nm,(want))int64_t vdim_tot = (int64_t)c->dn_vheads * c->dn_vdim;- LD4(dn_qkv, "in_proj_qkv.weight", (int64_t)c->dn_conv_dim * c->hidden);- LD4(dn_z, "in_proj_z.weight", vdim_tot * c->hidden);+ snprintf(nm,sizeof(nm),"model.layers.%d.linear_attn.in_proj_qkv.weight", ai);- load_tq(m, nm, c->hidden, c->dn_conv_dim, quantize_dense, &l->dn_qkv); QCOUNT(l->dn_qkv);++ load_tq(m, nm, c->hidden, c->dn_conv_dim, quantize_dense, "dnproj", &l->dn_qkv); QCOUNT(l->dn_qkv);+ snprintf(nm,sizeof(nm),"model.layers.%d.linear_attn.in_proj_z.weight", ai);- load_tq(m, nm, c->hidden, (int)vdim_tot, quantize_dense, &l->dn_z); QCOUNT(l->dn_z);++ load_tq(m, nm, c->hidden, (int)vdim_tot, quantize_dense, "dnproj", &l->dn_z); QCOUNT(l->dn_z);LD4(dn_b, "in_proj_b.weight", (int64_t)c->dn_vheads * c->hidden);LD4(dn_a, "in_proj_a.weight", (int64_t)c->dn_vheads * c->hidden);LD4(dn_conv,"conv1d.weight", (int64_t)c->dn_conv_dim * c->dn_convk);LD4(dn_dtbias, "dt_bias", c->dn_vheads);LD4(dn_alog,"A_log", c->dn_vheads);LD4(dn_norm, "norm.weight", c->dn_vdim);- LD4(dn_out, "out_proj.weight", (int64_t)c->hidden * vdim_tot);#undef LD4+ snprintf(nm,sizeof(nm),"model.layers.%d.linear_attn.out_proj.weight", ai);- load_tq(m, nm, (int)vdim_tot, c->hidden, quantize_dense, &l->dn_out); QCOUNT(l->dn_out);++ load_tq(m, nm, (int)vdim_tot, c->hidden, quantize_dense, "dnout", &l->dn_out); QCOUNT(l->dn_out);}}+ #undef QCOUNT+ if (quantize_dense)+ fprintf(stderr, "[dense-i8] %d matrices quantized during load, %.1f GB f32 freed\n", qcount, qfreed/1073741824.0);m->cache = calloc((size_t)c->n_layers, sizeof(LCache));for (int i = layer_begin; i < layer_end; i++) {m->cache[i].cap = cap;@@@ -1847,9 -2033,11 +2083,11 @@@ static void attention(Model *m, Layer *float *q = falloc((int64_t)S*q_out);float *k = falloc((int64_t)S*kv_out);float *vv= falloc((int64_t)S*kv_out);- matmul_d(q, x, &l->q, S, D, q_out);- matmul_d(k, x, &l->k, S, D, kv_out);- matmul_d(vv, x, &l->v, S, D, kv_out);+ /* Decode (S == 1): the projections the tier placed answer from VRAM,+ * one GEMV each; a prompt batch keeps the batched CPU matmul. */- if (!(S == 1 && qtd(l->qth_q, q, x, D, q_out))) matmul_d(q, x, l->q, S, D, q_out);- if (!(S == 1 && qtd(l->qth_k, k, x, D, kv_out))) matmul_d(k, x, l->k, S, D, kv_out);- if (!(S == 1 && qtd(l->qth_v, vv, x, D, kv_out))) matmul_d(vv, x, l->v, S, D, kv_out);++ if (!(S == 1 && qtd(l->qth_q, q, x, D, q_out))) matmul_d(q, x, &l->q, S, D, q_out);++ if (!(S == 1 && qtd(l->qth_k, k, x, D, kv_out))) matmul_d(k, x, &l->k, S, D, kv_out);++ if (!(S == 1 && qtd(l->qth_v, vv, x, D, kv_out))) matmul_d(vv, x, &l->v, S, D, kv_out);/* split q into query (first hd) and gate (next gate_dim), both per head */float *query = falloc((int64_t)S*H*hd);float *gate = falloc((int64_t)S*H*gate_dim);@@@ -1911,7 -2099,7 +2149,7 @@@float g = gate_dim ? gate[o] : 0.f;ag[o] = ctx[o] * (1.f / (1.f + expf(-g)));}- matmul_d(out, ag, &l->o, S, H*hd, D);- if (!(S == 1 && qtd(l->qth_o, out, ag, H*hd, D))) matmul_d(out, ag, l->o, S, H*hd, D);++ if (!(S == 1 && qtd(l->qth_o, out, ag, H*hd, D))) matmul_d(out, ag, &l->o, S, H*hd, D);free(q); free(k); free(vv); free(query); free(gate); free(ctx); free(ag);}@@@ -1944,10 -2132,10 +2182,10 @@@ static void qwen_shared_experts_cpu(Modif (B == 1) {for (int s=0;s<S;s++) {const float *xs=x+(int64_t)s*D;- matmul_d(g,xs,&l->sh_g,1,D,I);- matmul_d(u,xs,&l->sh_u,1,D,I);- if(!qtd(l->qth_shg,g,xs,D,I)) matmul_d(g,xs,l->sh_g,1,D,I);- if(!qtd(l->qth_shu,u,xs,D,I)) matmul_d(u,xs,l->sh_u,1,D,I);++ if(!qtd(l->qth_shg,g,xs,D,I)) matmul_d(g,xs,&l->sh_g,1,D,I);++ if(!qtd(l->qth_shu,u,xs,D,I)) matmul_d(u,xs,&l->sh_u,1,D,I);for(int i=0;i<I;i++){float sv=g[i];g[i]=(sv/(1.f+expf(-sv)))*u[i];}- matmul_d(hh,g,&l->sh_d,1,I,D);- if(!qtd(l->qth_shd,hh,g,I,D)) matmul_d(hh,g,l->sh_d,1,I,D);++ if(!qtd(l->qth_shd,hh,g,I,D)) matmul_d(hh,g,&l->sh_d,1,I,D);float sgate=1.f;if(l->sh_gate){float sg=0.f;for(int i=0;i<D;i++)sg+=xs[i]*l->sh_gate[i];sgate=1.f/(1.f+expf(-sg));}float *os=out+(int64_t)s*D;@@@ -2229,10 -2417,10 +2467,10 @@@ static void moe(Model *m, Layer *l, in{double _ts2 = tm_now();int Ish = c->shared_inter;- matmul_d(sh, xs, &l->sh_g, 1, D, Ish);- matmul_d(shu, xs, &l->sh_u, 1, D, Ish);- if (!qtd(l->qth_shg, sh, xs, D, Ish)) matmul_d(sh, xs, l->sh_g, 1, D, Ish);- if (!qtd(l->qth_shu, shu, xs, D, Ish)) matmul_d(shu, xs, l->sh_u, 1, D, Ish);++ if (!qtd(l->qth_shg, sh, xs, D, Ish)) matmul_d(sh, xs, &l->sh_g, 1, D, Ish);++ if (!qtd(l->qth_shu, shu, xs, D, Ish)) matmul_d(shu, xs, &l->sh_u, 1, D, Ish);for (int i = 0; i < Ish; i++) { float sv = sh[i]; sh[i] = (sv / (1.f + expf(-sv))) * shu[i]; }- matmul_d(shd, sh, &l->sh_d, 1, Ish, D);- if (!qtd(l->qth_shd, shd, sh, Ish, D)) matmul_d(shd, sh, l->sh_d, 1, Ish, D);++ if (!qtd(l->qth_shd, shd, sh, Ish, D)) matmul_d(shd, sh, &l->sh_d, 1, Ish, D);float sgate = 1.f;if (l->sh_gate) {float sg = 0.f; const float *wg = l->sh_gate;@@@ -2421,7 -2609,8 +2659,8 @@@ static void deltanet(Model *m, Layer *loutr[(int64_t)h * vdim + d] = val * zr[d] / (1.f + expf(-zr[d]));}}- matmul_d(out + (int64_t)s * H, outr, &l->dn_out, 1, value_dim, H);+ if (!qtd(l->qth_dnout, out + (int64_t)s * H, outr, value_dim, H))- matmul_d(out + (int64_t)s * H, outr, l->dn_out, 1, value_dim, H);++ matmul_d(out + (int64_t)s * H, outr, &l->dn_out, 1, value_dim, H);if (tm_on() && S==1){ g_dn_sub[3]+=tm_now()-_d0; }if (layer == 0 && s == 0 && getenv("DN_DBG")) {FILE *dbg = fopen(getenv("DN_DBG"), "wb");@@@ -2445,6 -2634,118 +2684,117 @@@free(conv_out); free(q); free(k); free(outv); free(outr); free(kv); free(delta);}+ /* The rest of the dense trunk, offered to the placer by name and layer with+ * the bytes of the dense-i8 copies (docs/qwen36-cuda-tier.md, "Placement"):+ * dnout -- the DeltaNet out_proj, one matrix per DeltaNet layer+ * attnproj -- q, k, v, o of every attention layer, offered as one item+ * shexp -- gate, up, down of the shared expert, every layer+ * Measured on ds (CPU, 8 threads): of the 37.5 ms a DeltaNet layer stack+ * costs per decoded token, 23.4 are the input projections (already placeable+ * as "dnproj"), 8.3 the out_proj and norm, 3.3 the convolution and 2.4 the+ * recurrence -- the matmuls are the cost, not the recurrence, so this is+ * where the trunk goes. Offer order after lmhead and dnproj: dnout, attnproj,+ * shexp, each in layer order, so a partial placement is whole layers. A+ * component is offered only when every matrix of it has a dense-i8 copy. */+ static void trunk_offer_dense(Model *m){+ Cfg *c = &m->c;+ for (int i = 0; i < c->n_layers; i++) {+ if (c->is_attn[i]) continue;- size_t b = qdw_bytes(m->L[i].dn_out);++ size_t b = qdw_bytes(&m->L[i].dn_out);+ if (b) qt_trunk_offer("dnout", i, b);+ }+ for (int i = 0; i < c->n_layers; i++) {+ if (!c->is_attn[i]) continue;+ Layer *l = &m->L[i];- size_t bq = qdw_bytes(l->q), bk = qdw_bytes(l->k), bv = qdw_bytes(l->v), bo = qdw_bytes(l->o);++ size_t bq = qdw_bytes(&l->q), bk = qdw_bytes(&l->k), bv = qdw_bytes(&l->v), bo = qdw_bytes(&l->o);+ if (bq && bk && bv && bo) qt_trunk_offer("attnproj", i, bq + bk + bv + bo);+ }+ for (int i = 0; i < c->n_layers; i++) {+ Layer *l = &m->L[i];- size_t bg = qdw_bytes(l->sh_g), bu = qdw_bytes(l->sh_u), bd = qdw_bytes(l->sh_d);++ size_t bg = qdw_bytes(&l->sh_g), bu = qdw_bytes(&l->sh_u), bd = qdw_bytes(&l->sh_d);+ if (bg && bu && bd) qt_trunk_offer("shexp", i, bg + bu + bd);+ }+ }+ /* After qt_init decided: upload what was placed, keep the handles in the+ * Layer. Every matrix falls back on its own, so a failed upload costs one+ * GEMV on the CPU, never the component. Returns the number of matrices+ * placed; `vram_bytes` gets their size. */+ static int trunk_place_dense(Model *m, double *vram_bytes){+ Cfg *c = &m->c; int placed = 0; double vram = 0;+ for (int i = 0; i < c->n_layers; i++) {+ Layer *l = &m->L[i];+ if (!c->is_attn[i]) {- l->qth_dnout = qdw_place(l->dn_out, qt_place_of("dnout", i));- if (l->qth_dnout) { placed++; vram += (double)qdw_bytes(l->dn_out); }++ l->qth_dnout = qdw_place(&l->dn_out, qt_place_of("dnout", i));++ if (l->qth_dnout) { placed++; vram += (double)qdw_bytes(&l->dn_out); }+ } else {+ int dev = qt_place_of("attnproj", i);- l->qth_q = qdw_place(l->q, dev); l->qth_k = qdw_place(l->k, dev);- l->qth_v = qdw_place(l->v, dev); l->qth_o = qdw_place(l->o, dev);- if (l->qth_q) { placed++; vram += (double)qdw_bytes(l->q); }- if (l->qth_k) { placed++; vram += (double)qdw_bytes(l->k); }- if (l->qth_v) { placed++; vram += (double)qdw_bytes(l->v); }- if (l->qth_o) { placed++; vram += (double)qdw_bytes(l->o); }++ l->qth_q = qdw_place(&l->q, dev); l->qth_k = qdw_place(&l->k, dev);++ l->qth_v = qdw_place(&l->v, dev); l->qth_o = qdw_place(&l->o, dev);++ if (l->qth_q) { placed++; vram += (double)qdw_bytes(&l->q); }++ if (l->qth_k) { placed++; vram += (double)qdw_bytes(&l->k); }++ if (l->qth_v) { placed++; vram += (double)qdw_bytes(&l->v); }++ if (l->qth_o) { placed++; vram += (double)qdw_bytes(&l->o); }+ }+ int dev = qt_place_of("shexp", i);- l->qth_shg = qdw_place(l->sh_g, dev); l->qth_shu = qdw_place(l->sh_u, dev); l->qth_shd = qdw_place(l->sh_d, dev);- if (l->qth_shg) { placed++; vram += (double)qdw_bytes(l->sh_g); }- if (l->qth_shu) { placed++; vram += (double)qdw_bytes(l->sh_u); }- if (l->qth_shd) { placed++; vram += (double)qdw_bytes(l->sh_d); }++ l->qth_shg = qdw_place(&l->sh_g, dev); l->qth_shu = qdw_place(&l->sh_u, dev); l->qth_shd = qdw_place(&l->sh_d, dev);++ if (l->qth_shg) { placed++; vram += (double)qdw_bytes(&l->sh_g); }++ if (l->qth_shu) { placed++; vram += (double)qdw_bytes(&l->sh_u); }++ if (l->qth_shd) { placed++; vram += (double)qdw_bytes(&l->sh_d); }+ }+ if (vram_bytes) *vram_bytes = vram;+ return placed;+ }++ /* Measured, not assumed. The placer prices a trunk component by the bytes it+ * saves on the CPU's memory bus, which presumes the GPU answers a GEMV faster+ * than the CPU does. Four Tesla M10 (sm_50) said otherwise: every placed+ * component ran slower there, lm_head 68.8 ms against 41.7 on the CPU, and+ * decode fell from 3.56 to 2.68 tok/s (#1652). So before any trunk upload,+ * one DeltaNet input projection (the most numerous placed matrix) is timed+ * both ways on the device that would host it, ten GEMVs each, best of three+ * rounds after a warm-up, and the trunk goes to VRAM only if the GPU wins.+ * Only the automatic placement is questioned: a hand-written COLI_PLACE+ * stands. COLI_TRUNK_PROBE=0 skips the probe and trusts the placer. The+ * probe's copy stays resident (one projection, ~25 MB on the 35B). */+ static int trunk_probe_gpu_wins(Model *m){+ const char *e = getenv("COLI_TRUNK_PROBE");+ if (e && *e == '0') return 1;+ if (!qt_place_is_auto()) return 1;+ Cfg *c = &m->c;- int qi = -1, dev = QT_PLACE_CPU;- for (int i = 0; i < c->n_layers && qi < 0; i++) {++ QW *w = NULL; int dev = QT_PLACE_CPU;++ for (int i = 0; i < c->n_layers && !w; i++) {+ if (c->is_attn[i]) continue;- for (int j = 0; j < g_qdw_n; j++)- if (g_qdw[j].w == m->L[i].dn_qkv) { qi = j; dev = qt_place_of("dnproj", i); break; }++ if (m->L[i].dn_qkv.q) { w = &m->L[i].dn_qkv; dev = qt_place_of("dnproj", i); }+ }- if (qi < 0) return 1; /* dense-i8 off: nothing will be placed */++ if (!w) return 1; /* dense-i8 off: nothing will be placed */+ if (dev == QT_PLACE_CPU) dev = qt_place_of("lmhead", 0);+ if (dev == QT_PLACE_CPU) return 1; /* nothing placed: nothing to measure */- int I = g_qdw[qi].I, O = g_qdw[qi].O;- int h = qt_dense_init(g_qdw[qi].q, g_qdw[qi].sc, I, O, dev);++ int I = w->I, O = w->O;++ int h = qt_dense_init(w->q, w->sc, I, O, dev);+ if (h < 0) return 1; /* cannot measure: the placer's word stands */+ float *x = malloc((size_t)I * sizeof(float)), *y = malloc((size_t)O * sizeof(float));+ if (!x || !y) { free(x); free(y); return 1; }+ for (int i = 0; i < I; i++) x[i] = sinf(0.37f * (float)i);+ double gpu = 1e30, cpu = 1e30;+ for (int r = 0; r < 3; r++) {+ for (int k = 0; k < 3; k++) if (!qt_dense_matmul(h, y, x, I, O)) { free(x); free(y); return 1; }+ double t0 = now_s();+ for (int k = 0; k < 10; k++) if (!qt_dense_matmul(h, y, x, I, O)) { free(x); free(y); return 1; }+ double tg = (now_s() - t0) / 10;- for (int k = 0; k < 3; k++) matmul_q(y, x, g_qdw[qi].q, g_qdw[qi].sc, I, O);++ for (int k = 0; k < 3; k++) matmul_q(y, x, w->q, w->sc, I, O);+ t0 = now_s();- for (int k = 0; k < 10; k++) matmul_q(y, x, g_qdw[qi].q, g_qdw[qi].sc, I, O);++ for (int k = 0; k < 10; k++) matmul_q(y, x, w->q, w->sc, I, O);+ double tc = (now_s() - t0) / 10;+ if (tg < gpu) gpu = tg;+ if (tc < cpu) cpu = tc;+ }+ free(x); free(y);+ int wins = gpu < cpu;+ fprintf(stderr, "[place] probe: one [%d x %d] int8 GEMV takes %.3f ms on CUDA dev %d, %.3f ms on the CPU -> trunk %s\n",+ O, I, gpu * 1e3, dev, cpu * 1e3, wins ? "to VRAM" : "stays on the CPU");+ return wins;+ }+static void layers_forward_range(Model *m, float *x, int S, int pos_base,int layer_begin, int layer_end,int allow_prefetch, FILE *lf) {@@@ -3477,8 -3800,36 +3849,10 @@@ int main(int argc, char **argv)g_expert_gs = m.c.expert_gs;if (g_expert_gs) fprintf(stderr, "[qwen36] group-scaled experts: gs=%d\n", g_expert_gs);fprintf(stderr, "resident weights loaded in %.1fs | RSS after load: %.2f GB\n", m.dense_load_s, rss_gb());- /* dense matrices are quantized to int8 during model_init above (COLI_DENSE_I8=0- * disables it) -- see load_tq/QW; model_init_range already logged the count. */- /* quantize the large dense matrices to int8 (COLI_DENSE_I8=0 disables) */- if (dense_i8_on()) {- double tq = now_s();- Cfg *qc = &m.c; int D2 = qc->hidden;- int q_out = qc->q_heads * qc->q_head_dim, kv_out = qc->kv_heads * qc->k_head_dim;- for (int i = 0; i < qc->n_layers; i++) {- Layer *l = &m.L[i];- qdw_register_as(l->q, D2, q_out, "attn"); qdw_register_as(l->k, D2, kv_out, "attn");- qdw_register_as(l->v, D2, kv_out, "attn"); qdw_register_as(l->o, qc->o_in, D2, "attn");- qdw_register_as(l->gate, D2, qc->n_experts, "router");- qdw_register_as(l->sh_g, D2, qc->shared_inter, "shexp"); qdw_register_as(l->sh_u, D2, qc->shared_inter, "shexp");- qdw_register_as(l->sh_d, qc->shared_inter, D2, "shexp");- qdw_register_as(l->dn_qkv, D2, qc->dn_conv_dim, "dnproj");- qdw_register_as(l->dn_z, D2, qc->dn_vheads * qc->dn_vdim, "dnproj");- qdw_register_as(l->dn_out, qc->dn_vheads * qc->dn_vdim, D2, "dnout");- }- qdw_register_as(m.lm_head, D2, qc->vocab, "lmhead");- /* Free the f32 originals -- the pointers only serve as lookup keys in- * matmul_d from here on (never dereferenced again).- * COLI_KEEP_F32=1 keeps them (debug). */- double freed = 0;- if (!getenv("COLI_KEEP_F32")) {- for (int i = 0; i < g_qdw_n; i++) {- freed += (double)g_qdw[i].I * g_qdw[i].O * sizeof(float);- free((void*)g_qdw[i].w);- }- }- fprintf(stderr, "[dense-i8] %d matrices quantized in %.1f s, %.1f GB f32 freed\n",- g_qdw_n, now_s()-tq, freed/1073741824.0);- }++ /* dense matrices are quantized to int8 (+ int4 planar per COLI_DENSE_BITS/++ * COLI_DENSE_INT4, see dense_int4_wanted) during model_init above++ * (COLI_DENSE_I8=0 disables it) -- see load_tq/QW; model_init_range++ * already logged the count and freed bytes. *//* Optional CUDA VRAM expert tier (COLI_CUDA=1): hot experts live in* DEVICE_LOCAL memory across the configured GPUs, misses fall back to the@@@ -3508,23 -3859,33 +3882,26 @@@* No entry (dense-i8 off) means nothing to offer, and the CPU path stands. */{int O_qkv = m.c.dn_conv_dim, O_z = m.c.dn_vheads * m.c.dn_vdim;- for (int i = 0; i < g_qdw_n; i++)- if (g_qdw[i].w == m.lm_head)- qt_trunk_offer("lmhead", 0, (size_t)g_qdw[i].I * g_qdw[i].O + (size_t)g_qdw[i].O * sizeof(float));+ if (m.lm_head.q)+ qt_trunk_offer("lmhead", 0, (size_t)m.lm_head.I * m.lm_head.O + (size_t)m.lm_head.O * sizeof(float));for (int i = 0; i < m.c.n_layers; i++) {if (m.c.is_attn[i]) continue;- int have = 0;- for (int j = 0; j < g_qdw_n; j++)- if (g_qdw[j].w == m.L[i].dn_qkv || g_qdw[j].w == m.L[i].dn_z) have++;- if (have == 2)+ if (m.L[i].dn_qkv.q && m.L[i].dn_z.q)qt_trunk_offer("dnproj", i, (size_t)(O_qkv + O_z) * m.c.hidden + (size_t)(O_qkv + O_z) * sizeof(float));}+ trunk_offer_dense(&m); /* dnout, attnproj, shexp: the rest of the per-token dense work */}if (qt_init(m.c.n_layers, m.c.n_experts, m.c.hidden, m.c.inter, cap, m.c.topk,m.c.expert_gs, expert_is_int4)) {fprintf(stderr, "[gpu] MoE experts -> CUDA VRAM tier\n");atexit(qt_shutdown);+ /* The placer predicted; measure before uploading a byte of trunk. */+ if (!trunk_probe_gpu_wins(&m)) qt_trunk_withdraw("measured slower than the CPU");/* R4 role split: park the dense-i8 lm_head on COLI_LMHEAD_GPU. The- * qdw entry keyed by m.lm_head holds the int8 rows + per-row scales- * the CPU path uses; the GPU applies the identical semantics. */- for (int i = 0; i < g_qdw_n; i++)- if (g_qdw[i].w == m.lm_head) {- qt_lmhead_init(g_qdw[i].q, g_qdw[i].sc, g_qdw[i].I, g_qdw[i].O);- break;- }+ * QW struct on m.lm_head holds the int8 rows + per-row scales the+ * CPU path uses; the GPU applies the identical semantics. */+ if (m.lm_head.q)+ qt_lmhead_init(m.lm_head.q, m.lm_head.sc, m.lm_head.I, m.lm_head.O);/* R4 step 2: DeltaNet input projections, per layer, wherever* COLI_PLACE puts them. qkv and z are both [O_x, hidden] int8 with* per-row scales, so fusing them is a concatenation along O -- two@@@ -4108,10 -4481,9 +4494,10 @@@ static int qwen36_edge_engine_openengine->model.embed = load_t_n(&engine->model, "model.embed_tokens.weight",(int64_t)config->vocab * config->hidden);- engine->model.lm_head = load_t_n(- &engine->model, "lm_head.weight",- (int64_t)config->vocab * config->hidden);+ /* quantize=0: this engine never ran the old post-hoc qdw_register pass+ * either (only main()'s static Model did), so lm_head stays f32-only here,+ * exactly as before. */- load_tq(&engine->model, "lm_head.weight", config->hidden, config->vocab, 0, &engine->model.lm_head);++ load_tq(&engine->model, "lm_head.weight", config->hidden, config->vocab, 0, "lmhead", &engine->model.lm_head);engine->model.final_norm = load_t_n(&engine->model, "model.norm.weight", config->hidden);engine->model.quant_bits = container_layer_is_int4(&engine->model, 0) ? 4 : 8;diff --cc c/tests/bench_qwen36_dense_batch.cindex 7e29d5b3,63f0fc22..36e19e55--- a/c/tests/bench_qwen36_dense_batch.c+++ b/c/tests/bench_qwen36_dense_batch.c@@@ -39,13 -39,13 +39,13 @@@ static double shared_run(Model *m,Layestatic int shared_benchmark(void) {Model m;memset(&m,0,sizeof(m));m.c.hidden=I;m.c.shared_inter=O;Layer l;memset(&l,0,sizeof(l));- l.sh_g=falloc((int64_t)O*I);l.sh_u=falloc((int64_t)O*I);- l.sh_d=falloc((int64_t)I*O);l.sh_gate=falloc(I);- for(int64_t i=0;i<(int64_t)O*I;i++){l.sh_g[i]=value(i,2);l.sh_u[i]=value(i,3);}- for(int64_t i=0;i<(int64_t)I*O;i++)l.sh_d[i]=value(i,4);+ l.sh_g.w=falloc((int64_t)O*I);l.sh_u.w=falloc((int64_t)O*I);+ l.sh_d.w=falloc((int64_t)I*O);l.sh_gate=falloc(I);+ for(int64_t i=0;i<(int64_t)O*I;i++){((float*)l.sh_g.w)[i]=value(i,2);((float*)l.sh_u.w)[i]=value(i,3);}+ for(int64_t i=0;i<(int64_t)I*O;i++)((float*)l.sh_d.w)[i]=value(i,4);for(int i=0;i<I;i++)l.sh_gate[i]=value(i,5);- qw_quantize(l.sh_g.w,I,O,&l.sh_g);qw_quantize(l.sh_u.w,I,O,&l.sh_u);qw_quantize(l.sh_d.w,O,I,&l.sh_d);- qdw_register(l.sh_g,I,O);qdw_register(l.sh_u,I,O);qdw_register(l.sh_d,O,I);- if(g_qdw_n!=3){fprintf(stderr,"FAIL: expected three dense-int8 copies, got %d\n",g_qdw_n);return 1;}++ qw_quantize(l.sh_g.w,I,O,NULL,&l.sh_g);qw_quantize(l.sh_u.w,I,O,NULL,&l.sh_u);qw_quantize(l.sh_d.w,O,I,NULL,&l.sh_d);+ if(!l.sh_g.q||!l.sh_u.q||!l.sh_d.q){fprintf(stderr,"FAIL: expected three dense-int8 copies\n");return 1;}float *x=falloc((int64_t)S*I),*seed=falloc((int64_t)S*I);float *a=falloc((int64_t)S*I),*b=falloc((int64_t)S*I);for(int64_t i=0;i<(int64_t)S*I;i++){x[i]=value(i,6);seed[i]=value(i,7);}diff --cc c/tests/test_qwen36_dense_batch.cindex 8f7807d6,69288725..d513e814--- a/c/tests/test_qwen36_dense_batch.c+++ b/c/tests/test_qwen36_dense_batch.c@@@ -69,13 -57,12 +69,13 @@@ static void shared_case(const char *forenum { S=12,D=64,I=32 };Model m;memset(&m,0,sizeof(m));m.c.hidden=D;m.c.shared_inter=I;Layer l;memset(&l,0,sizeof(l));- l.sh_g=falloc((int64_t)I*D);l.sh_u=falloc((int64_t)I*D);- l.sh_d=falloc((int64_t)D*I);l.sh_gate=falloc(D);- for(int64_t i=0;i<(int64_t)I*D;i++){l.sh_g[i]=input_value(i,2);l.sh_u[i]=input_value(i,3);}- for(int64_t i=0;i<(int64_t)D*I;i++)l.sh_d[i]=input_value(i,4);+ l.sh_g.w=falloc((int64_t)I*D);l.sh_u.w=falloc((int64_t)I*D);+ l.sh_d.w=falloc((int64_t)D*I);l.sh_gate=falloc(D);+ l.sh_g.I=D;l.sh_g.O=I; l.sh_u.I=D;l.sh_u.O=I; l.sh_d.I=I;l.sh_d.O=D;+ for(int64_t i=0;i<(int64_t)I*D;i++){((float*)l.sh_g.w)[i]=input_value(i,2);((float*)l.sh_u.w)[i]=input_value(i,3);}+ for(int64_t i=0;i<(int64_t)D*I;i++)((float*)l.sh_d.w)[i]=input_value(i,4);for(int i=0;i<D;i++)l.sh_gate[i]=input_value(i,5);- if(quantized){qw_quantize(l.sh_g.w,D,I,&l.sh_g);qw_quantize(l.sh_u.w,D,I,&l.sh_u);qw_quantize(l.sh_d.w,I,D,&l.sh_d);}- if(quantized){qdw_register(l.sh_g,D,I);qdw_register(l.sh_u,D,I);qdw_register(l.sh_d,I,D);}++ if(quantized){qw_quantize(l.sh_g.w,D,I,NULL,&l.sh_g);qw_quantize(l.sh_u.w,D,I,NULL,&l.sh_u);qw_quantize(l.sh_d.w,I,D,NULL,&l.sh_d);}float *x=falloc((int64_t)S*D),*seed=falloc((int64_t)S*D);float *ref=falloc((int64_t)S*D),*got=falloc((int64_t)S*D);float *g=falloc(I),*u=falloc(I),*hh=falloc(D);diff --cc c/tests/test_qwen36_dense_idot.cindex 00000000,c9c2ef47..c506c81amode 000000,100644..100644--- a/c/tests/test_qwen36_dense_idot.c+++ b/c/tests/test_qwen36_dense_idot.c@@@ -1,0 -1,127 +1,128 @@@+ /* The dense trunk's integer path (COLI_DENSE_IDOT, COLI_DENSE_BITS=4).+ *+ * Pins, with no model:+ * - dense_act_i8's vector path equals the scalar lrintf contract bit for bit+ * (same rounding, same scale) and its block sums are exact;+ * - pack_int4_g64_planar stores what the K1b layout expects: nibble v+8,+ * lo nibbles = elements 0..31 of the block, hi = 32..63, one scale per 64;+ * - matmul_d through the int4 grouped kernel and through the int8 integer+ * kernel both agree with the f32 reference within what an int8 activation+ * and an int4 weight allow, on every row of a random matrix, for one row+ * (decode) and for several rows (prefill);+ * - with neither flag the dispatch is the one before: the int8 f32-activation+ * kernel, byte-identical to its own output. */+ #include <stdio.h>+ #include <stdlib.h>+ #include <string.h>+ #include <math.h>++ #define main qwen36_main_unused+ #include "../qwen36.c"+ #undef main+ #include "../compat.h" /* setenv: MinGW has none */++ static int fails;+ static void ck(int ok, const char *what) { if (ok) { printf(" ok %s\n", what); return; } printf(" FAIL %s\n", what); fails++; }++ static unsigned g_seed = 4242;+ static float rnd(void) { g_seed = g_seed * 1103515245u + 12345u; return ((g_seed >> 8) & 0xFFFF) / 32768.f - 1.f; }++ static void ref_matmul(float *y, const float *x, const float *W, int S, int I, int O) {+ for (int s = 0; s < S; s++) for (int o = 0; o < O; o++) {+ double a = 0; for (int i = 0; i < I; i++) a += (double)x[(size_t)s * I + i] * W[(size_t)o * I + i];+ y[(size_t)s * O + o] = (float)a;+ }+ }+ static double rel_gap(const float *a, const float *b, int n) {+ double worst = 0, scale = 1e-6;+ for (int i = 0; i < n; i++) { double d = fabs((double)a[i] - b[i]); if (d > worst) worst = d; if (fabs((double)b[i]) > scale) scale = fabs((double)b[i]); }+ return worst / scale;+ }++ int main(void) {+ enum { I = 256, O = 96, S = 5 };+ printf("activation quantizer\n");+ {+ float x[I]; int8_t a[I], b[I]; int32_t sums[I / 64];+ for (int t = 0; t < 50; t++) {+ float mag = (t % 5 == 0) ? 1e-3f : 3.f;+ for (int i = 0; i < I; i++) x[i] = rnd() * mag;+ float sa = dense_act_i8(x, I, a, sums);+ float sb = qrow_i8(x, b, I);+ if (sa != sb || memcmp(a, b, I)) { ck(0, "vector quantizer equals the scalar contract"); break; }+ for (int g = 0; g < I / 64; g++) { int32_t s = 0; for (int k = 0; k < 64; k++) s += a[g * 64 + k]; if (s != sums[g]) { ck(0, "block sums are exact"); break; } }+ if (t == 49) ck(1, "vector quantizer equals the scalar lrintf contract on 50 random rows, block sums exact");+ }+ }+ printf("int4 planar packing\n");+ {+ float *W = malloc((size_t)O * I * sizeof(float));+ for (size_t i = 0; i < (size_t)O * I; i++) W[i] = rnd();+ uint8_t *q4 = malloc((size_t)O * I / 2); float *sg = malloc((size_t)O * (I / 64) * sizeof(float));+ pack_int4_g64_planar(W, q4, sg, O, I);+ int bad = 0;+ for (int o = 0; o < O && !bad; o++) for (int g = 0; g < I / 64 && !bad; g++) {+ const uint8_t *blk = q4 + (size_t)o * (I / 2) + g * 32;+ float s = sg[(size_t)o * (I / 64) + g];+ for (int k = 0; k < 64; k++) {+ int nib = k < 32 ? (blk[k] & 0xF) : (blk[k - 32] >> 4);+ float w = W[(size_t)o * I + g * 64 + k];+ int want = (int)lrintf(w / s); if (want > 7) want = 7; if (want < -8) want = -8;+ if (nib - 8 != want) { bad = 1; break; }+ }+ }+ ck(!bad, "nibble v+8 in the planar order, one absmax/7 scale per block of 64");+ free(W); free(q4); free(sg);+ }+ printf("matmul_d dispatch\n");+ {+ float *W = malloc((size_t)O * I * sizeof(float)), *x = malloc((size_t)S * I * sizeof(float));+ float *ref = malloc((size_t)S * O * sizeof(float)), *y = malloc((size_t)S * O * sizeof(float)), *y0 = malloc((size_t)S * O * sizeof(float));+ for (size_t i = 0; i < (size_t)O * I; i++) W[i] = rnd();+ for (size_t i = 0; i < (size_t)S * I; i++) x[i] = rnd() * 2.f;+ ref_matmul(ref, x, W, S, I, O);- /* the classic path (no flags): int8 rows, f32 activations */++ /* the classic path (no flags): int8 rows, f32 activations.++ * dense_idot_on caches its answer: this TU reads the env once, so++ * set it before the first call (matmul_d, via qw_quantize below). */+ setenv("COLI_DENSE_IDOT", "0", 1); setenv("COLI_DENSE_BITS", "8", 1);- qdw_register(W, I, O);- matmul_d(y0, x, W, S, I, O);++ QW w = {0}; qw_quantize(W, I, O, NULL, &w);++ matmul_d(y0, x, &w, S, I, O);+ double g0 = rel_gap(y0, ref, S * O);+ ck(g0 < 2e-2, "classic int8 path within 2% of the f32 reference (per-row int8)");- matmul_d(y, x, W, 1, I, O);++ matmul_d(y, x, &w, 1, I, O);+ ck(!memcmp(y, y0, (size_t)O * sizeof(float)), "one row and the first of a batch agree on the classic path");- /* the integer path on the same int8 rows */- g_qdw_n = 0;- /* dense_idot_on caches its answer: this TU reads the env once, so set it before the first call */++ qw_free(&w);++ free(W); free(x); free(ref); free(y); free(y0);+ }+ {+ /* fresh process-wide state for the flag readers: emulate by direct calls */+ float *W = malloc((size_t)O * I * sizeof(float)), *x = malloc((size_t)S * I * sizeof(float));+ float *ref = malloc((size_t)S * O * sizeof(float)), *y = malloc((size_t)S * O * sizeof(float));+ for (size_t i = 0; i < (size_t)O * I; i++) W[i] = rnd();+ for (size_t i = 0; i < (size_t)S * I; i++) x[i] = rnd() * 2.f;+ ref_matmul(ref, x, W, S, I, O);+ int8_t *q = malloc((size_t)O * I); float *sc = malloc((size_t)O * sizeof(float));+ for (int o = 0; o < O; o++) { /* the per-row int8 the engine builds at load */+ float am = 0.f; for (int i = 0; i < I; i++) { float a = fabsf(W[(size_t)o * I + i]); if (a > am) am = a; }+ float s = am > 1e-12f ? am / 127.f : 1.f; sc[o] = s;+ for (int i = 0; i < I; i++) { int v = (int)lrintf(W[(size_t)o * I + i] / s); if (v > 127) v = 127; if (v < -127) v = -127; q[(size_t)o * I + i] = (int8_t)v; }+ }+ int8_t *xq = malloc((size_t)S * I); float sx[S]; int32_t *xsg = malloc((size_t)S * (I / 64) * sizeof(int32_t));+ for (int s = 0; s < S; s++) sx[s] = dense_act_i8(x + (size_t)s * I, I, xq + (size_t)s * I, xsg + (size_t)s * (I / 64));+ matmul_q_idot(y, xq, sx, q, sc, S, I, O);+ ck(rel_gap(y, ref, S * O) < 3e-2, "int8 weights x int8 activations within 3% of the f32 reference, 5 rows");+ matmul_q_idot(y, xq, sx, q, sc, 1, I, O);+ ck(rel_gap(y, ref, O) < 3e-2, "same for one row");+ uint8_t *q4 = malloc((size_t)O * I / 2); float *sg = malloc((size_t)O * (I / 64) * sizeof(float));+ pack_int4_g64_planar(W, q4, sg, O, I);+ matmul_i4p_grouped_idot(y, xq, sx, xsg, q4, sg, S, I, O, 64);+ ck(rel_gap(y, ref, S * O) < 8e-2, "int4 blocks of 64 x int8 activations within 8% of the f32 reference, 5 rows");+ float y1[O];+ matmul_i4p_grouped_idot(y1, xq, sx, xsg, q4, sg, 1, I, O, 64);+ ck(!memcmp(y1, y, (size_t)O * sizeof(float)), "the one-row path and the row tile give the same bytes (the K1b contract)");+ free(W); free(x); free(ref); free(y); free(q); free(sc); free(xq); free(xsg); free(q4); free(sg);+ }+ if (fails) { printf("test_qwen36_dense_idot: %d failure(s)\n", fails); return 1; }+ printf("OK test_qwen36_dense_idot: the dense trunk's integer path\n");+ return 0;+ }diff --cc c/tests/test_qwen36_slot_int8.cindex 58d1c2e6,00000000..80d60391mode 100644,000000..100644--- a/c/tests/test_qwen36_slot_int8.c+++ b/c/tests/test_qwen36_slot_int8.c@@@ -1,96 -1,0 +1,103 @@@+/* Regression gate for wiring slot_ensure_int8() onto #1271's unpack_int4_to_int8+ * (the same branchless AVX2/NEON/scalar kernel load_expert_merged already uses+ * for the container's int4 read) instead of its own separate scalar+ * nibble-unpack loop. Integer nibble extraction + sign-extend is bit-exact by+ * construction (no reduction, no rounding), so this compares the NEW+ * slot_ensure_int8 (calling unpack_int4_to_int8) against a copy of the OLD+ * scalar loop it replaced -- not just "it compiles", a real regression check+ * that the two nibble-unpack formulas produce identical bytes. */+#define main qwen36_main_unused+#include "../qwen36.c"+#undef main++static int failures;+#define CHECK(cond, ...) do { if (!(cond)) { \+ fprintf(stderr,"FAIL %s:%d: ",__FILE__,__LINE__); \+ fprintf(stderr,__VA_ARGS__); fputc('\n',stderr); failures++; } } while (0)++/* Verbatim copy of slot_ensure_int8's pre-#1271 scalar loop (the code this+ * change replaces), used here only as the regression oracle. */+static void old_unpack(int8_t *dst, const uint8_t *p, int64_t len) {+ for (int64_t i = 0; i < len; i += 2) {+ uint8_t b = p[i >> 1];+ int8_t lo = (int8_t)(b & 0xF); if (lo & 8) lo -= 16;+ int8_t hi = (int8_t)((b >> 4) & 0xF); if (hi & 8) hi -= 16;+ dst[i] = lo; dst[i + 1] = hi;+ }+}++static uint8_t *random_packed(int64_t nbytes, uint32_t seed) {+ uint8_t *p = malloc((size_t)nbytes);+ uint32_t x = seed ? seed : 1;+ for (int64_t i = 0; i < nbytes; i++) {+ x ^= x << 13; x ^= x >> 17; x ^= x << 5; /* xorshift32 */+ p[i] = (uint8_t)x;+ }+ return p;+}++/* inter/hidden chosen so ng=inter*hidden and nd=hidden*inter land on both+ * sides of the AVX2 16-byte (32-element) vector boundary across the three- * segments (g/u sized ng, d sized nd). */++ * segments (g/u sized ng, d sized nd). ng/nd must stay even: nibble-packing++ * always yields ceil(n/2) bytes, and real containers guarantee even++ * inter*hidden (SIMD/GS64-block-aligned dims), so old_unpack -- a verbatim++ * copy of the pre-#1271 scalar loop, kept unmodified on purpose -- was never++ * written to handle an odd total (its last iteration writes dst[i+1]++ * unconditionally) and shouldn't be, either: that shape cannot occur. */+static void one_case(int inter, int hidden, const char *label) {+ Model m; memset(&m, 0, sizeof(m));+ m.c.inter = inter; m.c.hidden = hidden;+ int64_t ng = (int64_t)inter * hidden, nd = (int64_t)hidden * inter;++ CHECK(ng % 2 == 0, "%s: ng must be even (nibble-packed, see one_case)", label);++ if (ng % 2) return;++ Slot s; memset(&s, 0, sizeof(s));+ s.g4 = random_packed(ng / 2, 1001);+ s.u4 = random_packed(ng / 2, 2002);+ s.d4 = random_packed(nd / 2, 3003);++ slot_ensure_int8(&m, &s);+ CHECK(s.g != NULL, "%s: slot_ensure_int8 did not populate s->g", label);+ if (!s.g) return;++ int8_t *exp_g = malloc((size_t)ng), *exp_u = malloc((size_t)ng), *exp_d = malloc((size_t)nd);+ old_unpack(exp_g, s.g4, ng);+ old_unpack(exp_u, s.u4, ng);+ old_unpack(exp_d, s.d4, nd);++ CHECK(!memcmp(s.g, exp_g, (size_t)ng), "%s: gate segment differs from old scalar loop", label);+ CHECK(!memcmp(s.u, exp_u, (size_t)ng), "%s: up segment differs from old scalar loop", label);+ CHECK(!memcmp(s.d, exp_d, (size_t)nd), "%s: down segment differs from old scalar loop", label);+ printf("qwen36 slot_ensure_int8 exact: %s inter=%d hidden=%d (ng=%lld nd=%lld)\n",+ label, inter, hidden, (long long)ng, (long long)nd);++ free(exp_g); free(exp_u); free(exp_d);+ free(s.g); /* s.u/s.d point into the same block as s.g (slot_ensure_int8's `w`) */+ free(s.g4); free(s.u4); free(s.d4);+}++int main(void) {+ one_case(4, 8, "tiny, well under one vector"); /* ng=32, nd=32 */+ one_case(16, 16, "exact AVX2 vector boundary"); /* ng=256, nd=256, /16=16 (mult of 16-byte block) */+ one_case(17, 16, "vector body + scalar tail"); /* ng=272 -> nb=136, not mult of 16 */- one_case(37, 41, "odd, real-expert-shaped"); /* ng=1517, nd=1517 */++ one_case(37, 42, "odd, real-expert-shaped"); /* ng=1554, nd=1554 (even: see one_case) */+ one_case(2048, 512, "large, real Qwen3.6 expert scale"); /* ng=1048576, nd=1048576 */++ /* re-entrancy guard: slot_ensure_int8 must no-op when s->g is already set+ * (the "rematerialize only if evicted" contract) -- unrelated to which+ * unpack kernel runs, but worth pinning since this function was touched */+ {+ Model m; memset(&m, 0, sizeof(m)); m.c.inter = 4; m.c.hidden = 8;+ Slot s; memset(&s, 0, sizeof(s));+ s.g4 = random_packed(16, 42); s.u4 = random_packed(16, 43); s.d4 = random_packed(16, 44);+ slot_ensure_int8(&m, &s);+ int8_t *g_before = s.g;+ slot_ensure_int8(&m, &s); /* should be a no-op: s->g already non-NULL */+ CHECK(s.g == g_before, "slot_ensure_int8 re-ran when s->g was already set");+ free(s.g); free(s.g4); free(s.u4); free(s.d4);+ }++ if (failures) { fprintf(stderr, "qwen36 slot_ensure_int8: %d failure(s)\n", failures); return 1; }+ puts("qwen36 slot_ensure_int8: ok");+ return 0;+}diff --cc c/tests/test_qwen36_trunk_dense.cindex 00000000,02a9e1a3..14bb0a41mode 000000,100644..100644--- a/c/tests/test_qwen36_trunk_dense.c+++ b/c/tests/test_qwen36_trunk_dense.c@@@ -1,0 -1,163 +1,172 @@@+ /* The rest of the dense trunk offered to the VRAM placer: "dnout" (DeltaNet+ * out_proj), "attnproj" (q, k, v, o of an attention layer) and "shexp" (the+ * shared expert's gate, up, down), placed by name and layer and served from+ * VRAM through qt_dense handles kept in the Layer.+ *+ * What is pinned, on the fake CUDA backend (no GPU, no toolkit):+ * - trunk_offer_dense offers exactly the components whose every matrix has+ * a dense-i8 copy, with the bytes of those copies;+ * - trunk_place_dense uploads what the placer took and keeps one handle per+ * matrix, counting matrices and bytes;+ * - a placed matrix answers the same GEMV from VRAM as matmul_d does on the+ * CPU (same int8 rows, same per-row scales), and a component without a+ * handle keeps running matmul_d;+ * - the placer prices every one of them as a dense component: with room for+ * all, all go; with COLI_PLACE=off nothing is offered at all.+ * Include order as in tests/test_qwen36_tier_int8_engine.c: the engine, the+ * fake backend, then the tier, so the tier's statics live in this TU. */+ #define main qwen36_main_unused+ #include "../qwen36.c"+ #undef main++ #include "../compat.h" /* setenv/unsetenv: MinGW has neither */++ #include "qwen36_fake_cuda.h"++ #include "../qwen36_tier.c"++ static int fails;+ static void ck(int ok, const char *what) {+ if (ok) { printf(" ok %s\n", what); return; }+ printf(" FAIL %s\n", what);+ fails++;+ }++ /* Small but every dimension distinct, so a swapped I/O would show. */+ enum { NL = 2, D = 48, VH = 2, VD = 8, QH = 2, QD = 12, KVH = 1, KD = 8, SH = 20, NE = 4, IH = 16 };++ static unsigned g_seed = 12345;+ static float rnd(void) {+ g_seed = g_seed * 1103515245u + 12345u;+ return ((g_seed >> 8) & 0xFFFF) / 32768.f - 1.f;+ }+ static float *rnd_matrix(int rows, int cols) {+ float *w = malloc((size_t)rows * cols * sizeof(float));+ for (size_t i = 0; i < (size_t)rows * cols; i++) w[i] = rnd();+ return w;+ }++ /* One GEMV both ways: from VRAM through the handle and on the CPU through+ * matmul_d. The fake backend computes exactly what a real one does with these+ * bytes (x . int8 row, times the row scale), so the two must agree to float+ * accumulation order. */-static double gemv_gap(int hp1, const float *W, int I, int O, int *served) {++static double gemv_gap(int hp1, const QW *w, int I, int O, int *served) {+ float *x = rnd_matrix(1, I), *ya = calloc((size_t)O, sizeof(float)), *yb = calloc((size_t)O, sizeof(float));+ *served = qtd(hp1, ya, x, I, O);- matmul_d(yb, x, W, 1, I, O);++ matmul_d(yb, x, w, 1, I, O);+ double worst = 0, scale = 1e-6;+ for (int o = 0; o < O; o++) {+ double d = fabs((double)ya[o] - yb[o]);+ if (d > worst) worst = d;+ if (fabs((double)yb[o]) > scale) scale = fabs((double)yb[o]);+ }+ free(x); free(ya); free(yb);+ return worst / scale;+ }+++/* Quantize a freshly-built random matrix straight into *out (the QW the++ * Layer holds), then discard the f32 staging buffer -- the same "quantize++ * once, free the f32 copy" shape load_tq uses. */++static void register_qw(int rows, int cols, QW *out) {++ float *w = rnd_matrix(rows, cols);++ qw_quantize(w, cols, rows, NULL, out);++ free(w);++}+++ static void build(Model *m) {+ memset(m, 0, sizeof *m);+ Cfg *c = &m->c;+ c->n_layers = NL; c->hidden = D; c->n_experts = NE; c->inter = IH; c->topk = 1; c->expert_gs = 0;+ c->q_heads = QH; c->q_head_dim = QD; c->kv_heads = KVH; c->k_head_dim = KD; c->head_dim = KD; c->o_in = QH * KD;+ c->dn_vheads = VH; c->dn_vdim = VD; c->shared_inter = SH;+ c->is_attn = calloc(NL, 1); c->is_attn[1] = 1;+ m->L = calloc(NL, sizeof(Layer));+ Layer *dn = &m->L[0], *at = &m->L[1];- dn->dn_out = rnd_matrix(D, VH * VD); qdw_register(dn->dn_out, VH * VD, D);- dn->sh_g = rnd_matrix(SH, D); qdw_register(dn->sh_g, D, SH);- dn->sh_u = rnd_matrix(SH, D); qdw_register(dn->sh_u, D, SH);- dn->sh_d = rnd_matrix(D, SH); qdw_register(dn->sh_d, SH, D);- at->q = rnd_matrix(QH * QD, D); qdw_register(at->q, D, QH * QD);- at->k = rnd_matrix(KVH * KD, D); qdw_register(at->k, D, KVH * KD);- at->v = rnd_matrix(KVH * KD, D); qdw_register(at->v, D, KVH * KD);- at->o = rnd_matrix(D, QH * KD); qdw_register(at->o, QH * KD, D);++ register_qw(D, VH * VD, &dn->dn_out);++ register_qw(SH, D, &dn->sh_g);++ register_qw(SH, D, &dn->sh_u);++ register_qw(D, SH, &dn->sh_d);++ register_qw(QH * QD, D, &at->q);++ register_qw(KVH * KD, D, &at->k);++ register_qw(KVH * KD, D, &at->v);++ register_qw(D, QH * KD, &at->o);+ /* the attention layer's shared expert is INCOMPLETE on purpose: gate and+ * up have a dense-i8 copy, down does not (never registered), so "shexp"+ * for layer 1 must not be offered and its three handles must stay 0 */- at->sh_g = rnd_matrix(SH, D); qdw_register(at->sh_g, D, SH);- at->sh_u = rnd_matrix(SH, D); qdw_register(at->sh_u, D, SH);- at->sh_d = rnd_matrix(D, SH); /* no qdw_register */++ register_qw(SH, D, &at->sh_g);++ register_qw(SH, D, &at->sh_u);++ /* at->sh_d: left zeroed (q == NULL) -- no dense-i8 copy, never registered */+ }++ int main(void) {+ setenv("COLI_CUDA", "1", 1); setenv("COLI_GPUS", "0", 1);+ setenv("QT_NO_WARMSTART", "1", 1); setenv("HEAT_FILE", "", 1);+ setenv("COLI_PLACE", "", 1); /* "" == unset == auto */+ setenv("CUDA_EXPERT_GB", "1", 1); /* room for everything */+ unsetenv("COLI_DENSE_I8");+ /* the GEMV from VRAM is compared with the CPU's f32-activation kernel, the+ * contract the tier uploads: the integer dense path rounds the activation+ * and is measured elsewhere (test_qwen36_dense_idot) */+ setenv("COLI_DENSE_IDOT", "0", 1);+ fake_ndev = 1; fake_uploads = 0; fake_dense_compute = 1;++ Model m; build(&m);+ Layer *dn = &m.L[0], *at = &m.L[1];++ printf("offers\n");+ int before = G_offer_n;+ trunk_offer_dense(&m);+ /* dnout(layer 0) + attnproj(layer 1) + shexp(layer 0); NOT shexp(layer 1) */+ ck(G_offer_n - before == 3, "three components offered: dnout, attnproj, shexp of the complete layer only");- size_t want_attn = qdw_bytes(at->q) + qdw_bytes(at->k) + qdw_bytes(at->v) + qdw_bytes(at->o);++ size_t want_attn = qdw_bytes(&at->q) + qdw_bytes(&at->k) + qdw_bytes(&at->v) + qdw_bytes(&at->o);+ int seen_attn = 0, seen_dnout = 0, seen_shexp1 = 0;+ for (int o = before; o < G_offer_n; o++) {+ if (!strcmp(G_offer[o].name, "attnproj") && G_offer[o].layer == 1 && G_offer[o].bytes == want_attn) seen_attn = 1;- if (!strcmp(G_offer[o].name, "dnout") && G_offer[o].layer == 0 && G_offer[o].bytes == qdw_bytes(dn->dn_out)) seen_dnout = 1;++ if (!strcmp(G_offer[o].name, "dnout") && G_offer[o].layer == 0 && G_offer[o].bytes == qdw_bytes(&dn->dn_out)) seen_dnout = 1;+ if (!strcmp(G_offer[o].name, "shexp") && G_offer[o].layer == 1) seen_shexp1 = 1;+ }+ ck(seen_dnout, "dnout offered for the DeltaNet layer with the bytes of its dense-i8 copy");+ ck(seen_attn, "attnproj offered for the attention layer as the sum of q, k, v, o");+ ck(!seen_shexp1, "a shared expert missing one dense-i8 copy is not offered");- ck(qdw_bytes(at->sh_d) == 0, "a matrix without a dense-i8 copy reports zero bytes");++ ck(qdw_bytes(&at->sh_d) == 0, "a matrix without a dense-i8 copy reports zero bytes");++ printf("placement\n");+ ck(qt_init(NL, NE, D, IH, NE, 1, 0, 1), "tier starts (int4 mode, cap == n_experts)");+ ck(qt_place_of("dnout", 0) == 0 && qt_place_of("attnproj", 1) == 0 && qt_place_of("shexp", 0) == 0,+ "every offered component placed on the one device with room");+ ck(qt_place_of("shexp", 1) == QT_PLACE_CPU, "the component never offered stays on the CPU");+ double vram = 0;+ int placed = trunk_place_dense(&m, &vram);+ ck(placed == 8, "eight matrices placed: dnout, q k v o, and the complete shared expert");- ck(vram == (double)(qdw_bytes(dn->dn_out) + want_attn + qdw_bytes(dn->sh_g) + qdw_bytes(dn->sh_u) + qdw_bytes(dn->sh_d)),++ ck(vram == (double)(qdw_bytes(&dn->dn_out) + want_attn + qdw_bytes(&dn->sh_g) + qdw_bytes(&dn->sh_u) + qdw_bytes(&dn->sh_d)),+ "placed bytes are the sum of the dense-i8 copies uploaded");+ ck(fake_uploads == 8, "one upload per placed matrix");+ ck(dn->qth_dnout > 0 && at->qth_q > 0 && at->qth_k > 0 && at->qth_v > 0 && at->qth_o > 0, "handles kept in the Layer");+ ck(dn->qth_shg > 0 && dn->qth_shu > 0 && dn->qth_shd > 0, "shared expert handles kept in the DeltaNet layer");+ ck(at->qth_shg == 0 && at->qth_shu == 0 && at->qth_shd == 0, "no handle for the shared expert that was not offered");++ printf("GEMV from VRAM == matmul_d on the CPU\n");- struct { const char *name; int hp1; const float *w; int I, O; } mats[] = {- {"dn_out", dn->qth_dnout, dn->dn_out, VH * VD, D},- {"q", at->qth_q, at->q, D, QH * QD}, {"k", at->qth_k, at->k, D, KVH * KD},- {"v", at->qth_v, at->v, D, KVH * KD}, {"o", at->qth_o, at->o, QH * KD, D},- {"sh_g", dn->qth_shg, dn->sh_g, D, SH}, {"sh_u", dn->qth_shu, dn->sh_u, D, SH}, {"sh_d", dn->qth_shd, dn->sh_d, SH, D},++ struct { const char *name; int hp1; const QW *w; int I, O; } mats[] = {++ {"dn_out", dn->qth_dnout, &dn->dn_out, VH * VD, D},++ {"q", at->qth_q, &at->q, D, QH * QD}, {"k", at->qth_k, &at->k, D, KVH * KD},++ {"v", at->qth_v, &at->v, D, KVH * KD}, {"o", at->qth_o, &at->o, QH * KD, D},++ {"sh_g", dn->qth_shg, &dn->sh_g, D, SH}, {"sh_u", dn->qth_shu, &dn->sh_u, D, SH}, {"sh_d", dn->qth_shd, &dn->sh_d, SH, D},+ };+ for (size_t i = 0; i < sizeof mats / sizeof mats[0]; i++) {+ int served = 0;+ double gap = gemv_gap(mats[i].hp1, mats[i].w, mats[i].I, mats[i].O, &served);+ char what[128];+ snprintf(what, sizeof what, "%s: served from VRAM, relative gap %.2e", mats[i].name, gap);+ ck(served && gap < 1e-4, what);+ }+ {+ int served = 1; float x[D] = {0}, y[SH] = {0};+ served = qtd(at->qth_shd, y, x, SH, D);+ ck(!served, "a matrix with no handle is not served (the caller runs matmul_d)");+ }++ qt_shutdown();+ if (fails) { printf("test_qwen36_trunk_dense: %d failure(s)\n", fails); return 1; }+ printf("OK test_qwen36_trunk_dense: dnout, attnproj and shexp offered, placed and served from VRAM\n");+ return 0;+ }