Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Merge branch 'dev' into sse41-tier-upstream-next-v2 (+25 commits)

Jaapjan Tinbergendd1cf4e1678d
diff --cc c/Makefile
index b3bd50f6,55a8bfeb..dd78531c
--- a/c/Makefile
+++ b/c/Makefile
@@@ -1197,11 -1200,14 +1200,19 @@@ tests/test_qwen36_tok_truncated$(EXE):
tests/test_qwen36_encode_oom$(EXE): tests/test_qwen36_encode_oom.c qwen36.c expert_ffn.h qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
$(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)
+# slot_ensure_int8's wiring onto #1271's unpack_int4_to_int8: regression check
+# that its output matches the old separate scalar nibble-unpack loop it replaced.
+tests/test_qwen36_slot_int8$(EXE): tests/test_qwen36_slot_int8.c qwen36.c qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
+ $(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)
+
+ # the dense trunk's integer path: quantizer contract, int4 planar packing, dispatch.
+ tests/test_qwen36_dense_idot$(EXE): tests/test_qwen36_dense_idot.c qwen36.c expert_ffn.h qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
+ $(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)
+
+ # #1653: added tokens are split out before the regex pre-tokenizer, HF-style.
+ tests/test_qwen36_tokenizer$(EXE): tests/test_qwen36_tokenizer.c qwen36.c expert_ffn.h qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
+ $(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)
+
# Reproducible local timing evidence; intentionally not a noisy CI perf gate.
tests/bench_qwen36_dense_batch$(EXE): tests/bench_qwen36_dense_batch.c qwen36.c qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
$(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)
diff --cc c/qwen36.c
index aceb5331,49ced244..86e20043
--- a/c/qwen36.c
+++ b/c/qwen36.c
@@@ -629,28 -662,21 +662,40 @@@ typedef struct
int expert_gs; /* expert scale group size along input dim; 0 = per-row */
} Cfg;
+/* ---- Dense int8: a dense matrix that is quantized to int8 during load
+ * (load_tq, below matmul_d) instead of loaded as f32 and quantized in a
+ * separate pass afterward -- so the f32 staging buffer for THIS matrix alone
+ * is what's briefly resident, not every dense matrix in the model at once.
+ * `w` is the f32 copy: kept (and `q`/`sc` left NULL) when COLI_DENSE_I8=0,
+ * the reference/parity path; freed once `q`/`sc` are populated otherwise
+ * (COLI_KEEP_F32=1 keeps it alongside them, for debugging). matmul_d
+ * dispatches on q!=NULL directly -- no pointer-keyed scan. */
- typedef struct { const float *w; int8_t *q; float *sc; int I, O; } QW;
- static void qw_free(QW *w) { free((void*)w->w); free(w->q); free(w->sc); w->w = NULL; w->q = NULL; w->sc = NULL; }
++/* q/sc: int8 rows with one scale per row (the classic copy, what the VRAM
++ * tier uploads). q4/sg: the same matrix as int4 planar blocks of 64 with one
++ * scale per group (COLI_DENSE_BITS=4), the layout the K1b grouped kernel
++ * reads; ng = I/64 groups per row. */
++typedef struct { const float *w; int8_t *q; float *sc; int I, O; uint8_t *q4; float *sg; int ng; } QW;
++static void qw_free(QW *w) {
++ free((void*)w->w); free(w->q); free(w->sc); free(w->q4); free(w->sg);
++ w->w = NULL; w->q = NULL; w->sc = NULL; w->q4 = NULL; w->sg = NULL; w->ng = 0;
++}
+
/* ---------- per-layer dense weights ---------- */
typedef struct {
- float *in_ln, *post_ln, *q, *k, *v, *o, *qn, *kn, *gate, *gate_bias;
- float *sh_g, *sh_u, *sh_d, *sh_gate; /* shared expert (dense f32) + shared_expert_gate */
+ float *in_ln, *post_ln, *qn, *kn, *gate_bias;
+ QW q, k, v, o, gate;
+ QW sh_g, sh_u, sh_d; float *sh_gate; /* shared expert (dense, int8-during-load) + shared_expert_gate */
/* Gated DeltaNet (linear_attention) dense weights (f16->f32 via st_read_f32). */
- float *dn_qkv, *dn_z, *dn_b, *dn_a; /* in_proj_qkv/z/b/a */
+ QW dn_qkv, dn_z; float *dn_b, *dn_a; /* in_proj_qkv/z (int8-during-load), b/a (not dense-matmul'd) */
float *dn_conv; /* conv1d.weight [conv_dim, convk] (groups=conv_dim) */
float *dn_dtbias, *dn_alog; /* dt_bias[vh], A_log[vh] */
float *dn_norm; /* RMSNormGated weight [vdim] */
- float *dn_out; /* out_proj [hidden, value_dim] */
+ QW dn_out; /* out_proj [hidden, value_dim] */
+ /* VRAM copies the tier placed (qt_dense handle + 1, 0 = stays on the CPU):
+ * the DeltaNet out_proj, the attention q/k/v/o and the shared expert's
+ * three matrices. Offered per layer as "dnout", "attnproj", "shexp";
+ * see trunk_offer_dense / trunk_place_dense. */
+ int qth_dnout, qth_q, qth_k, qth_v, qth_o, qth_shg, qth_shu, qth_shd;
} Layer;
/* ---------- LRU expert cache (int8 weights + per-row float scales) ---------- */
@@@ -1010,9 -1041,134 +1061,128 @@@ static void matmul_qe(float *y, const f
}
/* ---- Dense int8: per-row quantized copies of the large f32 matrices.
- * matmul_d dispatches via pointer lookup to matmul_q; COLI_DENSE_I8=0 falls
- * back to f32 (reference path for parity tests). ~4x less memory traffic. */
-#define QDW_MAX 1024
-/* q/sc: int8 rows with one scale per row (the classic copy, what the VRAM
- * tier uploads). q4/sg: the same matrix as int4 planar blocks of 64 with one
- * scale per group (COLI_DENSE_BITS=4), the layout the K1b grouped kernel
- * reads; ng = I/64 groups per row. */
-static struct { const float *w; int8_t *q; float *sc; int I, O; uint8_t *q4; float *sg; int ng; } g_qdw[QDW_MAX];
-static int g_qdw_n = 0;
+ * matmul_d dispatches directly off QW.q (no pointer-keyed scan -- see QW,
+ * above Layer); COLI_DENSE_I8=0 falls back to f32 (QW.w, reference path for
+ * parity tests). ~4x less memory traffic. */
+ /* ---- Dense trunk, integer dot products ------------------------------------
+ *
+ * Every dense GEMV of a token (DeltaNet projections and out_proj, attention
+ * q/k/v/o, the shared expert, lm_head) used to multiply int8 weights by f32
+ * activations: each weight byte converted to f32 and fed to an FMA, eight
+ * weights per instruction. Measured on lm_head (248320 x 2048 int8, 508 MB)
+ * that runs at 29 GB/s on a 16-core AVX-512 host whose memory bus does 80:
+ * the kernel, not the bus, was the limit, and the dense part of a token is
+ * 1.9 GB of int8 on the 35B, three times the routed experts.
+ *
+ * The activation is now quantized to int8 once per call (one scale,
+ * amax/127, the qrow_i8 contract the expert IDOT already uses) and the dot
+ * is integer: 32 weights per instruction on AVX2 (maddubs), 64 on AVX-512
+ * VNNI, exact int32 sums scaled once per output. Not bit-identical to the
+ * f32 path (the activation is rounded): measured on the 35B, +1.0%
+ * perplexity on 4 x 512 tokens, lm_head 12.6 to 10.2 ms/token, decode
+ * 6.71 to 7.35 tok/s alone and 8.23 with the experts' int8 activations.
+ * COLI_DENSE_IDOT=0 restores the f32-activation kernel.
+ *
+ * COLI_DENSE_BITS=4 additionally stores the dense matrices as int4 in blocks
+ * of 64 with one scale per block, the K1b planar layout, halving the bytes
+ * the token reads; lm_head alone goes from 508 to 254 MB. It implies the
+ * integer dot (that layout has no f32 kernel). Same gate: measured. */
+ static int dense_idot_on(void){ static int v=-1; if(v<0){ const char *e=getenv("COLI_DENSE_IDOT"); v=!(e&&*e=='0'); } return v; }
+ static int dense_bits(void){ static int v=-1; if(v<0){ const char *e=getenv("COLI_DENSE_BITS"); v=(e&&atoi(e)==4)?4:8; } return v; }
+
+ /* Activation -> int8 with one scale, plus the int32 sum of every block of 64
+ * (the K1b kernel subtracts 8*sum per group because its nibbles are unsigned).
+ * Vectorized: the scalar lrintf loop was measured at ~7 us for 4096 values,
+ * which times ~150 GEMVs per token is a millisecond thrown away. Rounding is
+ * to nearest even in both paths, so the vector path equals qrow_i8 bit for bit. */
+ static float dense_act_i8(const float *x, int I, int8_t *xq, int32_t *xsg){
+ float amax = 0.f;
+ int i = 0;
+ #ifdef __AVX2__
+ {
+ __m256 am = _mm256_setzero_ps();
+ const __m256 sign = _mm256_set1_ps(-0.0f);
+ for (; i + 8 <= I; i += 8) am = _mm256_max_ps(am, _mm256_andnot_ps(sign, _mm256_loadu_ps(x + i)));
+ float tmp[8]; _mm256_storeu_ps(tmp, am);
+ for (int k = 0; k < 8; k++) if (tmp[k] > amax) amax = tmp[k];
+ }
+ #endif
+ for (; i < I; i++) { float a = fabsf(x[i]); if (a > amax) amax = a; }
+ float s = amax / 127.f; if (s < 1e-12f) s = 1e-12f;
+ float inv = 1.f / s;
+ i = 0;
+ #ifdef __AVX2__
+ {
+ const __m256 vinv = _mm256_set1_ps(inv);
+ for (; i + 32 <= I; i += 32) {
+ __m256i a = _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_loadu_ps(x + i), vinv));
+ __m256i b = _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_loadu_ps(x + i + 8), vinv));
+ __m256i c = _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_loadu_ps(x + i + 16), vinv));
+ __m256i d = _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_loadu_ps(x + i + 24), vinv));
+ /* packs interleave 128-bit lanes: fix the order with one permute */
+ __m256i ab = _mm256_packs_epi32(a, b), cd = _mm256_packs_epi32(c, d);
+ __m256i abcd = _mm256_packs_epi16(ab, cd);
+ abcd = _mm256_permutevar8x32_epi32(abcd, _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7));
+ _mm256_storeu_si256((__m256i *)(xq + i), abcd);
+ }
+ }
+ #endif
+ for (; i < I; i++) xq[i] = (int8_t)lrintf(x[i] * inv);
+ if (xsg) {
+ int ng = I / 64;
+ for (int g = 0; g < ng; g++) {
+ int32_t sum = 0;
+ for (int k = 0; k < 64; k++) sum += xq[g * 64 + k];
+ xsg[g] = sum;
+ }
+ }
+ return s;
+ }
+
+ /* f32 rows -> int4 in blocks of 64 with one f32 scale per block, packed as the
+ * K1b planar layout (unsigned nibbles v+8, block b: lo nibbles = elements
+ * b*64..b*64+31, hi = b*64+32..b*64+63). The quantizer is the symmetric
+ * absmax/7 the expert containers use. */
+ static void pack_int4_g64_planar(const float *w, uint8_t *q4, float *sg, int O, int I){
+ int rb = I / 2, ng = I / 64;
+ #pragma omp parallel for schedule(static)
+ for (int o = 0; o < O; o++) {
+ const float *wr = w + (int64_t)o * I;
+ uint8_t *row = q4 + (int64_t)o * rb;
+ float *sr = sg + (int64_t)o * ng;
+ for (int g = 0; g < ng; g++) {
+ const float *blk = wr + g * 64;
+ float amax = 0.f;
+ for (int k = 0; k < 64; k++) { float a = fabsf(blk[k]); if (a > amax) amax = a; }
+ /* absmax/7 is where the search starts; the scale is then refined
+ * by least squares against the rounded codes (s = w.q / q.q) and
+ * the candidate with the smallest squared error wins. Three
+ * rounds: measured on the 35B this recovers a third of the
+ * perplexity absmax alone loses at 4 bits. */
+ float best_s = amax / 7.f; if (best_s < 1e-8f) best_s = 1e-8f;
+ int best_q[64]; double best_err = 1e30;
+ float s = best_s;
+ for (int round = 0; round < 4; round++) {
+ int q[64]; double err = 0, wq = 0, qq = 0;
+ float inv = 1.f / s;
+ for (int k = 0; k < 64; k++) {
+ int v = (int)lrintf(blk[k] * inv); if (v > 7) v = 7; if (v < -8) v = -8;
+ q[k] = v; double d = (double)blk[k] - (double)v * s; err += d * d;
+ wq += (double)blk[k] * v; qq += (double)v * v;
+ }
+ if (err < best_err) { best_err = err; best_s = s; memcpy(best_q, q, sizeof q); }
+ if (qq <= 0) break;
+ float ns = (float)(wq / qq); /* least-squares scale for these codes */
+ if (ns <= 0.f || ns == s) break;
+ s = ns;
+ }
+ sr[g] = best_s;
+ uint8_t *dst = row + g * 32;
+ for (int k = 0; k < 32; k++)
+ dst[k] = (uint8_t)((best_q[k] + 8) | ((best_q[k + 32] + 8) << 4));
+ }
+ }
+ }
#ifdef COLI_QWEN_BATCH_TEST
static uint64_t g_qwen_matmul_d_calls;
#endif
@@@ -1022,15 -1178,30 +1192,36 @@@ static int dense_i8_on(void){ static in
* the time. */
static int kv_prefix_off(void){ const char *e=getenv("COLI_KV_PREFIX"); return e && *e=='0'; }
static int dense_batch_on(void){ const char *e=getenv("QWEN_DENSE_BATCH"); return !(e&&*e=='0'); }
+ /* COLI_DENSE_INT4 names which components take the int4 copy when
+ * COLI_DENSE_BITS=4: a comma list of lmhead, dnproj, dnout, attn, shexp,
+ * router; unset means all of them. The parts of the trunk pay 4 bits
+ * differently (measured: the attention projections and the DeltaNet input
+ * projections cost the most perplexity, lm_head the least per byte saved),
+ * so the default is chosen per component from the numbers, not for the
+ * whole trunk at once. */
+ static int dense_int4_wanted(const char *tag){
+ if (dense_bits() != 4 || !tag) return 0;
+ const char *e = getenv("COLI_DENSE_INT4");
+ if (!e || !*e) return 1;
+ size_t n = strlen(tag);
+ for (const char *p = e; *p; ) {
+ while (*p == ',' || *p == ' ') p++;
+ const char *q = p; while (*q && *q != ',' && *q != ' ') q++;
+ if ((size_t)(q - p) == n && !strncmp(p, tag, n)) return 1;
+ p = q;
+ }
+ return 0;
+ }
-static void qdw_register_as(const float *W, int I, int O, const char *tag){
- if (!W || !dense_i8_on() || g_qdw_n >= QDW_MAX) return;
+/* Per-row max-abs / round-clamp int8 quantization of an in-memory f32 matrix
- * W [O][I] row-major -- same math load_tq used to run post-hoc via qdw_register,
- * factored out so it can also run on a caller-owned buffer directly (tests
- * that synthesize weights in memory, without a shard file to load from). Does
- * not touch out->w -- the caller sets that (or leaves it, e.g. load_tq frees
- * it right after). */
- static void qw_quantize(const float *W, int I, int O, QW *out) {
++ * W [O][I] row-major -- same math load_tq runs during streamed load, factored
++ * out so it can also run on a caller-owned buffer directly (tests that
++ * synthesize weights in memory, without a shard file to load from). Does not
++ * touch out->w -- the caller sets that (or leaves it, e.g. load_tq frees it
++ * right after). `tag` selects the int4 planar copy per dense_int4_wanted
++ * (NULL/COLI_DENSE_BITS!=4: skipped, out->q4 stays NULL). */
++static void qw_quantize(const float *W, int I, int O, const char *tag, QW *out) {
int8_t *q = malloc((size_t)O*I); float *sc = malloc((size_t)O*sizeof(float));
- if (!q || !sc) { free(q); free(sc); return; }
+ if (!q || !sc) { fprintf(stderr, "OOM qw_quantize\n"); exit(1); }
#pragma omp parallel for schedule(static)
for (int o = 0; o < O; o++) {
const float *r = W + (int64_t)o*I; float am = 0.f;
@@@ -1039,21 -1210,71 +1230,63 @@@
int8_t *d = q + (int64_t)o*I;
for (int i = 0; i < I; i++) { int v = (int)lrintf(r[i]*inv); if (v>127) v=127; if (v<-127) v=-127; d[i] = (int8_t)v; }
}
- g_qdw[g_qdw_n].w=W; g_qdw[g_qdw_n].q=q; g_qdw[g_qdw_n].sc=sc; g_qdw[g_qdw_n].I=I; g_qdw[g_qdw_n].O=O;
- g_qdw[g_qdw_n].q4=NULL; g_qdw[g_qdw_n].sg=NULL; g_qdw[g_qdw_n].ng=0;
- if (dense_int4_wanted(tag) && I%64==0) {
- uint8_t *q4=malloc((size_t)O*(I/2)); float *sg=malloc((size_t)O*(I/64)*sizeof(float));
+ out->q = q; out->sc = sc; out->I = I; out->O = O;
++ out->q4 = NULL; out->sg = NULL; out->ng = 0;
++ if (dense_int4_wanted(tag) && I % 64 == 0) {
++ uint8_t *q4 = malloc((size_t)O*(I/2)); float *sg = malloc((size_t)O*(I/64)*sizeof(float));
+ if (q4 && sg) {
+ pack_int4_g64_planar(W, q4, sg, O, I);
- g_qdw[g_qdw_n].q4=q4; g_qdw[g_qdw_n].sg=sg; g_qdw[g_qdw_n].ng=I/64;
++ out->q4 = q4; out->sg = sg; out->ng = I/64;
+ } else { free(q4); free(sg); }
+ }
- g_qdw_n++;
}
-static void qdw_register(const float *W, int I, int O){ qdw_register_as(W, I, O, NULL); }
-static void matmul_d(float *y, const float *x, const float *W, int S, int I, int O){
+static void matmul_d(float *y, const float *x, const QW *w, int S, int I, int O){
#ifdef COLI_QWEN_BATCH_TEST
g_qwen_matmul_d_calls++;
#endif
- for (int i = 0; i < g_qdw_n; i++) if (g_qdw[i].w == W && g_qdw[i].I == I) {
- if (g_qdw[i].q4 || dense_idot_on()) {
+ if (w->q) {
++ if (w->q4 || dense_idot_on()) {
+ /* integer dot: the activation rows to int8 once, then the K1b
+ * grouped kernel (int4 planar) or the per-row int8 kernel */
+ int ng = I / 64;
+ int8_t *xq = malloc((size_t)S * I);
+ float *sx = malloc((size_t)S * sizeof(float));
- int32_t *xsg = g_qdw[i].q4 ? malloc((size_t)S * ng * sizeof(int32_t)) : NULL;
- if (xq && sx && (!g_qdw[i].q4 || xsg)) {
++ int32_t *xsg = w->q4 ? malloc((size_t)S * ng * sizeof(int32_t)) : NULL;
++ if (xq && sx && (!w->q4 || xsg)) {
+ for (int s = 0; s < S; s++)
+ sx[s] = dense_act_i8(x + (int64_t)s * I, I, xq + (int64_t)s * I, xsg ? xsg + (int64_t)s * ng : NULL);
- if (g_qdw[i].q4) matmul_i4p_grouped_idot(y, xq, sx, xsg, g_qdw[i].q4, g_qdw[i].sg, S, I, O, 64);
- else matmul_q_idot(y, xq, sx, g_qdw[i].q, g_qdw[i].sc, S, I, O);
++ if (w->q4) matmul_i4p_grouped_idot(y, xq, sx, xsg, w->q4, w->sg, S, I, O, 64);
++ else matmul_q_idot(y, xq, sx, w->q, w->sc, S, I, O);
+ free(xq); free(sx); free(xsg);
+ return;
+ }
+ free(xq); free(sx); free(xsg); /* out of memory: the f32 path below */
+ }
if (S > 1 && dense_batch_on())
- matmul_q_batch(y, x, g_qdw[i].q, g_qdw[i].sc, S, I, O);
+ matmul_q_batch(y, x, w->q, w->sc, S, I, O);
else
- for (int s = 0; s < S; s++) matmul_q(y+(int64_t)s*O, x+(int64_t)s*I, g_qdw[i].q, g_qdw[i].sc, I, O);
+ for (int s = 0; s < S; s++) matmul_q(y+(int64_t)s*O, x+(int64_t)s*I, w->q, w->sc, I, O);
return;
}
- matmul(y, x, W, S, I, O);
+ matmul(y, x, w->w, S, I, O);
}
+ /* A dense matrix the tier placed in VRAM (handle+1 kept in the Layer, 0 = CPU):
+ * one GEMV from the device, or 0 and the caller runs matmul_d as before. The
+ * tier turns a failing handle off itself, so the fallback is permanent. */
+ static inline int qtd(int hp1, float *y, const float *x, int I, int O){
+ return hp1 > 0 && qt_dense_matmul(hp1 - 1, y, x, I, O);
+ }
-/* Bytes of W's dense-i8 copy (int8 rows + per-row scales), 0 when there is
++/* Bytes of w's dense-i8 copy (int8 rows + per-row scales), 0 when there is
+ * none (COLI_DENSE_I8=0): nothing to offer, the CPU path stands. */
-static size_t qdw_bytes(const float *W){
- for (int i = 0; i < g_qdw_n; i++)
- if (g_qdw[i].w == W) return (size_t)g_qdw[i].I * g_qdw[i].O + (size_t)g_qdw[i].O * sizeof(float);
- return 0;
++static size_t qdw_bytes(const QW *w){
++ return w->q ? (size_t)w->I * w->O + (size_t)w->O * sizeof(float) : 0;
+ }
-/* Upload W's dense-i8 copy to `dev`; handle+1, or 0 when it stays on the CPU. */
-static int qdw_place(const float *W, int dev){
- if (dev == QT_PLACE_CPU || !W) return 0;
- for (int i = 0; i < g_qdw_n; i++)
- if (g_qdw[i].w == W) {
- int h = qt_dense_init(g_qdw[i].q, g_qdw[i].sc, g_qdw[i].I, g_qdw[i].O, dev);
- return h >= 0 ? h + 1 : 0;
- }
- return 0;
++/* Upload w's dense-i8 copy to `dev`; handle+1, or 0 when it stays on the CPU. */
++static int qdw_place(const QW *w, int dev){
++ if (dev == QT_PLACE_CPU || !w->q) return 0;
++ int h = qt_dense_init(w->q, w->sc, w->I, w->O, dev);
++ return h >= 0 ? h + 1 : 0;
+ }
/* rmsnorm over a row of length D (in-place capable: out may == x).
* Qwen3_5MoeRMSNorm: out = (x * rsqrt(mean(x^2)+eps)) * (1.0 + weight). */
@@@ -1270,24 -1491,6 +1503,27 @@@ static float *load_t_n(Model *m, const
return p;
}
- /* Dense matrix load, quantized to int8 DURING loading rather than in a
- * separate pass over the whole model afterward (see QW, above Layer):
- * reads `name` (I*O elements, same size discipline as load_t_n), and when
- * `quantize` && COLI_DENSE_I8 is on, quantizes it via qw_quantize and frees
- * the f32 staging buffer right away -- so at most one dense matrix's f32 copy
- * is ever resident at a time, not the whole model's. `quantize` is false for
- * loaders that never ran through the old post-hoc qdw_register pass either
- * (the Segment/Edge adapters build partial or auxiliary models straight off
++/* Dense matrix load, quantized to int8 (+ int4 planar per `tag`, see
++ * dense_int4_wanted) DURING loading rather than in a separate pass over the
++ * whole model afterward (see QW, above Layer): reads `name` (I*O elements,
++ * same size discipline as load_t_n), and when `quantize` && COLI_DENSE_I8 is
++ * on, quantizes it via qw_quantize and frees the f32 staging buffer right
++ * away -- so at most one dense matrix's f32 copy is ever resident at a time,
++ * not the whole model's. `quantize` is false for loaders that never ran
++ * through the old post-hoc qdw_register pass either (the Segment/Edge
++ * adapters build partial or auxiliary models straight off
+ * model_init_range/load_t_n, never main()'s dense-i8 block) -- passing it
- * through keeps their f32-only behavior exactly as it was. */
- static void load_tq(Model *m, const char *name, int I, int O, int quantize, QW *out) {
++ * through keeps their f32-only behavior exactly as it was; `tag` is unused
++ * on that path. */
++static void load_tq(Model *m, const char *name, int I, int O, int quantize, const char *tag, QW *out) {
+ float *p = load_t_n(m, name, (int64_t)I * O);
+ out->w = p; out->q = NULL; out->sc = NULL; out->I = I; out->O = O;
++ out->q4 = NULL; out->sg = NULL; out->ng = 0;
+ if (!quantize || !dense_i8_on()) return;
- qw_quantize(p, I, O, out);
++ qw_quantize(p, I, O, tag, out);
+ if (getenv("COLI_KEEP_F32")) out->w = p; else { free(p); out->w = NULL; }
+}
+
static void model_init_range(Model *m, const char *snap, int cap, int bits,
int layer_begin, int layer_end,
int load_boundaries, int allocate_state) {
@@@ -1313,17 -1516,9 +1549,17 @@@
exit(1);
}
double t0 = now_s();
+ /* Quantize during load only for the full-model path (main()'s static Model,
+ * load_boundaries=1): the Segment/Edge adapters build partial or auxiliary
+ * models straight off this same loop and never ran the old post-hoc
+ * qdw_register pass either, so gating on load_boundaries keeps their
+ * f32-only numerics exactly as they were. */
+ int quantize_dense = load_boundaries && dense_i8_on();
+ int qcount = 0; double qfreed = 0;
if (load_boundaries) {
- m->embed = load_t_n(m, "model.embed_tokens.weight", (int64_t)c->vocab * c->hidden);
- m->lm_head = load_t_n(m, "lm_head.weight", (int64_t)c->vocab * c->hidden);
+ m->embed = load_t_n(m, "model.embed_tokens.weight", (int64_t)c->vocab * c->hidden);
- load_tq(m, "lm_head.weight", c->hidden, c->vocab, quantize_dense, &m->lm_head);
++ load_tq(m, "lm_head.weight", c->hidden, c->vocab, quantize_dense, "lmhead", &m->lm_head);
+ if (m->lm_head.q) { qcount++; qfreed += (double)c->hidden * c->vocab * sizeof(float); }
m->final_norm = load_t_n(m, "model.norm.weight", c->hidden);
}
m->L = calloc((size_t)c->n_layers, sizeof(Layer));
@@@ -1342,10 -1535,8 +1578,10 @@@
#define LD(field, suffix, want) snprintf(nm,sizeof(nm),"model.layers.%d." suffix,ai); l->field = load_t_n(m,nm,(want))
LD(in_ln, "input_layernorm.weight", c->hidden);
LD(post_ln,"post_attention_layernorm.weight", c->hidden);
- LD(gate, "mlp.gate.weight", (int64_t)c->n_experts * c->hidden);
#undef LD
+ snprintf(nm,sizeof(nm),"model.layers.%d.mlp.gate.weight", ai);
- load_tq(m, nm, c->hidden, c->n_experts, quantize_dense, &l->gate);
++ load_tq(m, nm, c->hidden, c->n_experts, quantize_dense, "router", &l->gate);
+ QCOUNT(l->gate);
/* q/k norms are per-head [head_dim]; only on attention layers, load if present */
if (c->has_qk_norm) {
snprintf(nm,sizeof(nm),"model.layers.%d.self_attn.q_norm.weight", ai);
@@@ -1357,51 -1548,42 +1593,51 @@@
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.gate.e_score_correction_bias", ai);
if (st_has(&m->S, nm)) { l->gate_bias = falloc(c->n_experts); st_read_f32(&m->S, nm, l->gate_bias, 0); }
else l->gate_bias = NULL;
- /* shared expert (dense f32) */
- #define LD2(field, suffix, want) snprintf(nm,sizeof(nm),"model.layers.%d.mlp.shared_expert." suffix,ai); l->field = load_t_n(m,nm,(want))
- LD2(sh_g, "gate_proj.weight", (int64_t)c->shared_inter * c->hidden);
- LD2(sh_u, "up_proj.weight", (int64_t)c->shared_inter * c->hidden);
- LD2(sh_d, "down_proj.weight", (int64_t)c->hidden * c->shared_inter);
- #undef LD2
+ /* shared expert (dense, int8-during-load) */
+ snprintf(nm,sizeof(nm),"model.layers.%d.mlp.shared_expert.gate_proj.weight", ai);
- load_tq(m, nm, c->hidden, c->shared_inter, quantize_dense, &l->sh_g); QCOUNT(l->sh_g);
++ load_tq(m, nm, c->hidden, c->shared_inter, quantize_dense, "shexp", &l->sh_g); QCOUNT(l->sh_g);
+ snprintf(nm,sizeof(nm),"model.layers.%d.mlp.shared_expert.up_proj.weight", ai);
- load_tq(m, nm, c->hidden, c->shared_inter, quantize_dense, &l->sh_u); QCOUNT(l->sh_u);
++ load_tq(m, nm, c->hidden, c->shared_inter, quantize_dense, "shexp", &l->sh_u); QCOUNT(l->sh_u);
+ snprintf(nm,sizeof(nm),"model.layers.%d.mlp.shared_expert.down_proj.weight", ai);
- load_tq(m, nm, c->shared_inter, c->hidden, quantize_dense, &l->sh_d); QCOUNT(l->sh_d);
++ load_tq(m, nm, c->shared_inter, c->hidden, quantize_dense, "shexp", &l->sh_d); QCOUNT(l->sh_d);
/* shared_expert_gate: Linear(hidden -> 1), sigmoid-gated shared expert */
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.shared_expert_gate.weight", ai);
l->sh_gate = st_has(&m->S, nm) ? load_t_n(m, nm, c->hidden) : NULL;
if (c->is_attn[i]) {
- /* Gated Attention (full_attention) layer */
- #define LD3(field, suffix, want) snprintf(nm,sizeof(nm),"model.layers.%d.self_attn." suffix,ai); l->field = load_t_n(m,nm,(want))
- LD3(q, "q_proj.weight", (int64_t)c->q_heads * c->q_head_dim * c->hidden);
- LD3(k, "k_proj.weight", (int64_t)c->kv_heads * c->k_head_dim * c->hidden);
- LD3(v, "v_proj.weight", (int64_t)c->kv_heads * c->v_head_dim * c->hidden);
- LD3(o, "o_proj.weight", (int64_t)c->hidden * c->o_in);
- #undef LD3
- l->dn_qkv=l->dn_z=l->dn_b=l->dn_a=l->dn_conv=NULL;
- l->dn_dtbias=l->dn_alog=l->dn_norm=l->dn_out=NULL;
+ /* Gated Attention (full_attention) layer, dense projections int8-during-load */
+ snprintf(nm,sizeof(nm),"model.layers.%d.self_attn.q_proj.weight", ai);
- load_tq(m, nm, c->hidden, q_out, quantize_dense, &l->q); QCOUNT(l->q);
++ load_tq(m, nm, c->hidden, q_out, quantize_dense, "attn", &l->q); QCOUNT(l->q);
+ snprintf(nm,sizeof(nm),"model.layers.%d.self_attn.k_proj.weight", ai);
- load_tq(m, nm, c->hidden, kv_out, quantize_dense, &l->k); QCOUNT(l->k);
++ load_tq(m, nm, c->hidden, kv_out, quantize_dense, "attn", &l->k); QCOUNT(l->k);
+ snprintf(nm,sizeof(nm),"model.layers.%d.self_attn.v_proj.weight", ai);
- load_tq(m, nm, c->hidden, kv_out, quantize_dense, &l->v); QCOUNT(l->v);
++ load_tq(m, nm, c->hidden, kv_out, quantize_dense, "attn", &l->v); QCOUNT(l->v);
+ snprintf(nm,sizeof(nm),"model.layers.%d.self_attn.o_proj.weight", ai);
- load_tq(m, nm, c->o_in, c->hidden, quantize_dense, &l->o); QCOUNT(l->o);
++ load_tq(m, nm, c->o_in, c->hidden, quantize_dense, "attn", &l->o); QCOUNT(l->o);
+ l->dn_qkv=l->dn_z=(QW){0}; l->dn_b=l->dn_a=l->dn_conv=NULL;
+ l->dn_dtbias=l->dn_alog=l->dn_norm=NULL; l->dn_out=(QW){0};
} else {
/* Gated DeltaNet (linear_attention) layer */
- l->q=l->k=l->v=l->o=NULL;
+ l->q=l->k=l->v=l->o=(QW){0};
#define LD4(field, suffix, want) snprintf(nm,sizeof(nm),"model.layers.%d.linear_attn." suffix,ai); l->field = load_t_n(m,nm,(want))
int64_t vdim_tot = (int64_t)c->dn_vheads * c->dn_vdim;
- LD4(dn_qkv, "in_proj_qkv.weight", (int64_t)c->dn_conv_dim * c->hidden);
- LD4(dn_z, "in_proj_z.weight", vdim_tot * c->hidden);
+ snprintf(nm,sizeof(nm),"model.layers.%d.linear_attn.in_proj_qkv.weight", ai);
- load_tq(m, nm, c->hidden, c->dn_conv_dim, quantize_dense, &l->dn_qkv); QCOUNT(l->dn_qkv);
++ load_tq(m, nm, c->hidden, c->dn_conv_dim, quantize_dense, "dnproj", &l->dn_qkv); QCOUNT(l->dn_qkv);
+ snprintf(nm,sizeof(nm),"model.layers.%d.linear_attn.in_proj_z.weight", ai);
- load_tq(m, nm, c->hidden, (int)vdim_tot, quantize_dense, &l->dn_z); QCOUNT(l->dn_z);
++ load_tq(m, nm, c->hidden, (int)vdim_tot, quantize_dense, "dnproj", &l->dn_z); QCOUNT(l->dn_z);
LD4(dn_b, "in_proj_b.weight", (int64_t)c->dn_vheads * c->hidden);
LD4(dn_a, "in_proj_a.weight", (int64_t)c->dn_vheads * c->hidden);
LD4(dn_conv,"conv1d.weight", (int64_t)c->dn_conv_dim * c->dn_convk);
LD4(dn_dtbias, "dt_bias", c->dn_vheads);
LD4(dn_alog,"A_log", c->dn_vheads);
LD4(dn_norm, "norm.weight", c->dn_vdim);
- LD4(dn_out, "out_proj.weight", (int64_t)c->hidden * vdim_tot);
#undef LD4
+ snprintf(nm,sizeof(nm),"model.layers.%d.linear_attn.out_proj.weight", ai);
- load_tq(m, nm, (int)vdim_tot, c->hidden, quantize_dense, &l->dn_out); QCOUNT(l->dn_out);
++ load_tq(m, nm, (int)vdim_tot, c->hidden, quantize_dense, "dnout", &l->dn_out); QCOUNT(l->dn_out);
}
}
+ #undef QCOUNT
+ if (quantize_dense)
+ fprintf(stderr, "[dense-i8] %d matrices quantized during load, %.1f GB f32 freed\n", qcount, qfreed/1073741824.0);
m->cache = calloc((size_t)c->n_layers, sizeof(LCache));
for (int i = layer_begin; i < layer_end; i++) {
m->cache[i].cap = cap;
@@@ -1847,9 -2033,11 +2083,11 @@@ static void attention(Model *m, Layer *
float *q = falloc((int64_t)S*q_out);
float *k = falloc((int64_t)S*kv_out);
float *vv= falloc((int64_t)S*kv_out);
- matmul_d(q, x, &l->q, S, D, q_out);
- matmul_d(k, x, &l->k, S, D, kv_out);
- matmul_d(vv, x, &l->v, S, D, kv_out);
+ /* Decode (S == 1): the projections the tier placed answer from VRAM,
+ * one GEMV each; a prompt batch keeps the batched CPU matmul. */
- if (!(S == 1 && qtd(l->qth_q, q, x, D, q_out))) matmul_d(q, x, l->q, S, D, q_out);
- if (!(S == 1 && qtd(l->qth_k, k, x, D, kv_out))) matmul_d(k, x, l->k, S, D, kv_out);
- if (!(S == 1 && qtd(l->qth_v, vv, x, D, kv_out))) matmul_d(vv, x, l->v, S, D, kv_out);
++ if (!(S == 1 && qtd(l->qth_q, q, x, D, q_out))) matmul_d(q, x, &l->q, S, D, q_out);
++ if (!(S == 1 && qtd(l->qth_k, k, x, D, kv_out))) matmul_d(k, x, &l->k, S, D, kv_out);
++ if (!(S == 1 && qtd(l->qth_v, vv, x, D, kv_out))) matmul_d(vv, x, &l->v, S, D, kv_out);
/* split q into query (first hd) and gate (next gate_dim), both per head */
float *query = falloc((int64_t)S*H*hd);
float *gate = falloc((int64_t)S*H*gate_dim);
@@@ -1911,7 -2099,7 +2149,7 @@@
float g = gate_dim ? gate[o] : 0.f;
ag[o] = ctx[o] * (1.f / (1.f + expf(-g)));
}
- matmul_d(out, ag, &l->o, S, H*hd, D);
- if (!(S == 1 && qtd(l->qth_o, out, ag, H*hd, D))) matmul_d(out, ag, l->o, S, H*hd, D);
++ if (!(S == 1 && qtd(l->qth_o, out, ag, H*hd, D))) matmul_d(out, ag, &l->o, S, H*hd, D);
free(q); free(k); free(vv); free(query); free(gate); free(ctx); free(ag);
}
@@@ -1944,10 -2132,10 +2182,10 @@@ static void qwen_shared_experts_cpu(Mod
if (B == 1) {
for (int s=0;s<S;s++) {
const float *xs=x+(int64_t)s*D;
- matmul_d(g,xs,&l->sh_g,1,D,I);
- matmul_d(u,xs,&l->sh_u,1,D,I);
- if(!qtd(l->qth_shg,g,xs,D,I)) matmul_d(g,xs,l->sh_g,1,D,I);
- if(!qtd(l->qth_shu,u,xs,D,I)) matmul_d(u,xs,l->sh_u,1,D,I);
++ if(!qtd(l->qth_shg,g,xs,D,I)) matmul_d(g,xs,&l->sh_g,1,D,I);
++ if(!qtd(l->qth_shu,u,xs,D,I)) matmul_d(u,xs,&l->sh_u,1,D,I);
for(int i=0;i<I;i++){float sv=g[i];g[i]=(sv/(1.f+expf(-sv)))*u[i];}
- matmul_d(hh,g,&l->sh_d,1,I,D);
- if(!qtd(l->qth_shd,hh,g,I,D)) matmul_d(hh,g,l->sh_d,1,I,D);
++ if(!qtd(l->qth_shd,hh,g,I,D)) matmul_d(hh,g,&l->sh_d,1,I,D);
float sgate=1.f;
if(l->sh_gate){float sg=0.f;for(int i=0;i<D;i++)sg+=xs[i]*l->sh_gate[i];sgate=1.f/(1.f+expf(-sg));}
float *os=out+(int64_t)s*D;
@@@ -2229,10 -2417,10 +2467,10 @@@ static void moe(Model *m, Layer *l, in
{
double _ts2 = tm_now();
int Ish = c->shared_inter;
- matmul_d(sh, xs, &l->sh_g, 1, D, Ish);
- matmul_d(shu, xs, &l->sh_u, 1, D, Ish);
- if (!qtd(l->qth_shg, sh, xs, D, Ish)) matmul_d(sh, xs, l->sh_g, 1, D, Ish);
- if (!qtd(l->qth_shu, shu, xs, D, Ish)) matmul_d(shu, xs, l->sh_u, 1, D, Ish);
++ if (!qtd(l->qth_shg, sh, xs, D, Ish)) matmul_d(sh, xs, &l->sh_g, 1, D, Ish);
++ if (!qtd(l->qth_shu, shu, xs, D, Ish)) matmul_d(shu, xs, &l->sh_u, 1, D, Ish);
for (int i = 0; i < Ish; i++) { float sv = sh[i]; sh[i] = (sv / (1.f + expf(-sv))) * shu[i]; }
- matmul_d(shd, sh, &l->sh_d, 1, Ish, D);
- if (!qtd(l->qth_shd, shd, sh, Ish, D)) matmul_d(shd, sh, l->sh_d, 1, Ish, D);
++ if (!qtd(l->qth_shd, shd, sh, Ish, D)) matmul_d(shd, sh, &l->sh_d, 1, Ish, D);
float sgate = 1.f;
if (l->sh_gate) {
float sg = 0.f; const float *wg = l->sh_gate;
@@@ -2421,7 -2609,8 +2659,8 @@@ static void deltanet(Model *m, Layer *l
outr[(int64_t)h * vdim + d] = val * zr[d] / (1.f + expf(-zr[d]));
}
}
- matmul_d(out + (int64_t)s * H, outr, &l->dn_out, 1, value_dim, H);
+ if (!qtd(l->qth_dnout, out + (int64_t)s * H, outr, value_dim, H))
- matmul_d(out + (int64_t)s * H, outr, l->dn_out, 1, value_dim, H);
++ matmul_d(out + (int64_t)s * H, outr, &l->dn_out, 1, value_dim, H);
if (tm_on() && S==1){ g_dn_sub[3]+=tm_now()-_d0; }
if (layer == 0 && s == 0 && getenv("DN_DBG")) {
FILE *dbg = fopen(getenv("DN_DBG"), "wb");
@@@ -2445,6 -2634,118 +2684,117 @@@
free(conv_out); free(q); free(k); free(outv); free(outr); free(kv); free(delta);
}
+ /* The rest of the dense trunk, offered to the placer by name and layer with
+ * the bytes of the dense-i8 copies (docs/qwen36-cuda-tier.md, "Placement"):
+ * dnout -- the DeltaNet out_proj, one matrix per DeltaNet layer
+ * attnproj -- q, k, v, o of every attention layer, offered as one item
+ * shexp -- gate, up, down of the shared expert, every layer
+ * Measured on ds (CPU, 8 threads): of the 37.5 ms a DeltaNet layer stack
+ * costs per decoded token, 23.4 are the input projections (already placeable
+ * as "dnproj"), 8.3 the out_proj and norm, 3.3 the convolution and 2.4 the
+ * recurrence -- the matmuls are the cost, not the recurrence, so this is
+ * where the trunk goes. Offer order after lmhead and dnproj: dnout, attnproj,
+ * shexp, each in layer order, so a partial placement is whole layers. A
+ * component is offered only when every matrix of it has a dense-i8 copy. */
+ static void trunk_offer_dense(Model *m){
+ Cfg *c = &m->c;
+ for (int i = 0; i < c->n_layers; i++) {
+ if (c->is_attn[i]) continue;
- size_t b = qdw_bytes(m->L[i].dn_out);
++ size_t b = qdw_bytes(&m->L[i].dn_out);
+ if (b) qt_trunk_offer("dnout", i, b);
+ }
+ for (int i = 0; i < c->n_layers; i++) {
+ if (!c->is_attn[i]) continue;
+ Layer *l = &m->L[i];
- size_t bq = qdw_bytes(l->q), bk = qdw_bytes(l->k), bv = qdw_bytes(l->v), bo = qdw_bytes(l->o);
++ size_t bq = qdw_bytes(&l->q), bk = qdw_bytes(&l->k), bv = qdw_bytes(&l->v), bo = qdw_bytes(&l->o);
+ if (bq && bk && bv && bo) qt_trunk_offer("attnproj", i, bq + bk + bv + bo);
+ }
+ for (int i = 0; i < c->n_layers; i++) {
+ Layer *l = &m->L[i];
- size_t bg = qdw_bytes(l->sh_g), bu = qdw_bytes(l->sh_u), bd = qdw_bytes(l->sh_d);
++ size_t bg = qdw_bytes(&l->sh_g), bu = qdw_bytes(&l->sh_u), bd = qdw_bytes(&l->sh_d);
+ if (bg && bu && bd) qt_trunk_offer("shexp", i, bg + bu + bd);
+ }
+ }
+ /* After qt_init decided: upload what was placed, keep the handles in the
+ * Layer. Every matrix falls back on its own, so a failed upload costs one
+ * GEMV on the CPU, never the component. Returns the number of matrices
+ * placed; `vram_bytes` gets their size. */
+ static int trunk_place_dense(Model *m, double *vram_bytes){
+ Cfg *c = &m->c; int placed = 0; double vram = 0;
+ for (int i = 0; i < c->n_layers; i++) {
+ Layer *l = &m->L[i];
+ if (!c->is_attn[i]) {
- l->qth_dnout = qdw_place(l->dn_out, qt_place_of("dnout", i));
- if (l->qth_dnout) { placed++; vram += (double)qdw_bytes(l->dn_out); }
++ l->qth_dnout = qdw_place(&l->dn_out, qt_place_of("dnout", i));
++ if (l->qth_dnout) { placed++; vram += (double)qdw_bytes(&l->dn_out); }
+ } else {
+ int dev = qt_place_of("attnproj", i);
- l->qth_q = qdw_place(l->q, dev); l->qth_k = qdw_place(l->k, dev);
- l->qth_v = qdw_place(l->v, dev); l->qth_o = qdw_place(l->o, dev);
- if (l->qth_q) { placed++; vram += (double)qdw_bytes(l->q); }
- if (l->qth_k) { placed++; vram += (double)qdw_bytes(l->k); }
- if (l->qth_v) { placed++; vram += (double)qdw_bytes(l->v); }
- if (l->qth_o) { placed++; vram += (double)qdw_bytes(l->o); }
++ l->qth_q = qdw_place(&l->q, dev); l->qth_k = qdw_place(&l->k, dev);
++ l->qth_v = qdw_place(&l->v, dev); l->qth_o = qdw_place(&l->o, dev);
++ if (l->qth_q) { placed++; vram += (double)qdw_bytes(&l->q); }
++ if (l->qth_k) { placed++; vram += (double)qdw_bytes(&l->k); }
++ if (l->qth_v) { placed++; vram += (double)qdw_bytes(&l->v); }
++ if (l->qth_o) { placed++; vram += (double)qdw_bytes(&l->o); }
+ }
+ int dev = qt_place_of("shexp", i);
- l->qth_shg = qdw_place(l->sh_g, dev); l->qth_shu = qdw_place(l->sh_u, dev); l->qth_shd = qdw_place(l->sh_d, dev);
- if (l->qth_shg) { placed++; vram += (double)qdw_bytes(l->sh_g); }
- if (l->qth_shu) { placed++; vram += (double)qdw_bytes(l->sh_u); }
- if (l->qth_shd) { placed++; vram += (double)qdw_bytes(l->sh_d); }
++ l->qth_shg = qdw_place(&l->sh_g, dev); l->qth_shu = qdw_place(&l->sh_u, dev); l->qth_shd = qdw_place(&l->sh_d, dev);
++ if (l->qth_shg) { placed++; vram += (double)qdw_bytes(&l->sh_g); }
++ if (l->qth_shu) { placed++; vram += (double)qdw_bytes(&l->sh_u); }
++ if (l->qth_shd) { placed++; vram += (double)qdw_bytes(&l->sh_d); }
+ }
+ if (vram_bytes) *vram_bytes = vram;
+ return placed;
+ }
+
+ /* Measured, not assumed. The placer prices a trunk component by the bytes it
+ * saves on the CPU's memory bus, which presumes the GPU answers a GEMV faster
+ * than the CPU does. Four Tesla M10 (sm_50) said otherwise: every placed
+ * component ran slower there, lm_head 68.8 ms against 41.7 on the CPU, and
+ * decode fell from 3.56 to 2.68 tok/s (#1652). So before any trunk upload,
+ * one DeltaNet input projection (the most numerous placed matrix) is timed
+ * both ways on the device that would host it, ten GEMVs each, best of three
+ * rounds after a warm-up, and the trunk goes to VRAM only if the GPU wins.
+ * Only the automatic placement is questioned: a hand-written COLI_PLACE
+ * stands. COLI_TRUNK_PROBE=0 skips the probe and trusts the placer. The
+ * probe's copy stays resident (one projection, ~25 MB on the 35B). */
+ static int trunk_probe_gpu_wins(Model *m){
+ const char *e = getenv("COLI_TRUNK_PROBE");
+ if (e && *e == '0') return 1;
+ if (!qt_place_is_auto()) return 1;
+ Cfg *c = &m->c;
- int qi = -1, dev = QT_PLACE_CPU;
- for (int i = 0; i < c->n_layers && qi < 0; i++) {
++ QW *w = NULL; int dev = QT_PLACE_CPU;
++ for (int i = 0; i < c->n_layers && !w; i++) {
+ if (c->is_attn[i]) continue;
- for (int j = 0; j < g_qdw_n; j++)
- if (g_qdw[j].w == m->L[i].dn_qkv) { qi = j; dev = qt_place_of("dnproj", i); break; }
++ if (m->L[i].dn_qkv.q) { w = &m->L[i].dn_qkv; dev = qt_place_of("dnproj", i); }
+ }
- if (qi < 0) return 1; /* dense-i8 off: nothing will be placed */
++ if (!w) return 1; /* dense-i8 off: nothing will be placed */
+ if (dev == QT_PLACE_CPU) dev = qt_place_of("lmhead", 0);
+ if (dev == QT_PLACE_CPU) return 1; /* nothing placed: nothing to measure */
- int I = g_qdw[qi].I, O = g_qdw[qi].O;
- int h = qt_dense_init(g_qdw[qi].q, g_qdw[qi].sc, I, O, dev);
++ int I = w->I, O = w->O;
++ int h = qt_dense_init(w->q, w->sc, I, O, dev);
+ if (h < 0) return 1; /* cannot measure: the placer's word stands */
+ float *x = malloc((size_t)I * sizeof(float)), *y = malloc((size_t)O * sizeof(float));
+ if (!x || !y) { free(x); free(y); return 1; }
+ for (int i = 0; i < I; i++) x[i] = sinf(0.37f * (float)i);
+ double gpu = 1e30, cpu = 1e30;
+ for (int r = 0; r < 3; r++) {
+ for (int k = 0; k < 3; k++) if (!qt_dense_matmul(h, y, x, I, O)) { free(x); free(y); return 1; }
+ double t0 = now_s();
+ for (int k = 0; k < 10; k++) if (!qt_dense_matmul(h, y, x, I, O)) { free(x); free(y); return 1; }
+ double tg = (now_s() - t0) / 10;
- for (int k = 0; k < 3; k++) matmul_q(y, x, g_qdw[qi].q, g_qdw[qi].sc, I, O);
++ for (int k = 0; k < 3; k++) matmul_q(y, x, w->q, w->sc, I, O);
+ t0 = now_s();
- for (int k = 0; k < 10; k++) matmul_q(y, x, g_qdw[qi].q, g_qdw[qi].sc, I, O);
++ for (int k = 0; k < 10; k++) matmul_q(y, x, w->q, w->sc, I, O);
+ double tc = (now_s() - t0) / 10;
+ if (tg < gpu) gpu = tg;
+ if (tc < cpu) cpu = tc;
+ }
+ free(x); free(y);
+ int wins = gpu < cpu;
+ fprintf(stderr, "[place] probe: one [%d x %d] int8 GEMV takes %.3f ms on CUDA dev %d, %.3f ms on the CPU -> trunk %s\n",
+ O, I, gpu * 1e3, dev, cpu * 1e3, wins ? "to VRAM" : "stays on the CPU");
+ return wins;
+ }
+
static void layers_forward_range(Model *m, float *x, int S, int pos_base,
int layer_begin, int layer_end,
int allow_prefetch, FILE *lf) {
@@@ -3477,8 -3800,36 +3849,10 @@@ int main(int argc, char **argv)
g_expert_gs = m.c.expert_gs;
if (g_expert_gs) fprintf(stderr, "[qwen36] group-scaled experts: gs=%d\n", g_expert_gs);
fprintf(stderr, "resident weights loaded in %.1fs | RSS after load: %.2f GB\n", m.dense_load_s, rss_gb());
- /* dense matrices are quantized to int8 during model_init above (COLI_DENSE_I8=0
- * disables it) -- see load_tq/QW; model_init_range already logged the count. */
- /* quantize the large dense matrices to int8 (COLI_DENSE_I8=0 disables) */
- if (dense_i8_on()) {
- double tq = now_s();
- Cfg *qc = &m.c; int D2 = qc->hidden;
- int q_out = qc->q_heads * qc->q_head_dim, kv_out = qc->kv_heads * qc->k_head_dim;
- for (int i = 0; i < qc->n_layers; i++) {
- Layer *l = &m.L[i];
- qdw_register_as(l->q, D2, q_out, "attn"); qdw_register_as(l->k, D2, kv_out, "attn");
- qdw_register_as(l->v, D2, kv_out, "attn"); qdw_register_as(l->o, qc->o_in, D2, "attn");
- qdw_register_as(l->gate, D2, qc->n_experts, "router");
- qdw_register_as(l->sh_g, D2, qc->shared_inter, "shexp"); qdw_register_as(l->sh_u, D2, qc->shared_inter, "shexp");
- qdw_register_as(l->sh_d, qc->shared_inter, D2, "shexp");
- qdw_register_as(l->dn_qkv, D2, qc->dn_conv_dim, "dnproj");
- qdw_register_as(l->dn_z, D2, qc->dn_vheads * qc->dn_vdim, "dnproj");
- qdw_register_as(l->dn_out, qc->dn_vheads * qc->dn_vdim, D2, "dnout");
- }
- qdw_register_as(m.lm_head, D2, qc->vocab, "lmhead");
- /* Free the f32 originals -- the pointers only serve as lookup keys in
- * matmul_d from here on (never dereferenced again).
- * COLI_KEEP_F32=1 keeps them (debug). */
- double freed = 0;
- if (!getenv("COLI_KEEP_F32")) {
- for (int i = 0; i < g_qdw_n; i++) {
- freed += (double)g_qdw[i].I * g_qdw[i].O * sizeof(float);
- free((void*)g_qdw[i].w);
- }
- }
- fprintf(stderr, "[dense-i8] %d matrices quantized in %.1f s, %.1f GB f32 freed\n",
- g_qdw_n, now_s()-tq, freed/1073741824.0);
- }
++ /* dense matrices are quantized to int8 (+ int4 planar per COLI_DENSE_BITS/
++ * COLI_DENSE_INT4, see dense_int4_wanted) during model_init above
++ * (COLI_DENSE_I8=0 disables it) -- see load_tq/QW; model_init_range
++ * already logged the count and freed bytes. */
/* Optional CUDA VRAM expert tier (COLI_CUDA=1): hot experts live in
* DEVICE_LOCAL memory across the configured GPUs, misses fall back to the
@@@ -3508,23 -3859,33 +3882,26 @@@
* No entry (dense-i8 off) means nothing to offer, and the CPU path stands. */
{
int O_qkv = m.c.dn_conv_dim, O_z = m.c.dn_vheads * m.c.dn_vdim;
- for (int i = 0; i < g_qdw_n; i++)
- if (g_qdw[i].w == m.lm_head)
- qt_trunk_offer("lmhead", 0, (size_t)g_qdw[i].I * g_qdw[i].O + (size_t)g_qdw[i].O * sizeof(float));
+ if (m.lm_head.q)
+ qt_trunk_offer("lmhead", 0, (size_t)m.lm_head.I * m.lm_head.O + (size_t)m.lm_head.O * sizeof(float));
for (int i = 0; i < m.c.n_layers; i++) {
if (m.c.is_attn[i]) continue;
- int have = 0;
- for (int j = 0; j < g_qdw_n; j++)
- if (g_qdw[j].w == m.L[i].dn_qkv || g_qdw[j].w == m.L[i].dn_z) have++;
- if (have == 2)
+ if (m.L[i].dn_qkv.q && m.L[i].dn_z.q)
qt_trunk_offer("dnproj", i, (size_t)(O_qkv + O_z) * m.c.hidden + (size_t)(O_qkv + O_z) * sizeof(float));
}
+ trunk_offer_dense(&m); /* dnout, attnproj, shexp: the rest of the per-token dense work */
}
if (qt_init(m.c.n_layers, m.c.n_experts, m.c.hidden, m.c.inter, cap, m.c.topk,
m.c.expert_gs, expert_is_int4)) {
fprintf(stderr, "[gpu] MoE experts -> CUDA VRAM tier\n");
atexit(qt_shutdown);
+ /* The placer predicted; measure before uploading a byte of trunk. */
+ if (!trunk_probe_gpu_wins(&m)) qt_trunk_withdraw("measured slower than the CPU");
/* R4 role split: park the dense-i8 lm_head on COLI_LMHEAD_GPU. The
- * qdw entry keyed by m.lm_head holds the int8 rows + per-row scales
- * the CPU path uses; the GPU applies the identical semantics. */
- for (int i = 0; i < g_qdw_n; i++)
- if (g_qdw[i].w == m.lm_head) {
- qt_lmhead_init(g_qdw[i].q, g_qdw[i].sc, g_qdw[i].I, g_qdw[i].O);
- break;
- }
+ * QW struct on m.lm_head holds the int8 rows + per-row scales the
+ * CPU path uses; the GPU applies the identical semantics. */
+ if (m.lm_head.q)
+ qt_lmhead_init(m.lm_head.q, m.lm_head.sc, m.lm_head.I, m.lm_head.O);
/* R4 step 2: DeltaNet input projections, per layer, wherever
* COLI_PLACE puts them. qkv and z are both [O_x, hidden] int8 with
* per-row scales, so fusing them is a concatenation along O -- two
@@@ -4108,10 -4481,9 +4494,10 @@@ static int qwen36_edge_engine_open
engine->model.embed = load_t_n(
&engine->model, "model.embed_tokens.weight",
(int64_t)config->vocab * config->hidden);
- engine->model.lm_head = load_t_n(
- &engine->model, "lm_head.weight",
- (int64_t)config->vocab * config->hidden);
+ /* quantize=0: this engine never ran the old post-hoc qdw_register pass
+ * either (only main()'s static Model did), so lm_head stays f32-only here,
+ * exactly as before. */
- load_tq(&engine->model, "lm_head.weight", config->hidden, config->vocab, 0, &engine->model.lm_head);
++ load_tq(&engine->model, "lm_head.weight", config->hidden, config->vocab, 0, "lmhead", &engine->model.lm_head);
engine->model.final_norm = load_t_n(
&engine->model, "model.norm.weight", config->hidden);
engine->model.quant_bits = container_layer_is_int4(&engine->model, 0) ? 4 : 8;
diff --cc c/tests/bench_qwen36_dense_batch.c
index 7e29d5b3,63f0fc22..36e19e55
--- a/c/tests/bench_qwen36_dense_batch.c
+++ b/c/tests/bench_qwen36_dense_batch.c
@@@ -39,13 -39,13 +39,13 @@@ static double shared_run(Model *m,Laye
static int shared_benchmark(void) {
Model m;memset(&m,0,sizeof(m));m.c.hidden=I;m.c.shared_inter=O;
Layer l;memset(&l,0,sizeof(l));
- l.sh_g=falloc((int64_t)O*I);l.sh_u=falloc((int64_t)O*I);
- l.sh_d=falloc((int64_t)I*O);l.sh_gate=falloc(I);
- for(int64_t i=0;i<(int64_t)O*I;i++){l.sh_g[i]=value(i,2);l.sh_u[i]=value(i,3);}
- for(int64_t i=0;i<(int64_t)I*O;i++)l.sh_d[i]=value(i,4);
+ l.sh_g.w=falloc((int64_t)O*I);l.sh_u.w=falloc((int64_t)O*I);
+ l.sh_d.w=falloc((int64_t)I*O);l.sh_gate=falloc(I);
+ for(int64_t i=0;i<(int64_t)O*I;i++){((float*)l.sh_g.w)[i]=value(i,2);((float*)l.sh_u.w)[i]=value(i,3);}
+ for(int64_t i=0;i<(int64_t)I*O;i++)((float*)l.sh_d.w)[i]=value(i,4);
for(int i=0;i<I;i++)l.sh_gate[i]=value(i,5);
- qw_quantize(l.sh_g.w,I,O,&l.sh_g);qw_quantize(l.sh_u.w,I,O,&l.sh_u);qw_quantize(l.sh_d.w,O,I,&l.sh_d);
- qdw_register(l.sh_g,I,O);qdw_register(l.sh_u,I,O);qdw_register(l.sh_d,O,I);
- if(g_qdw_n!=3){fprintf(stderr,"FAIL: expected three dense-int8 copies, got %d\n",g_qdw_n);return 1;}
++ qw_quantize(l.sh_g.w,I,O,NULL,&l.sh_g);qw_quantize(l.sh_u.w,I,O,NULL,&l.sh_u);qw_quantize(l.sh_d.w,O,I,NULL,&l.sh_d);
+ if(!l.sh_g.q||!l.sh_u.q||!l.sh_d.q){fprintf(stderr,"FAIL: expected three dense-int8 copies\n");return 1;}
float *x=falloc((int64_t)S*I),*seed=falloc((int64_t)S*I);
float *a=falloc((int64_t)S*I),*b=falloc((int64_t)S*I);
for(int64_t i=0;i<(int64_t)S*I;i++){x[i]=value(i,6);seed[i]=value(i,7);}
diff --cc c/tests/test_qwen36_dense_batch.c
index 8f7807d6,69288725..d513e814
--- a/c/tests/test_qwen36_dense_batch.c
+++ b/c/tests/test_qwen36_dense_batch.c
@@@ -69,13 -57,12 +69,13 @@@ static void shared_case(const char *for
enum { S=12,D=64,I=32 };
Model m;memset(&m,0,sizeof(m));m.c.hidden=D;m.c.shared_inter=I;
Layer l;memset(&l,0,sizeof(l));
- l.sh_g=falloc((int64_t)I*D);l.sh_u=falloc((int64_t)I*D);
- l.sh_d=falloc((int64_t)D*I);l.sh_gate=falloc(D);
- for(int64_t i=0;i<(int64_t)I*D;i++){l.sh_g[i]=input_value(i,2);l.sh_u[i]=input_value(i,3);}
- for(int64_t i=0;i<(int64_t)D*I;i++)l.sh_d[i]=input_value(i,4);
+ l.sh_g.w=falloc((int64_t)I*D);l.sh_u.w=falloc((int64_t)I*D);
+ l.sh_d.w=falloc((int64_t)D*I);l.sh_gate=falloc(D);
+ l.sh_g.I=D;l.sh_g.O=I; l.sh_u.I=D;l.sh_u.O=I; l.sh_d.I=I;l.sh_d.O=D;
+ for(int64_t i=0;i<(int64_t)I*D;i++){((float*)l.sh_g.w)[i]=input_value(i,2);((float*)l.sh_u.w)[i]=input_value(i,3);}
+ for(int64_t i=0;i<(int64_t)D*I;i++)((float*)l.sh_d.w)[i]=input_value(i,4);
for(int i=0;i<D;i++)l.sh_gate[i]=input_value(i,5);
- if(quantized){qw_quantize(l.sh_g.w,D,I,&l.sh_g);qw_quantize(l.sh_u.w,D,I,&l.sh_u);qw_quantize(l.sh_d.w,I,D,&l.sh_d);}
- if(quantized){qdw_register(l.sh_g,D,I);qdw_register(l.sh_u,D,I);qdw_register(l.sh_d,I,D);}
++ if(quantized){qw_quantize(l.sh_g.w,D,I,NULL,&l.sh_g);qw_quantize(l.sh_u.w,D,I,NULL,&l.sh_u);qw_quantize(l.sh_d.w,I,D,NULL,&l.sh_d);}
float *x=falloc((int64_t)S*D),*seed=falloc((int64_t)S*D);
float *ref=falloc((int64_t)S*D),*got=falloc((int64_t)S*D);
float *g=falloc(I),*u=falloc(I),*hh=falloc(D);
diff --cc c/tests/test_qwen36_dense_idot.c
index 00000000,c9c2ef47..c506c81a
mode 000000,100644..100644
--- a/c/tests/test_qwen36_dense_idot.c
+++ b/c/tests/test_qwen36_dense_idot.c
@@@ -1,0 -1,127 +1,128 @@@
+ /* The dense trunk's integer path (COLI_DENSE_IDOT, COLI_DENSE_BITS=4).
+ *
+ * Pins, with no model:
+ * - dense_act_i8's vector path equals the scalar lrintf contract bit for bit
+ * (same rounding, same scale) and its block sums are exact;
+ * - pack_int4_g64_planar stores what the K1b layout expects: nibble v+8,
+ * lo nibbles = elements 0..31 of the block, hi = 32..63, one scale per 64;
+ * - matmul_d through the int4 grouped kernel and through the int8 integer
+ * kernel both agree with the f32 reference within what an int8 activation
+ * and an int4 weight allow, on every row of a random matrix, for one row
+ * (decode) and for several rows (prefill);
+ * - with neither flag the dispatch is the one before: the int8 f32-activation
+ * kernel, byte-identical to its own output. */
+ #include <stdio.h>
+ #include <stdlib.h>
+ #include <string.h>
+ #include <math.h>
+
+ #define main qwen36_main_unused
+ #include "../qwen36.c"
+ #undef main
+ #include "../compat.h" /* setenv: MinGW has none */
+
+ static int fails;
+ static void ck(int ok, const char *what) { if (ok) { printf(" ok %s\n", what); return; } printf(" FAIL %s\n", what); fails++; }
+
+ static unsigned g_seed = 4242;
+ static float rnd(void) { g_seed = g_seed * 1103515245u + 12345u; return ((g_seed >> 8) & 0xFFFF) / 32768.f - 1.f; }
+
+ static void ref_matmul(float *y, const float *x, const float *W, int S, int I, int O) {
+ for (int s = 0; s < S; s++) for (int o = 0; o < O; o++) {
+ double a = 0; for (int i = 0; i < I; i++) a += (double)x[(size_t)s * I + i] * W[(size_t)o * I + i];
+ y[(size_t)s * O + o] = (float)a;
+ }
+ }
+ static double rel_gap(const float *a, const float *b, int n) {
+ double worst = 0, scale = 1e-6;
+ for (int i = 0; i < n; i++) { double d = fabs((double)a[i] - b[i]); if (d > worst) worst = d; if (fabs((double)b[i]) > scale) scale = fabs((double)b[i]); }
+ return worst / scale;
+ }
+
+ int main(void) {
+ enum { I = 256, O = 96, S = 5 };
+ printf("activation quantizer\n");
+ {
+ float x[I]; int8_t a[I], b[I]; int32_t sums[I / 64];
+ for (int t = 0; t < 50; t++) {
+ float mag = (t % 5 == 0) ? 1e-3f : 3.f;
+ for (int i = 0; i < I; i++) x[i] = rnd() * mag;
+ float sa = dense_act_i8(x, I, a, sums);
+ float sb = qrow_i8(x, b, I);
+ if (sa != sb || memcmp(a, b, I)) { ck(0, "vector quantizer equals the scalar contract"); break; }
+ for (int g = 0; g < I / 64; g++) { int32_t s = 0; for (int k = 0; k < 64; k++) s += a[g * 64 + k]; if (s != sums[g]) { ck(0, "block sums are exact"); break; } }
+ if (t == 49) ck(1, "vector quantizer equals the scalar lrintf contract on 50 random rows, block sums exact");
+ }
+ }
+ printf("int4 planar packing\n");
+ {
+ float *W = malloc((size_t)O * I * sizeof(float));
+ for (size_t i = 0; i < (size_t)O * I; i++) W[i] = rnd();
+ uint8_t *q4 = malloc((size_t)O * I / 2); float *sg = malloc((size_t)O * (I / 64) * sizeof(float));
+ pack_int4_g64_planar(W, q4, sg, O, I);
+ int bad = 0;
+ for (int o = 0; o < O && !bad; o++) for (int g = 0; g < I / 64 && !bad; g++) {
+ const uint8_t *blk = q4 + (size_t)o * (I / 2) + g * 32;
+ float s = sg[(size_t)o * (I / 64) + g];
+ for (int k = 0; k < 64; k++) {
+ int nib = k < 32 ? (blk[k] & 0xF) : (blk[k - 32] >> 4);
+ float w = W[(size_t)o * I + g * 64 + k];
+ int want = (int)lrintf(w / s); if (want > 7) want = 7; if (want < -8) want = -8;
+ if (nib - 8 != want) { bad = 1; break; }
+ }
+ }
+ ck(!bad, "nibble v+8 in the planar order, one absmax/7 scale per block of 64");
+ free(W); free(q4); free(sg);
+ }
+ printf("matmul_d dispatch\n");
+ {
+ float *W = malloc((size_t)O * I * sizeof(float)), *x = malloc((size_t)S * I * sizeof(float));
+ float *ref = malloc((size_t)S * O * sizeof(float)), *y = malloc((size_t)S * O * sizeof(float)), *y0 = malloc((size_t)S * O * sizeof(float));
+ for (size_t i = 0; i < (size_t)O * I; i++) W[i] = rnd();
+ for (size_t i = 0; i < (size_t)S * I; i++) x[i] = rnd() * 2.f;
+ ref_matmul(ref, x, W, S, I, O);
- /* the classic path (no flags): int8 rows, f32 activations */
++ /* the classic path (no flags): int8 rows, f32 activations.
++ * dense_idot_on caches its answer: this TU reads the env once, so
++ * set it before the first call (matmul_d, via qw_quantize below). */
+ setenv("COLI_DENSE_IDOT", "0", 1); setenv("COLI_DENSE_BITS", "8", 1);
- qdw_register(W, I, O);
- matmul_d(y0, x, W, S, I, O);
++ QW w = {0}; qw_quantize(W, I, O, NULL, &w);
++ matmul_d(y0, x, &w, S, I, O);
+ double g0 = rel_gap(y0, ref, S * O);
+ ck(g0 < 2e-2, "classic int8 path within 2% of the f32 reference (per-row int8)");
- matmul_d(y, x, W, 1, I, O);
++ matmul_d(y, x, &w, 1, I, O);
+ ck(!memcmp(y, y0, (size_t)O * sizeof(float)), "one row and the first of a batch agree on the classic path");
- /* the integer path on the same int8 rows */
- g_qdw_n = 0;
- /* dense_idot_on caches its answer: this TU reads the env once, so set it before the first call */
++ qw_free(&w);
++ free(W); free(x); free(ref); free(y); free(y0);
+ }
+ {
+ /* fresh process-wide state for the flag readers: emulate by direct calls */
+ float *W = malloc((size_t)O * I * sizeof(float)), *x = malloc((size_t)S * I * sizeof(float));
+ float *ref = malloc((size_t)S * O * sizeof(float)), *y = malloc((size_t)S * O * sizeof(float));
+ for (size_t i = 0; i < (size_t)O * I; i++) W[i] = rnd();
+ for (size_t i = 0; i < (size_t)S * I; i++) x[i] = rnd() * 2.f;
+ ref_matmul(ref, x, W, S, I, O);
+ int8_t *q = malloc((size_t)O * I); float *sc = malloc((size_t)O * sizeof(float));
+ for (int o = 0; o < O; o++) { /* the per-row int8 the engine builds at load */
+ float am = 0.f; for (int i = 0; i < I; i++) { float a = fabsf(W[(size_t)o * I + i]); if (a > am) am = a; }
+ float s = am > 1e-12f ? am / 127.f : 1.f; sc[o] = s;
+ for (int i = 0; i < I; i++) { int v = (int)lrintf(W[(size_t)o * I + i] / s); if (v > 127) v = 127; if (v < -127) v = -127; q[(size_t)o * I + i] = (int8_t)v; }
+ }
+ int8_t *xq = malloc((size_t)S * I); float sx[S]; int32_t *xsg = malloc((size_t)S * (I / 64) * sizeof(int32_t));
+ for (int s = 0; s < S; s++) sx[s] = dense_act_i8(x + (size_t)s * I, I, xq + (size_t)s * I, xsg + (size_t)s * (I / 64));
+ matmul_q_idot(y, xq, sx, q, sc, S, I, O);
+ ck(rel_gap(y, ref, S * O) < 3e-2, "int8 weights x int8 activations within 3% of the f32 reference, 5 rows");
+ matmul_q_idot(y, xq, sx, q, sc, 1, I, O);
+ ck(rel_gap(y, ref, O) < 3e-2, "same for one row");
+ uint8_t *q4 = malloc((size_t)O * I / 2); float *sg = malloc((size_t)O * (I / 64) * sizeof(float));
+ pack_int4_g64_planar(W, q4, sg, O, I);
+ matmul_i4p_grouped_idot(y, xq, sx, xsg, q4, sg, S, I, O, 64);
+ ck(rel_gap(y, ref, S * O) < 8e-2, "int4 blocks of 64 x int8 activations within 8% of the f32 reference, 5 rows");
+ float y1[O];
+ matmul_i4p_grouped_idot(y1, xq, sx, xsg, q4, sg, 1, I, O, 64);
+ ck(!memcmp(y1, y, (size_t)O * sizeof(float)), "the one-row path and the row tile give the same bytes (the K1b contract)");
+ free(W); free(x); free(ref); free(y); free(q); free(sc); free(xq); free(xsg); free(q4); free(sg);
+ }
+ if (fails) { printf("test_qwen36_dense_idot: %d failure(s)\n", fails); return 1; }
+ printf("OK test_qwen36_dense_idot: the dense trunk's integer path\n");
+ return 0;
+ }
diff --cc c/tests/test_qwen36_slot_int8.c
index 58d1c2e6,00000000..80d60391
mode 100644,000000..100644
--- a/c/tests/test_qwen36_slot_int8.c
+++ b/c/tests/test_qwen36_slot_int8.c
@@@ -1,96 -1,0 +1,103 @@@
+/* Regression gate for wiring slot_ensure_int8() onto #1271's unpack_int4_to_int8
+ * (the same branchless AVX2/NEON/scalar kernel load_expert_merged already uses
+ * for the container's int4 read) instead of its own separate scalar
+ * nibble-unpack loop. Integer nibble extraction + sign-extend is bit-exact by
+ * construction (no reduction, no rounding), so this compares the NEW
+ * slot_ensure_int8 (calling unpack_int4_to_int8) against a copy of the OLD
+ * scalar loop it replaced -- not just "it compiles", a real regression check
+ * that the two nibble-unpack formulas produce identical bytes. */
+#define main qwen36_main_unused
+#include "../qwen36.c"
+#undef main
+
+static int failures;
+#define CHECK(cond, ...) do { if (!(cond)) { \
+ fprintf(stderr,"FAIL %s:%d: ",__FILE__,__LINE__); \
+ fprintf(stderr,__VA_ARGS__); fputc('\n',stderr); failures++; } } while (0)
+
+/* Verbatim copy of slot_ensure_int8's pre-#1271 scalar loop (the code this
+ * change replaces), used here only as the regression oracle. */
+static void old_unpack(int8_t *dst, const uint8_t *p, int64_t len) {
+ for (int64_t i = 0; i < len; i += 2) {
+ uint8_t b = p[i >> 1];
+ int8_t lo = (int8_t)(b & 0xF); if (lo & 8) lo -= 16;
+ int8_t hi = (int8_t)((b >> 4) & 0xF); if (hi & 8) hi -= 16;
+ dst[i] = lo; dst[i + 1] = hi;
+ }
+}
+
+static uint8_t *random_packed(int64_t nbytes, uint32_t seed) {
+ uint8_t *p = malloc((size_t)nbytes);
+ uint32_t x = seed ? seed : 1;
+ for (int64_t i = 0; i < nbytes; i++) {
+ x ^= x << 13; x ^= x >> 17; x ^= x << 5; /* xorshift32 */
+ p[i] = (uint8_t)x;
+ }
+ return p;
+}
+
+/* inter/hidden chosen so ng=inter*hidden and nd=hidden*inter land on both
+ * sides of the AVX2 16-byte (32-element) vector boundary across the three
- * segments (g/u sized ng, d sized nd). */
++ * segments (g/u sized ng, d sized nd). ng/nd must stay even: nibble-packing
++ * always yields ceil(n/2) bytes, and real containers guarantee even
++ * inter*hidden (SIMD/GS64-block-aligned dims), so old_unpack -- a verbatim
++ * copy of the pre-#1271 scalar loop, kept unmodified on purpose -- was never
++ * written to handle an odd total (its last iteration writes dst[i+1]
++ * unconditionally) and shouldn't be, either: that shape cannot occur. */
+static void one_case(int inter, int hidden, const char *label) {
+ Model m; memset(&m, 0, sizeof(m));
+ m.c.inter = inter; m.c.hidden = hidden;
+ int64_t ng = (int64_t)inter * hidden, nd = (int64_t)hidden * inter;
++ CHECK(ng % 2 == 0, "%s: ng must be even (nibble-packed, see one_case)", label);
++ if (ng % 2) return;
+
+ Slot s; memset(&s, 0, sizeof(s));
+ s.g4 = random_packed(ng / 2, 1001);
+ s.u4 = random_packed(ng / 2, 2002);
+ s.d4 = random_packed(nd / 2, 3003);
+
+ slot_ensure_int8(&m, &s);
+ CHECK(s.g != NULL, "%s: slot_ensure_int8 did not populate s->g", label);
+ if (!s.g) return;
+
+ int8_t *exp_g = malloc((size_t)ng), *exp_u = malloc((size_t)ng), *exp_d = malloc((size_t)nd);
+ old_unpack(exp_g, s.g4, ng);
+ old_unpack(exp_u, s.u4, ng);
+ old_unpack(exp_d, s.d4, nd);
+
+ CHECK(!memcmp(s.g, exp_g, (size_t)ng), "%s: gate segment differs from old scalar loop", label);
+ CHECK(!memcmp(s.u, exp_u, (size_t)ng), "%s: up segment differs from old scalar loop", label);
+ CHECK(!memcmp(s.d, exp_d, (size_t)nd), "%s: down segment differs from old scalar loop", label);
+ printf("qwen36 slot_ensure_int8 exact: %s inter=%d hidden=%d (ng=%lld nd=%lld)\n",
+ label, inter, hidden, (long long)ng, (long long)nd);
+
+ free(exp_g); free(exp_u); free(exp_d);
+ free(s.g); /* s.u/s.d point into the same block as s.g (slot_ensure_int8's `w`) */
+ free(s.g4); free(s.u4); free(s.d4);
+}
+
+int main(void) {
+ one_case(4, 8, "tiny, well under one vector"); /* ng=32, nd=32 */
+ one_case(16, 16, "exact AVX2 vector boundary"); /* ng=256, nd=256, /16=16 (mult of 16-byte block) */
+ one_case(17, 16, "vector body + scalar tail"); /* ng=272 -> nb=136, not mult of 16 */
- one_case(37, 41, "odd, real-expert-shaped"); /* ng=1517, nd=1517 */
++ one_case(37, 42, "odd, real-expert-shaped"); /* ng=1554, nd=1554 (even: see one_case) */
+ one_case(2048, 512, "large, real Qwen3.6 expert scale"); /* ng=1048576, nd=1048576 */
+
+ /* re-entrancy guard: slot_ensure_int8 must no-op when s->g is already set
+ * (the "rematerialize only if evicted" contract) -- unrelated to which
+ * unpack kernel runs, but worth pinning since this function was touched */
+ {
+ Model m; memset(&m, 0, sizeof(m)); m.c.inter = 4; m.c.hidden = 8;
+ Slot s; memset(&s, 0, sizeof(s));
+ s.g4 = random_packed(16, 42); s.u4 = random_packed(16, 43); s.d4 = random_packed(16, 44);
+ slot_ensure_int8(&m, &s);
+ int8_t *g_before = s.g;
+ slot_ensure_int8(&m, &s); /* should be a no-op: s->g already non-NULL */
+ CHECK(s.g == g_before, "slot_ensure_int8 re-ran when s->g was already set");
+ free(s.g); free(s.g4); free(s.u4); free(s.d4);
+ }
+
+ if (failures) { fprintf(stderr, "qwen36 slot_ensure_int8: %d failure(s)\n", failures); return 1; }
+ puts("qwen36 slot_ensure_int8: ok");
+ return 0;
+}
diff --cc c/tests/test_qwen36_trunk_dense.c
index 00000000,02a9e1a3..14bb0a41
mode 000000,100644..100644
--- a/c/tests/test_qwen36_trunk_dense.c
+++ b/c/tests/test_qwen36_trunk_dense.c
@@@ -1,0 -1,163 +1,172 @@@
+ /* The rest of the dense trunk offered to the VRAM placer: "dnout" (DeltaNet
+ * out_proj), "attnproj" (q, k, v, o of an attention layer) and "shexp" (the
+ * shared expert's gate, up, down), placed by name and layer and served from
+ * VRAM through qt_dense handles kept in the Layer.
+ *
+ * What is pinned, on the fake CUDA backend (no GPU, no toolkit):
+ * - trunk_offer_dense offers exactly the components whose every matrix has
+ * a dense-i8 copy, with the bytes of those copies;
+ * - trunk_place_dense uploads what the placer took and keeps one handle per
+ * matrix, counting matrices and bytes;
+ * - a placed matrix answers the same GEMV from VRAM as matmul_d does on the
+ * CPU (same int8 rows, same per-row scales), and a component without a
+ * handle keeps running matmul_d;
+ * - the placer prices every one of them as a dense component: with room for
+ * all, all go; with COLI_PLACE=off nothing is offered at all.
+ * Include order as in tests/test_qwen36_tier_int8_engine.c: the engine, the
+ * fake backend, then the tier, so the tier's statics live in this TU. */
+ #define main qwen36_main_unused
+ #include "../qwen36.c"
+ #undef main
+
+ #include "../compat.h" /* setenv/unsetenv: MinGW has neither */
+
+ #include "qwen36_fake_cuda.h"
+
+ #include "../qwen36_tier.c"
+
+ static int fails;
+ static void ck(int ok, const char *what) {
+ if (ok) { printf(" ok %s\n", what); return; }
+ printf(" FAIL %s\n", what);
+ fails++;
+ }
+
+ /* Small but every dimension distinct, so a swapped I/O would show. */
+ enum { NL = 2, D = 48, VH = 2, VD = 8, QH = 2, QD = 12, KVH = 1, KD = 8, SH = 20, NE = 4, IH = 16 };
+
+ static unsigned g_seed = 12345;
+ static float rnd(void) {
+ g_seed = g_seed * 1103515245u + 12345u;
+ return ((g_seed >> 8) & 0xFFFF) / 32768.f - 1.f;
+ }
+ static float *rnd_matrix(int rows, int cols) {
+ float *w = malloc((size_t)rows * cols * sizeof(float));
+ for (size_t i = 0; i < (size_t)rows * cols; i++) w[i] = rnd();
+ return w;
+ }
+
+ /* One GEMV both ways: from VRAM through the handle and on the CPU through
+ * matmul_d. The fake backend computes exactly what a real one does with these
+ * bytes (x . int8 row, times the row scale), so the two must agree to float
+ * accumulation order. */
-static double gemv_gap(int hp1, const float *W, int I, int O, int *served) {
++static double gemv_gap(int hp1, const QW *w, int I, int O, int *served) {
+ float *x = rnd_matrix(1, I), *ya = calloc((size_t)O, sizeof(float)), *yb = calloc((size_t)O, sizeof(float));
+ *served = qtd(hp1, ya, x, I, O);
- matmul_d(yb, x, W, 1, I, O);
++ matmul_d(yb, x, w, 1, I, O);
+ double worst = 0, scale = 1e-6;
+ for (int o = 0; o < O; o++) {
+ double d = fabs((double)ya[o] - yb[o]);
+ if (d > worst) worst = d;
+ if (fabs((double)yb[o]) > scale) scale = fabs((double)yb[o]);
+ }
+ free(x); free(ya); free(yb);
+ return worst / scale;
+ }
+
++/* Quantize a freshly-built random matrix straight into *out (the QW the
++ * Layer holds), then discard the f32 staging buffer -- the same "quantize
++ * once, free the f32 copy" shape load_tq uses. */
++static void register_qw(int rows, int cols, QW *out) {
++ float *w = rnd_matrix(rows, cols);
++ qw_quantize(w, cols, rows, NULL, out);
++ free(w);
++}
++
+ static void build(Model *m) {
+ memset(m, 0, sizeof *m);
+ Cfg *c = &m->c;
+ c->n_layers = NL; c->hidden = D; c->n_experts = NE; c->inter = IH; c->topk = 1; c->expert_gs = 0;
+ c->q_heads = QH; c->q_head_dim = QD; c->kv_heads = KVH; c->k_head_dim = KD; c->head_dim = KD; c->o_in = QH * KD;
+ c->dn_vheads = VH; c->dn_vdim = VD; c->shared_inter = SH;
+ c->is_attn = calloc(NL, 1); c->is_attn[1] = 1;
+ m->L = calloc(NL, sizeof(Layer));
+ Layer *dn = &m->L[0], *at = &m->L[1];
- dn->dn_out = rnd_matrix(D, VH * VD); qdw_register(dn->dn_out, VH * VD, D);
- dn->sh_g = rnd_matrix(SH, D); qdw_register(dn->sh_g, D, SH);
- dn->sh_u = rnd_matrix(SH, D); qdw_register(dn->sh_u, D, SH);
- dn->sh_d = rnd_matrix(D, SH); qdw_register(dn->sh_d, SH, D);
- at->q = rnd_matrix(QH * QD, D); qdw_register(at->q, D, QH * QD);
- at->k = rnd_matrix(KVH * KD, D); qdw_register(at->k, D, KVH * KD);
- at->v = rnd_matrix(KVH * KD, D); qdw_register(at->v, D, KVH * KD);
- at->o = rnd_matrix(D, QH * KD); qdw_register(at->o, QH * KD, D);
++ register_qw(D, VH * VD, &dn->dn_out);
++ register_qw(SH, D, &dn->sh_g);
++ register_qw(SH, D, &dn->sh_u);
++ register_qw(D, SH, &dn->sh_d);
++ register_qw(QH * QD, D, &at->q);
++ register_qw(KVH * KD, D, &at->k);
++ register_qw(KVH * KD, D, &at->v);
++ register_qw(D, QH * KD, &at->o);
+ /* the attention layer's shared expert is INCOMPLETE on purpose: gate and
+ * up have a dense-i8 copy, down does not (never registered), so "shexp"
+ * for layer 1 must not be offered and its three handles must stay 0 */
- at->sh_g = rnd_matrix(SH, D); qdw_register(at->sh_g, D, SH);
- at->sh_u = rnd_matrix(SH, D); qdw_register(at->sh_u, D, SH);
- at->sh_d = rnd_matrix(D, SH); /* no qdw_register */
++ register_qw(SH, D, &at->sh_g);
++ register_qw(SH, D, &at->sh_u);
++ /* at->sh_d: left zeroed (q == NULL) -- no dense-i8 copy, never registered */
+ }
+
+ int main(void) {
+ setenv("COLI_CUDA", "1", 1); setenv("COLI_GPUS", "0", 1);
+ setenv("QT_NO_WARMSTART", "1", 1); setenv("HEAT_FILE", "", 1);
+ setenv("COLI_PLACE", "", 1); /* "" == unset == auto */
+ setenv("CUDA_EXPERT_GB", "1", 1); /* room for everything */
+ unsetenv("COLI_DENSE_I8");
+ /* the GEMV from VRAM is compared with the CPU's f32-activation kernel, the
+ * contract the tier uploads: the integer dense path rounds the activation
+ * and is measured elsewhere (test_qwen36_dense_idot) */
+ setenv("COLI_DENSE_IDOT", "0", 1);
+ fake_ndev = 1; fake_uploads = 0; fake_dense_compute = 1;
+
+ Model m; build(&m);
+ Layer *dn = &m.L[0], *at = &m.L[1];
+
+ printf("offers\n");
+ int before = G_offer_n;
+ trunk_offer_dense(&m);
+ /* dnout(layer 0) + attnproj(layer 1) + shexp(layer 0); NOT shexp(layer 1) */
+ ck(G_offer_n - before == 3, "three components offered: dnout, attnproj, shexp of the complete layer only");
- size_t want_attn = qdw_bytes(at->q) + qdw_bytes(at->k) + qdw_bytes(at->v) + qdw_bytes(at->o);
++ size_t want_attn = qdw_bytes(&at->q) + qdw_bytes(&at->k) + qdw_bytes(&at->v) + qdw_bytes(&at->o);
+ int seen_attn = 0, seen_dnout = 0, seen_shexp1 = 0;
+ for (int o = before; o < G_offer_n; o++) {
+ if (!strcmp(G_offer[o].name, "attnproj") && G_offer[o].layer == 1 && G_offer[o].bytes == want_attn) seen_attn = 1;
- if (!strcmp(G_offer[o].name, "dnout") && G_offer[o].layer == 0 && G_offer[o].bytes == qdw_bytes(dn->dn_out)) seen_dnout = 1;
++ if (!strcmp(G_offer[o].name, "dnout") && G_offer[o].layer == 0 && G_offer[o].bytes == qdw_bytes(&dn->dn_out)) seen_dnout = 1;
+ if (!strcmp(G_offer[o].name, "shexp") && G_offer[o].layer == 1) seen_shexp1 = 1;
+ }
+ ck(seen_dnout, "dnout offered for the DeltaNet layer with the bytes of its dense-i8 copy");
+ ck(seen_attn, "attnproj offered for the attention layer as the sum of q, k, v, o");
+ ck(!seen_shexp1, "a shared expert missing one dense-i8 copy is not offered");
- ck(qdw_bytes(at->sh_d) == 0, "a matrix without a dense-i8 copy reports zero bytes");
++ ck(qdw_bytes(&at->sh_d) == 0, "a matrix without a dense-i8 copy reports zero bytes");
+
+ printf("placement\n");
+ ck(qt_init(NL, NE, D, IH, NE, 1, 0, 1), "tier starts (int4 mode, cap == n_experts)");
+ ck(qt_place_of("dnout", 0) == 0 && qt_place_of("attnproj", 1) == 0 && qt_place_of("shexp", 0) == 0,
+ "every offered component placed on the one device with room");
+ ck(qt_place_of("shexp", 1) == QT_PLACE_CPU, "the component never offered stays on the CPU");
+ double vram = 0;
+ int placed = trunk_place_dense(&m, &vram);
+ ck(placed == 8, "eight matrices placed: dnout, q k v o, and the complete shared expert");
- ck(vram == (double)(qdw_bytes(dn->dn_out) + want_attn + qdw_bytes(dn->sh_g) + qdw_bytes(dn->sh_u) + qdw_bytes(dn->sh_d)),
++ ck(vram == (double)(qdw_bytes(&dn->dn_out) + want_attn + qdw_bytes(&dn->sh_g) + qdw_bytes(&dn->sh_u) + qdw_bytes(&dn->sh_d)),
+ "placed bytes are the sum of the dense-i8 copies uploaded");
+ ck(fake_uploads == 8, "one upload per placed matrix");
+ ck(dn->qth_dnout > 0 && at->qth_q > 0 && at->qth_k > 0 && at->qth_v > 0 && at->qth_o > 0, "handles kept in the Layer");
+ ck(dn->qth_shg > 0 && dn->qth_shu > 0 && dn->qth_shd > 0, "shared expert handles kept in the DeltaNet layer");
+ ck(at->qth_shg == 0 && at->qth_shu == 0 && at->qth_shd == 0, "no handle for the shared expert that was not offered");
+
+ printf("GEMV from VRAM == matmul_d on the CPU\n");
- struct { const char *name; int hp1; const float *w; int I, O; } mats[] = {
- {"dn_out", dn->qth_dnout, dn->dn_out, VH * VD, D},
- {"q", at->qth_q, at->q, D, QH * QD}, {"k", at->qth_k, at->k, D, KVH * KD},
- {"v", at->qth_v, at->v, D, KVH * KD}, {"o", at->qth_o, at->o, QH * KD, D},
- {"sh_g", dn->qth_shg, dn->sh_g, D, SH}, {"sh_u", dn->qth_shu, dn->sh_u, D, SH}, {"sh_d", dn->qth_shd, dn->sh_d, SH, D},
++ struct { const char *name; int hp1; const QW *w; int I, O; } mats[] = {
++ {"dn_out", dn->qth_dnout, &dn->dn_out, VH * VD, D},
++ {"q", at->qth_q, &at->q, D, QH * QD}, {"k", at->qth_k, &at->k, D, KVH * KD},
++ {"v", at->qth_v, &at->v, D, KVH * KD}, {"o", at->qth_o, &at->o, QH * KD, D},
++ {"sh_g", dn->qth_shg, &dn->sh_g, D, SH}, {"sh_u", dn->qth_shu, &dn->sh_u, D, SH}, {"sh_d", dn->qth_shd, &dn->sh_d, SH, D},
+ };
+ for (size_t i = 0; i < sizeof mats / sizeof mats[0]; i++) {
+ int served = 0;
+ double gap = gemv_gap(mats[i].hp1, mats[i].w, mats[i].I, mats[i].O, &served);
+ char what[128];
+ snprintf(what, sizeof what, "%s: served from VRAM, relative gap %.2e", mats[i].name, gap);
+ ck(served && gap < 1e-4, what);
+ }
+ {
+ int served = 1; float x[D] = {0}, y[SH] = {0};
+ served = qtd(at->qth_shd, y, x, SH, D);
+ ck(!served, "a matrix with no handle is not served (the caller runs matmul_d)");
+ }
+
+ qt_shutdown();
+ if (fails) { printf("test_qwen36_trunk_dense: %d failure(s)\n", fails); return 1; }
+ printf("OK test_qwen36_trunk_dense: dnout, attnproj and shexp offered, placed and served from VRAM\n");
+ return 0;
+ }