Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

perf(inkling): batch shared experts during prefill

JustVugga774efdbab83
diff --git a/c/Makefile b/c/Makefile
index d336df04..f2e04e30 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1123,6 +1123,13 @@ tests/test_serve_codec$(EXE): tests/test_serve_codec.c serve_codec.h compat.h
tests/test_inkling_serve_framing$(EXE): tests/test_inkling_serve_framing.c inkling.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h serve_codec.h $(INK_CUDA_OBJ) $(METAL_OBJ)
$(CC) $(CFLAGS) $< $(INK_CUDA_OBJ) $(METAL_OBJ) -o $@ $(LDFLAGS)
+tests/test_inkling_shared_batch$(EXE): tests/test_inkling_shared_batch.c inkling.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h serve_codec.h $(INK_CUDA_OBJ) $(METAL_OBJ)
+ $(CC) $(CFLAGS) $< $(INK_CUDA_OBJ) $(METAL_OBJ) -o $@ $(LDFLAGS)
+
+# Reproducible local A/B for the shared-expert prefill path. This is timing
+# evidence, not a CI gate: build and run it explicitly on the target CPU.
+tests/bench_inkling_shared_batch$(EXE): tests/bench_inkling_shared_batch.c inkling.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h serve_codec.h $(INK_CUDA_OBJ) $(METAL_OBJ)
+
tests/test_inkling_cache_index$(EXE): tests/test_inkling_cache_index.c inkling.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h serve_codec.h $(INK_CUDA_OBJ) $(METAL_OBJ)
$(CC) $(CFLAGS) $< $(INK_CUDA_OBJ) $(METAL_OBJ) -o $@ $(LDFLAGS)
diff --git a/c/inkling.c b/c/inkling.c
index 0956de50..d584f244 100644
--- a/c/inkling.c
+++ b/c/inkling.c
@@ -212,18 +212,75 @@ static void matmul_h(float *y, const float *x, const uint16_t *W, int S, int I,
#pragma omp parallel for schedule(static)
for (int o = 0; o < O; o++) {
const uint16_t *w = W + (int64_t)o * I;
- for (int s = 0; s < S; s++) {
- const uint16_t *xs = xh + (int64_t)s * I;
+ int s = 0;
+ for (; s + 3 < S; s += 4) {
+ const uint16_t *x0 = xh + (int64_t)(s+0)*I;
+ const uint16_t *x1 = xh + (int64_t)(s+1)*I;
+ const uint16_t *x2 = xh + (int64_t)(s+2)*I;
+ const uint16_t *x3 = xh + (int64_t)(s+3)*I;
+ __m512 a0 = _mm512_setzero_ps(), a1 = a0, a2 = a0, a3 = a0;
+ for (int i = 0; i < I; i += 32) {
+ __m512bh wb = (__m512bh)_mm512_loadu_si512(w + i);
+ a0 = _mm512_dpbf16_ps(a0, (__m512bh)_mm512_loadu_si512(x0 + i), wb);
+ a1 = _mm512_dpbf16_ps(a1, (__m512bh)_mm512_loadu_si512(x1 + i), wb);
+ a2 = _mm512_dpbf16_ps(a2, (__m512bh)_mm512_loadu_si512(x2 + i), wb);
+ a3 = _mm512_dpbf16_ps(a3, (__m512bh)_mm512_loadu_si512(x3 + i), wb);
+ }
+ y[(int64_t)(s+0)*O + o] = _mm512_reduce_add_ps(a0);
+ y[(int64_t)(s+1)*O + o] = _mm512_reduce_add_ps(a1);
+ y[(int64_t)(s+2)*O + o] = _mm512_reduce_add_ps(a2);
+ y[(int64_t)(s+3)*O + o] = _mm512_reduce_add_ps(a3);
+ }
+ for (; s < S; s++) {
+ const uint16_t *xs = xh + (int64_t)s*I;
__m512 acc = _mm512_setzero_ps();
for (int i = 0; i < I; i += 32)
acc = _mm512_dpbf16_ps(acc, (__m512bh)_mm512_loadu_si512(xs + i),
(__m512bh)_mm512_loadu_si512(w + i));
- y[(int64_t)s * O + o] = _mm512_reduce_add_ps(acc);
+ y[(int64_t)s*O + o] = _mm512_reduce_add_ps(acc);
}
}
free(xh);
return;
}
+#endif
+ /* Prefill tile: decode each bf16 weight once for four independent rows.
+ * Explicit mul+add (rather than a C expression that the compiler may fuse)
+ * matches the non-FMA scalar oracle bit for bit. Every SIMD lane still
+ * accumulates i=0..I-1 in the original order. */
+#if defined(__AVX2__)
+ if (S > 1) {
+ #pragma omp parallel for schedule(static)
+ for (int o = 0; o < O; o++) {
+ const uint16_t *w = W + (int64_t)o * I;
+ int s = 0;
+ for (; s + 3 < S; s += 4) {
+ const float *x0 = x + (int64_t)(s+0)*I;
+ const float *x1 = x + (int64_t)(s+1)*I;
+ const float *x2 = x + (int64_t)(s+2)*I;
+ const float *x3 = x + (int64_t)(s+3)*I;
+ __m128 acc = _mm_setzero_ps();
+ for (int i = 0; i < I; i++) {
+ union { uint32_t u; float f; } v = { (uint32_t)w[i] << 16 };
+ __m128 xv = _mm_set_ps(x3[i], x2[i], x1[i], x0[i]);
+ acc = _mm_add_ps(acc, _mm_mul_ps(xv, _mm_set1_ps(v.f)));
+ }
+ float a[4]; _mm_storeu_ps(a, acc);
+ y[(int64_t)(s+0)*O + o] = a[0]; y[(int64_t)(s+1)*O + o] = a[1];
+ y[(int64_t)(s+2)*O + o] = a[2]; y[(int64_t)(s+3)*O + o] = a[3];
+ }
+ for (; s < S; s++) {
+ const float *xs = x + (int64_t)s * I;
+ float acc = 0.f;
+ for (int i = 0; i < I; i++) {
+ union { uint32_t u; float f; } v = { (uint32_t)w[i] << 16 };
+ acc += xs[i] * v.f;
+ }
+ y[(int64_t)s * O + o] = acc;
+ }
+ }
+ return;
+ }
#endif
#pragma omp parallel for schedule(static)
for (int o = 0; o < O; o++) {
@@ -245,16 +302,71 @@ static void matmul_h(float *y, const float *x, const uint16_t *W, int S, int I,
* una scala f32 ogni `gs` elementi lungo I (ng = ceil(I/gs) scale per riga).
* Differenza da matmul_q4 (per-riga): la scala cambia DENTRO la riga, quindi
* l'accumulo va chiuso a ogni gruppo invece che una volta sola a fine riga. */
+#if defined(__AVX2__) && defined(__FMA__)
+static inline float matmul_i4g_row_fma(const float *xs, const uint8_t *w,
+ const float *sc, int I, int gs, int ng) {
+ __m128 acc = _mm_setzero_ps();
+ for (int g = 0; g < ng; g++) {
+ int i0 = g*gs, i1 = i0+gs; if (i1 > I) i1 = I;
+ __m128 part = _mm_setzero_ps();
+ for (int i = i0; i < i1; i++) {
+ uint8_t b = w[i >> 1]; int q = (i & 1) ? (b >> 4) : (b & 15);
+ part = _mm_fmadd_ss(_mm_set_ss(xs[i]), _mm_set_ss((float)(q-8)), part);
+ }
+ acc = _mm_add_ss(acc, _mm_mul_ss(part, _mm_set_ss(sc[g])));
+ }
+ return _mm_cvtss_f32(acc);
+}
+#endif
static void matmul_i4g(float *y, const float *x, const uint8_t *p, const float *scale,
int S, int I, int O, int gs) {
int ng = (I + gs - 1) / gs;
int64_t rb = (I + 1) / 2;
+#if defined(__AVX2__) && defined(__FMA__)
+ /* Four prompt rows share nibble unpack and scale loads. The explicit FMA
+ * for each lane and the group-boundary mul+add match the scalar kernel's
+ * generated operation order bit for bit. */
+ if (S > 1) {
+ #pragma omp parallel for schedule(static)
+ for (int o = 0; o < O; o++) {
+ const uint8_t *w = p + (int64_t)o * rb;
+ const float *sc = scale + (int64_t)o * ng;
+ int s = 0;
+ for (; s + 3 < S; s += 4) {
+ const float *x0 = x + (int64_t)(s+0)*I, *x1 = x + (int64_t)(s+1)*I;
+ const float *x2 = x + (int64_t)(s+2)*I, *x3 = x + (int64_t)(s+3)*I;
+ __m128 acc = _mm_setzero_ps();
+ for (int g = 0; g < ng; g++) {
+ int i0 = g*gs, i1 = i0+gs; if (i1 > I) i1 = I;
+ __m128 part = _mm_setzero_ps();
+ for (int i = i0; i < i1; i++) {
+ uint8_t b = w[i >> 1]; int q = (i & 1) ? (b >> 4) : (b & 15);
+ __m128 xv = _mm_set_ps(x3[i], x2[i], x1[i], x0[i]);
+ part = _mm_fmadd_ps(xv, _mm_set1_ps((float)(q-8)), part);
+ }
+ acc = _mm_add_ps(acc, _mm_mul_ps(part, _mm_set1_ps(sc[g])));
+ }
+ float a[4]; _mm_storeu_ps(a, acc);
+ y[(int64_t)(s+0)*O + o] = a[0]; y[(int64_t)(s+1)*O + o] = a[1];
+ y[(int64_t)(s+2)*O + o] = a[2]; y[(int64_t)(s+3)*O + o] = a[3];
+ }
+ for (; s < S; s++) {
+ const float *xs = x + (int64_t)s*I;
+ y[(int64_t)s*O + o] = matmul_i4g_row_fma(xs, w, sc, I, gs, ng);
+ }
+ }
+ return;
+ }
+#endif
#pragma omp parallel for schedule(static)
for (int o = 0; o < O; o++) {
const uint8_t *w = p + (int64_t)o * rb;
const float *sc = scale + (int64_t)o * ng;
for (int s = 0; s < S; s++) {
const float *xs = x + (int64_t)s * I;
+#if defined(__AVX2__) && defined(__FMA__)
+ y[(int64_t)s * O + o] = matmul_i4g_row_fma(xs, w, sc, I, gs, ng);
+#else
float acc = 0.f;
for (int g = 0; g < ng; g++) {
int i0 = g * gs, i1 = i0 + gs; if (i1 > I) i1 = I;
@@ -267,6 +379,7 @@ static void matmul_i4g(float *y, const float *x, const uint8_t *p, const float *
acc += part * sc[g]; /* scala chiusa per gruppo */
}
y[(int64_t)s * O + o] = acc;
+#endif
}
}
}
@@ -285,7 +398,13 @@ static void matmul_i8r(float *y, const float *x, const int8_t *q, const float *s
}
}
+#ifdef COLI_INKLING_SHARED_BATCH_TEST
+static uint64_t g_matmul_w_calls;
+#endif
static void matmul_w(float *y, const float *x, Wt W, int S, int I, int O) {
+#ifdef COLI_INKLING_SHARED_BATCH_TEST
+ g_matmul_w_calls++;
+#endif
#ifdef COLI_CUDA
if (W.dev) {
if (ink_cuda_matmul_bf16(y, x, W.dev, S, I, O) == 0) return;
@@ -1273,24 +1392,70 @@ static void dense_mlp(Model *m, Layer *l, float *x, int S, float *out) {
* (3) compute. */
/* shared experts for all S positions: gamma inside (before down_proj is
* linear, so applied at the end). Factored out so the Metal path can run it
- * on the CPU while the last routed-expert round is in flight on the GPU. */
+ * on the CPU while the last routed-expert round is in flight on the GPU.
+ *
+ * Prefill batches positions so each shared matrix is traversed once per chunk,
+ * rather than once per token. Decode (S=1) deliberately stays on the original
+ * scalar path: its scratch remains tiny and the compiler sees a one-row GEMV.
+ * Bound the batch scratch to 64 MiB so a long prompt cannot turn this speedup
+ * into a memory spike. INK_SHARED_BATCH=0 restores the scalar path; a positive
+ * value sets a smaller maximum row count for deterministic A/B tests. */
+static int shared_batch_rows(int S, int D, int I) {
+ if (S <= 1) return 1;
+ const char *env = getenv("INK_SHARED_BATCH");
+ if (env) {
+ int requested = atoi(env);
+ if (requested <= 0) return 1;
+ if (requested < S) S = requested;
+ }
+ int64_t row_bytes = ((int64_t)2*I + D) * (int64_t)sizeof(float);
+ int64_t bounded = (64LL << 20) / (row_bytes > 0 ? row_bytes : 1);
+ if (bounded < 1) bounded = 1;
+ if (S > bounded) S = (int)bounded;
+ return S;
+}
static void shared_experts_cpu(Model *m, Layer *l, const float *x, int S,
float *out, const float *wgt,
float *g, float *u, float *hh) {
Cfg *c = &m->c;
int D = c->hidden, K = c->topk, I = c->moe_inter, ns = c->n_shared;
- for (int s = 0; s < S; s++) {
- const float *xs = x + (int64_t)s*D;
- float *os = out + (int64_t)s*D;
- const float *w = wgt + (int64_t)s*(K+ns);
+ if (ns <= 0 || S <= 0) return;
+ int B = shared_batch_rows(S, D, I);
+ if (B == 1) {
+ for (int s = 0; s < S; s++) {
+ const float *xs = x + (int64_t)s*D;
+ float *os = out + (int64_t)s*D;
+ const float *w = wgt + (int64_t)s*(K+ns);
+ for (int j = 0; j < ns; j++) {
+ matmul_w(g, xs, wt_off_i(l->sh_g, (int64_t)j*I*D, D), 1, D, I);
+ matmul_w(u, xs, wt_off_i(l->sh_u, (int64_t)j*I*D, D), 1, D, I);
+ for (int i = 0; i < I; i++) g[i] = siluf(g[i]) * u[i];
+ matmul_w(hh, g, wt_off_i(l->sh_d, (int64_t)j*D*I, I), 1, I, D);
+ for (int d = 0; d < D; d++) os[d] += w[K+j] * hh[d];
+ }
+ }
+ return;
+ }
+ float *bg = falloc((int64_t)2*B*I), *bu = bg + (int64_t)B*I;
+ float *bh = falloc((int64_t)B*D);
+ for (int base = 0; base < S; base += B) {
+ int rows = S - base < B ? S - base : B;
for (int j = 0; j < ns; j++) {
- matmul_w(g, xs, wt_off_i(l->sh_g, (int64_t)j*I*D, D), 1, D, I);
- matmul_w(u, xs, wt_off_i(l->sh_u, (int64_t)j*I*D, D), 1, D, I);
- for (int i = 0; i < I; i++) g[i] = siluf(g[i]) * u[i];
- matmul_w(hh, g, wt_off_i(l->sh_d, (int64_t)j*D*I, I), 1, I, D);
- for (int d = 0; d < D; d++) os[d] += w[K+j] * hh[d];
+ matmul_w(bg, x + (int64_t)base*D,
+ wt_off_i(l->sh_g, (int64_t)j*I*D, D), rows, D, I);
+ matmul_w(bu, x + (int64_t)base*D,
+ wt_off_i(l->sh_u, (int64_t)j*I*D, D), rows, D, I);
+ for (int64_t q = 0; q < (int64_t)rows*I; q++) bg[q] = siluf(bg[q]) * bu[q];
+ matmul_w(bh, bg, wt_off_i(l->sh_d, (int64_t)j*D*I, I), rows, I, D);
+ for (int s = 0; s < rows; s++) {
+ float *os = out + (int64_t)(base+s)*D;
+ const float *w = wgt + (int64_t)(base+s)*(K+ns);
+ const float *hs = bh + (int64_t)s*D;
+ for (int d = 0; d < D; d++) os[d] += w[K+j] * hs[d];
+ }
}
}
+ free(bg); free(bh);
}
static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
diff --git a/c/tests/bench_inkling_shared_batch.c b/c/tests/bench_inkling_shared_batch.c
new file mode 100644
index 00000000..e7a22a8b
--- /dev/null
+++ b/c/tests/bench_inkling_shared_batch.c
@@ -0,0 +1,118 @@
+/* Optional A/B for Inkling shared-expert prefill batching. The dimensions
+ * make the bf16 shared weights larger than the LLC of an ordinary desktop,
+ * while keeping the whole process comfortably below 100 MiB. Correctness is
+ * a bit-exact precondition; timing is reported but never used as a CI gate.
+ *
+ * make tests/bench_inkling_shared_batch ARCH=native
+ * OMP_NUM_THREADS=<physical cores> ./tests/bench_inkling_shared_batch
+ */
+#define COLI_INKLING_SHARED_BATCH_TEST 1
+#define main inkling_main_unused
+#include "../inkling.c"
+#undef main
+
+enum { S = 16, D = 2048, I = 1024, K = 2, NS = 2, REPS = 3 };
+
+static uint16_t to_bf16(float x) {
+ union { float f; uint32_t u; } v = { x };
+ return (uint16_t)(v.u >> 16);
+}
+
+static float val(int64_t i, int salt) {
+ int v = (int)((i * 29 + salt * 43) % 127);
+ return (float)(v - 63) / (float)(256 + salt);
+}
+
+static void fill_f32(float *p, int64_t n, int salt) {
+ for (int64_t i = 0; i < n; i++) p[i] = val(i, salt);
+}
+
+static void fill_bf16(uint16_t *p, int64_t n, int salt) {
+ for (int64_t i = 0; i < n; i++) p[i] = to_bf16(val(i, salt));
+}
+
+static void init_q4(Wt *w, int rows, int cols, int salt) {
+ const int gs=64, rb=(cols+1)/2, ng=(cols+gs-1)/gs;
+ w->qbits=4; w->gs=gs; w->qn=(int64_t)rows*rb;
+ w->q4=malloc((size_t)w->qn); w->qs=malloc((size_t)rows*ng*sizeof(float));
+ if(!w->q4||!w->qs){fputs("OOM q4 weights\n",stderr);exit(2);}
+ for(int64_t i=0;i<w->qn;i++){
+ int lo=(int)((i*5+salt)%16),hi=(int)((i*11+salt*3)%16);
+ w->q4[i]=(uint8_t)(lo|(hi<<4));
+ }
+ for(int64_t i=0;i<(int64_t)rows*ng;i++)w->qs[i]=0.01f*(float)(1+(i+salt)%7);
+}
+
+static double one(Model *m, Layer *l, const float *x, const float *wgt,
+ const float *seed, float *out, const char *mode) {
+ float *g = falloc(2 * I), *u = g + I, *hh = falloc(D);
+ if (mode) setenv("INK_SHARED_BATCH", mode, 1);
+ else unsetenv("INK_SHARED_BATCH");
+ memcpy(out, seed, (size_t)S * D * sizeof(float));
+ double t0 = now_s();
+ shared_experts_cpu(m, l, x, S, out, wgt, g, u, hh);
+ double dt = now_s() - t0;
+ free(g); free(hh);
+ return dt;
+}
+
+static int measure(const char *format, double weight_mib, Model *m, Layer *l,
+ const float *x, const float *wgt, const float *seed,
+ float *scalar, float *batch) {
+ one(m,l,x,wgt,seed,scalar,"0"); one(m,l,x,wgt,seed,batch,NULL);
+ if(memcmp(scalar,batch,(size_t)S*D*sizeof(float))){
+ int shown=0;float worst=0.f;
+ for(int q=0;q<S*D;q++)if(scalar[q]!=batch[q]){
+ float d=fabsf(scalar[q]-batch[q]);if(d>worst)worst=d;
+ if(shown++<4)fprintf(stderr,"%s diff[%d] scalar=%a batch=%a delta=%g\n",
+ format,q,scalar[q],batch[q],d);
+ }
+ fprintf(stderr,"FAIL %s: %d values differ, worst=%g\n",format,shown,worst);return 1;
+ }
+ double ts=0.0,tb=0.0;
+ for(int r=0;r<REPS;r++){
+ if(r&1){tb+=one(m,l,x,wgt,seed,batch,NULL);ts+=one(m,l,x,wgt,seed,scalar,"0");}
+ else{ts+=one(m,l,x,wgt,seed,scalar,"0");tb+=one(m,l,x,wgt,seed,batch,NULL);}
+ }
+ ts/=REPS;tb/=REPS;
+ printf("inkling shared %s: S=%d D=%d I=%d shared=%d bytes=%.1f MiB\n",
+ format,S,D,I,NS,weight_mib);
+ printf("scalar %.6f s batch %.6f s speedup %.2fx calls %d -> %d\n",
+ ts,tb,ts/tb,S*NS*3,NS*3);
+ return 0;
+}
+
+int main(void) {
+ Model m; memset(&m, 0, sizeof(m));
+ m.c.hidden = D; m.c.moe_inter = I; m.c.topk = K; m.c.n_shared = NS;
+ Layer l; memset(&l, 0, sizeof(l));
+ l.sh_g.h = malloc((size_t)NS * I * D * sizeof(uint16_t));
+ l.sh_u.h = malloc((size_t)NS * I * D * sizeof(uint16_t));
+ l.sh_d.h = malloc((size_t)NS * D * I * sizeof(uint16_t));
+ float *x = falloc((int64_t)S * D);
+ float *wgt = falloc((int64_t)S * (K + NS));
+ float *seed = falloc((int64_t)S * D);
+ float *scalar = falloc((int64_t)S * D), *batch = falloc((int64_t)S * D);
+ if (!l.sh_g.h || !l.sh_u.h || !l.sh_d.h) { fputs("OOM weights\n", stderr); return 2; }
+ fill_bf16(l.sh_g.h, (int64_t)NS * I * D, 1);
+ fill_bf16(l.sh_u.h, (int64_t)NS * I * D, 2);
+ fill_bf16(l.sh_d.h, (int64_t)NS * D * I, 3);
+ fill_f32(x, (int64_t)S * D, 4);
+ fill_f32(wgt, (int64_t)S * (K + NS), 5);
+ fill_f32(seed, (int64_t)S * D, 6);
+
+ int failed=measure("bf16",3.0*NS*D*I*sizeof(uint16_t)/1048576.0,
+ &m,&l,x,wgt,seed,scalar,batch);
+ free(l.sh_g.h);free(l.sh_u.h);free(l.sh_d.h);
+ memset(&l,0,sizeof(l));
+ init_q4(&l.sh_g,NS*I,D,1);init_q4(&l.sh_u,NS*I,D,2);init_q4(&l.sh_d,NS*D,I,3);
+ double qbytes=(double)(l.sh_g.qn+l.sh_u.qn+l.sh_d.qn);
+ qbytes+=(double)(NS*I*((D+63)/64)*2+NS*D*((I+63)/64))*sizeof(float);
+ failed|=measure("int4-g64",qbytes/1048576.0,&m,&l,x,wgt,seed,scalar,batch);
+
+ unsetenv("INK_SHARED_BATCH");
+ free(l.sh_g.q4);free(l.sh_g.qs);free(l.sh_u.q4);free(l.sh_u.qs);
+ free(l.sh_d.q4);free(l.sh_d.qs);
+ free(x); free(wgt); free(seed); free(scalar); free(batch);
+ return failed;
+}
diff --git a/c/tests/test_inkling_shared_batch.c b/c/tests/test_inkling_shared_batch.c
new file mode 100644
index 00000000..505e2af2
--- /dev/null
+++ b/c/tests/test_inkling_shared_batch.c
@@ -0,0 +1,106 @@
+/* Model-free oracle for Inkling shared-expert prefill batching. It drives the
+ * production helper in all resident formats (f32 oracle, bf16 checkpoint,
+ * int4-g64 sidecar), compares against the old scalar path bit for bit, and
+ * proves that the number of weight traversals is per chunk rather than per
+ * prompt position. */
+#define COLI_INKLING_SHARED_BATCH_TEST 1
+#define main inkling_main_unused
+#include "../inkling.c"
+#undef main
+
+static int failures;
+#define CHECK(cond, ...) do { if (!(cond)) { \
+ fprintf(stderr,"FAIL %s:%d: ",__FILE__,__LINE__); \
+ fprintf(stderr,__VA_ARGS__); fputc('\n',stderr); failures++; } } while (0)
+
+static float value(int64_t i, int salt) {
+ int v = (int)((i * 37 + salt * 17) % 101);
+ return (float)(v - 50) / (float)(71 + salt);
+}
+static void fill(float *p, int64_t n, int salt) {
+ for (int64_t i=0;i<n;i++) p[i]=value(i,salt);
+}
+static uint16_t bf16(float x) {
+ union { float f; uint32_t u; } v={x}; return (uint16_t)(v.u>>16);
+}
+static void fill_h(uint16_t *p, int64_t n, int salt) {
+ for (int64_t i=0;i<n;i++) p[i]=bf16(value(i,salt));
+}
+static void init_q4(Wt *w, int rows, int cols, int salt) {
+ int rb=(cols+1)/2,ng=(cols+63)/64;
+ w->qbits=4;w->gs=64;w->qn=(int64_t)rows*rb;
+ w->q4=malloc((size_t)w->qn);w->qs=malloc((size_t)rows*ng*sizeof(float));
+ CHECK(w->q4&&w->qs,"q4 allocation failed");if(!w->q4||!w->qs)exit(2);
+ for(int64_t i=0;i<w->qn;i++){
+ int lo=(int)((i*5+salt)%16),hi=(int)((i*11+salt*3)%16);
+ w->q4[i]=(uint8_t)(lo|(hi<<4));
+ }
+ for(int64_t i=0;i<(int64_t)rows*ng;i++)w->qs[i]=0.01f*(float)(1+(i+salt)%7);
+}
+static void free_w(Wt *w){free(w->f);free(w->h);free(w->q4);free(w->qs);}
+
+static void compare_paths(const char *format,Model *m,Layer *l,int S,
+ const float *x,const float *wgt,const float *seed){
+ Cfg *c=&m->c;int D=c->hidden,I=c->moe_inter,ns=c->n_shared;
+ size_t out_bytes=(size_t)S*D*sizeof(float);
+ float *scalar=falloc((int64_t)S*D),*out=falloc((int64_t)S*D);
+ float *g=falloc((int64_t)2*I),*u=g+I,*hh=falloc(D);
+ memcpy(scalar,seed,out_bytes);setenv("INK_SHARED_BATCH","0",1);
+ g_matmul_w_calls=0;shared_experts_cpu(m,l,x,S,scalar,wgt,g,u,hh);
+ CHECK(g_matmul_w_calls==(uint64_t)S*ns*3,
+ "%s scalar calls=%llu expected=%d",format,
+ (unsigned long long)g_matmul_w_calls,S*ns*3);
+
+ const char *modes[]={NULL,"3"};
+ for(int z=0;z<2;z++){
+ memcpy(out,seed,out_bytes);
+ if(modes[z])setenv("INK_SHARED_BATCH",modes[z],1);else unsetenv("INK_SHARED_BATCH");
+ g_matmul_w_calls=0;shared_experts_cpu(m,l,x,S,out,wgt,g,u,hh);
+ int chunks=modes[z]?(S+2)/3:1,expected=chunks*ns*3;
+ CHECK(!memcmp(out,scalar,out_bytes),"%s mode=%s is not scalar bit-exact",
+ format,modes[z]?modes[z]:"default");
+ CHECK(g_matmul_w_calls==(uint64_t)expected,
+ "%s mode=%s calls=%llu expected=%d",format,modes[z]?modes[z]:"default",
+ (unsigned long long)g_matmul_w_calls,expected);
+ printf("inkling shared: format=%s S=%d mode=%s calls=%d -> %llu\n",
+ format,S,modes[z]?modes[z]:"default",S*ns*3,
+ (unsigned long long)g_matmul_w_calls);
+ }
+
+ /* Decode must remain the original one-row GEMV path. */
+ memcpy(out,seed,(size_t)D*sizeof(float));unsetenv("INK_SHARED_BATCH");
+ g_matmul_w_calls=0;shared_experts_cpu(m,l,x,1,out,wgt,g,u,hh);
+ CHECK(!memcmp(out,scalar,(size_t)D*sizeof(float)),"%s decode row changed",format);
+ CHECK(g_matmul_w_calls==(uint64_t)ns*3,"%s decode used batch shape",format);
+ free(g);free(hh);free(scalar);free(out);
+}
+
+static void run_format(const char *format,int kind,int S,int D,int I){
+ enum{K=2,NS=2};Model m;memset(&m,0,sizeof(m));
+ m.c.hidden=D;m.c.moe_inter=I;m.c.topk=K;m.c.n_shared=NS;
+ Layer l;memset(&l,0,sizeof(l));int64_t gi=(int64_t)NS*I*D,di=(int64_t)NS*D*I;
+ if(kind==0){
+ l.sh_g.f=falloc(gi);l.sh_u.f=falloc(gi);l.sh_d.f=falloc(di);
+ fill(l.sh_g.f,gi,1);fill(l.sh_u.f,gi,2);fill(l.sh_d.f,di,3);
+ }else if(kind==1){
+ l.sh_g.h=malloc((size_t)gi*2);l.sh_u.h=malloc((size_t)gi*2);l.sh_d.h=malloc((size_t)di*2);
+ CHECK(l.sh_g.h&&l.sh_u.h&&l.sh_d.h,"bf16 allocation failed");
+ if(!l.sh_g.h||!l.sh_u.h||!l.sh_d.h)exit(2);
+ fill_h(l.sh_g.h,gi,1);fill_h(l.sh_u.h,gi,2);fill_h(l.sh_d.h,di,3);
+ }else{
+ init_q4(&l.sh_g,NS*I,D,1);init_q4(&l.sh_u,NS*I,D,2);init_q4(&l.sh_d,NS*D,I,3);
+ }
+ float *x=falloc((int64_t)S*D),*w=falloc((int64_t)S*(K+NS)),*seed=falloc((int64_t)S*D);
+ fill(x,(int64_t)S*D,4);fill(w,(int64_t)S*(K+NS),5);fill(seed,(int64_t)S*D,6);
+ compare_paths(format,&m,&l,S,x,w,seed);
+ free_w(&l.sh_g);free_w(&l.sh_u);free_w(&l.sh_d);free(x);free(w);free(seed);
+}
+
+int main(void){
+ run_format("f32",0,12,17,13);
+ run_format("bf16",1,12,32,16);
+ run_format("int4-g64",2,12,64,64);
+ unsetenv("INK_SHARED_BATCH");
+ if(failures){fprintf(stderr,"inkling shared batch: %d failure(s)\n",failures);return 1;}
+ puts("inkling shared batch: ok");return 0;
+}
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index bc45e51d..747099a3 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -353,6 +353,7 @@ Read **only** by `c/inkling.c`.
| `PIN_N` | `cap / 2` | Experts pinned per layer. Measured on the 975B: `cap/4` (19/layer) gave 83.6% hit / 0.32 tok/s, 40/layer gave 95.6% / 0.80 tok/s — decode fills run at queue depth ~1, so every pinned expert removes a ~35 ms stall. Clamped to `cap - 8`. |
| `REP_PEN` | `1.1` | Repetition penalty over a 128-token history (prompt tail + emitted). |
| `INK_DENSE_Q4` | auto | Use the `dense-int4g64/` sidecar for dense weights when that directory exists. `=0` forces the unquantized dense path. |
+| `INK_SHARED_BATCH` | auto | Prefill rows per shared-expert batch, bounded to 64 MiB of scratch. `=0` restores the scalar per-token path for A/B/debugging; a positive value caps the chunk size. Decode (`S=1`) is unchanged. |
| `INK_METAL_MIN_S` | `1` | Minimum batch S to send the MoE block to Metal. `=2` restores the prefill-only gate (which mattered when the residency set was absent and per-block `useResource` churn cost ~135 ms). |
| `INK_PREFIX_LOG` | unset | Log the KV-prefix reuse decision and its reason, as `K3_PREFIX_LOG` does for K3. |
| `GPU_DEV` | `0` | CUDA device index for the inkling CUDA backend. |
diff --git a/docs/inkling.md b/docs/inkling.md
index d5fae47b..3ecedaaa 100644
--- a/docs/inkling.md
+++ b/docs/inkling.md
@@ -191,6 +191,7 @@ SNAP=~/Models/inkling_i4 ./c/inkling -f warmup_prompts.txt -n 32
| `USAGE_SAVE=0` | don't rewrite the history (benchmark runs) |
| `NOGPU=1` / `GPU_DEV=<n>` | disable CUDA / select device |
| `IDOT=0` | byte-exact scalar int kernels (debugging) |
+| `INK_SHARED_BATCH=0` | disable shared-expert prefill batching for a controlled A/B; positive values cap rows per chunk. The default uses up to 64 MiB of bounded scratch and never changes decode (`S=1`) |
| `TOPP=<p>` | adaptive routing: keep routed experts up to cumulative weight `p`, drop the tail. **Trims the routing** — fewer experts read per token (the lever that matters on a disk-bound host), but a different computation from the declared top-k. Off by default; the run reports `[topp] … N/M routed used (X% trimmed)` so the trade is measurable. Same semantics as `TOPP` in `colibri.c` and `K3_TOPP` in `kimi_k3.c` |
| First positional arg | expert-cache cap per layer (`0` = auto-size from free RAM) |