Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
perf(inkling): batch shared experts during prefill
JustVugga774efdbab83
diff --git a/c/Makefile b/c/Makefileindex d336df04..f2e04e30 100644--- a/c/Makefile+++ b/c/Makefile@@ -1123,6 +1123,13 @@ tests/test_serve_codec$(EXE): tests/test_serve_codec.c serve_codec.h compat.htests/test_inkling_serve_framing$(EXE): tests/test_inkling_serve_framing.c inkling.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h serve_codec.h $(INK_CUDA_OBJ) $(METAL_OBJ)$(CC) $(CFLAGS) $< $(INK_CUDA_OBJ) $(METAL_OBJ) -o $@ $(LDFLAGS)+tests/test_inkling_shared_batch$(EXE): tests/test_inkling_shared_batch.c inkling.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h serve_codec.h $(INK_CUDA_OBJ) $(METAL_OBJ)+ $(CC) $(CFLAGS) $< $(INK_CUDA_OBJ) $(METAL_OBJ) -o $@ $(LDFLAGS)++# Reproducible local A/B for the shared-expert prefill path. This is timing+# evidence, not a CI gate: build and run it explicitly on the target CPU.+tests/bench_inkling_shared_batch$(EXE): tests/bench_inkling_shared_batch.c inkling.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h serve_codec.h $(INK_CUDA_OBJ) $(METAL_OBJ)+tests/test_inkling_cache_index$(EXE): tests/test_inkling_cache_index.c inkling.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h serve_codec.h $(INK_CUDA_OBJ) $(METAL_OBJ)$(CC) $(CFLAGS) $< $(INK_CUDA_OBJ) $(METAL_OBJ) -o $@ $(LDFLAGS)diff --git a/c/inkling.c b/c/inkling.cindex 0956de50..d584f244 100644--- a/c/inkling.c+++ b/c/inkling.c@@ -212,18 +212,75 @@ static void matmul_h(float *y, const float *x, const uint16_t *W, int S, int I,#pragma omp parallel for schedule(static)for (int o = 0; o < O; o++) {const uint16_t *w = W + (int64_t)o * I;- for (int s = 0; s < S; s++) {- const uint16_t *xs = xh + (int64_t)s * I;+ int s = 0;+ for (; s + 3 < S; s += 4) {+ const uint16_t *x0 = xh + (int64_t)(s+0)*I;+ const uint16_t *x1 = xh + (int64_t)(s+1)*I;+ const uint16_t *x2 = xh + (int64_t)(s+2)*I;+ const uint16_t *x3 = xh + (int64_t)(s+3)*I;+ __m512 a0 = _mm512_setzero_ps(), a1 = a0, a2 = a0, a3 = a0;+ for (int i = 0; i < I; i += 32) {+ __m512bh wb = (__m512bh)_mm512_loadu_si512(w + i);+ a0 = _mm512_dpbf16_ps(a0, (__m512bh)_mm512_loadu_si512(x0 + i), wb);+ a1 = _mm512_dpbf16_ps(a1, (__m512bh)_mm512_loadu_si512(x1 + i), wb);+ a2 = _mm512_dpbf16_ps(a2, (__m512bh)_mm512_loadu_si512(x2 + i), wb);+ a3 = _mm512_dpbf16_ps(a3, (__m512bh)_mm512_loadu_si512(x3 + i), wb);+ }+ y[(int64_t)(s+0)*O + o] = _mm512_reduce_add_ps(a0);+ y[(int64_t)(s+1)*O + o] = _mm512_reduce_add_ps(a1);+ y[(int64_t)(s+2)*O + o] = _mm512_reduce_add_ps(a2);+ y[(int64_t)(s+3)*O + o] = _mm512_reduce_add_ps(a3);+ }+ for (; s < S; s++) {+ const uint16_t *xs = xh + (int64_t)s*I;__m512 acc = _mm512_setzero_ps();for (int i = 0; i < I; i += 32)acc = _mm512_dpbf16_ps(acc, (__m512bh)_mm512_loadu_si512(xs + i),(__m512bh)_mm512_loadu_si512(w + i));- y[(int64_t)s * O + o] = _mm512_reduce_add_ps(acc);+ y[(int64_t)s*O + o] = _mm512_reduce_add_ps(acc);}}free(xh);return;}+#endif+ /* Prefill tile: decode each bf16 weight once for four independent rows.+ * Explicit mul+add (rather than a C expression that the compiler may fuse)+ * matches the non-FMA scalar oracle bit for bit. Every SIMD lane still+ * accumulates i=0..I-1 in the original order. */+#if defined(__AVX2__)+ if (S > 1) {+ #pragma omp parallel for schedule(static)+ for (int o = 0; o < O; o++) {+ const uint16_t *w = W + (int64_t)o * I;+ int s = 0;+ for (; s + 3 < S; s += 4) {+ const float *x0 = x + (int64_t)(s+0)*I;+ const float *x1 = x + (int64_t)(s+1)*I;+ const float *x2 = x + (int64_t)(s+2)*I;+ const float *x3 = x + (int64_t)(s+3)*I;+ __m128 acc = _mm_setzero_ps();+ for (int i = 0; i < I; i++) {+ union { uint32_t u; float f; } v = { (uint32_t)w[i] << 16 };+ __m128 xv = _mm_set_ps(x3[i], x2[i], x1[i], x0[i]);+ acc = _mm_add_ps(acc, _mm_mul_ps(xv, _mm_set1_ps(v.f)));+ }+ float a[4]; _mm_storeu_ps(a, acc);+ y[(int64_t)(s+0)*O + o] = a[0]; y[(int64_t)(s+1)*O + o] = a[1];+ y[(int64_t)(s+2)*O + o] = a[2]; y[(int64_t)(s+3)*O + o] = a[3];+ }+ for (; s < S; s++) {+ const float *xs = x + (int64_t)s * I;+ float acc = 0.f;+ for (int i = 0; i < I; i++) {+ union { uint32_t u; float f; } v = { (uint32_t)w[i] << 16 };+ acc += xs[i] * v.f;+ }+ y[(int64_t)s * O + o] = acc;+ }+ }+ return;+ }#endif#pragma omp parallel for schedule(static)for (int o = 0; o < O; o++) {@@ -245,16 +302,71 @@ static void matmul_h(float *y, const float *x, const uint16_t *W, int S, int I,* una scala f32 ogni `gs` elementi lungo I (ng = ceil(I/gs) scale per riga).* Differenza da matmul_q4 (per-riga): la scala cambia DENTRO la riga, quindi* l'accumulo va chiuso a ogni gruppo invece che una volta sola a fine riga. */+#if defined(__AVX2__) && defined(__FMA__)+static inline float matmul_i4g_row_fma(const float *xs, const uint8_t *w,+ const float *sc, int I, int gs, int ng) {+ __m128 acc = _mm_setzero_ps();+ for (int g = 0; g < ng; g++) {+ int i0 = g*gs, i1 = i0+gs; if (i1 > I) i1 = I;+ __m128 part = _mm_setzero_ps();+ for (int i = i0; i < i1; i++) {+ uint8_t b = w[i >> 1]; int q = (i & 1) ? (b >> 4) : (b & 15);+ part = _mm_fmadd_ss(_mm_set_ss(xs[i]), _mm_set_ss((float)(q-8)), part);+ }+ acc = _mm_add_ss(acc, _mm_mul_ss(part, _mm_set_ss(sc[g])));+ }+ return _mm_cvtss_f32(acc);+}+#endifstatic void matmul_i4g(float *y, const float *x, const uint8_t *p, const float *scale,int S, int I, int O, int gs) {int ng = (I + gs - 1) / gs;int64_t rb = (I + 1) / 2;+#if defined(__AVX2__) && defined(__FMA__)+ /* Four prompt rows share nibble unpack and scale loads. The explicit FMA+ * for each lane and the group-boundary mul+add match the scalar kernel's+ * generated operation order bit for bit. */+ if (S > 1) {+ #pragma omp parallel for schedule(static)+ for (int o = 0; o < O; o++) {+ const uint8_t *w = p + (int64_t)o * rb;+ const float *sc = scale + (int64_t)o * ng;+ int s = 0;+ for (; s + 3 < S; s += 4) {+ const float *x0 = x + (int64_t)(s+0)*I, *x1 = x + (int64_t)(s+1)*I;+ const float *x2 = x + (int64_t)(s+2)*I, *x3 = x + (int64_t)(s+3)*I;+ __m128 acc = _mm_setzero_ps();+ for (int g = 0; g < ng; g++) {+ int i0 = g*gs, i1 = i0+gs; if (i1 > I) i1 = I;+ __m128 part = _mm_setzero_ps();+ for (int i = i0; i < i1; i++) {+ uint8_t b = w[i >> 1]; int q = (i & 1) ? (b >> 4) : (b & 15);+ __m128 xv = _mm_set_ps(x3[i], x2[i], x1[i], x0[i]);+ part = _mm_fmadd_ps(xv, _mm_set1_ps((float)(q-8)), part);+ }+ acc = _mm_add_ps(acc, _mm_mul_ps(part, _mm_set1_ps(sc[g])));+ }+ float a[4]; _mm_storeu_ps(a, acc);+ y[(int64_t)(s+0)*O + o] = a[0]; y[(int64_t)(s+1)*O + o] = a[1];+ y[(int64_t)(s+2)*O + o] = a[2]; y[(int64_t)(s+3)*O + o] = a[3];+ }+ for (; s < S; s++) {+ const float *xs = x + (int64_t)s*I;+ y[(int64_t)s*O + o] = matmul_i4g_row_fma(xs, w, sc, I, gs, ng);+ }+ }+ return;+ }+#endif#pragma omp parallel for schedule(static)for (int o = 0; o < O; o++) {const uint8_t *w = p + (int64_t)o * rb;const float *sc = scale + (int64_t)o * ng;for (int s = 0; s < S; s++) {const float *xs = x + (int64_t)s * I;+#if defined(__AVX2__) && defined(__FMA__)+ y[(int64_t)s * O + o] = matmul_i4g_row_fma(xs, w, sc, I, gs, ng);+#elsefloat acc = 0.f;for (int g = 0; g < ng; g++) {int i0 = g * gs, i1 = i0 + gs; if (i1 > I) i1 = I;@@ -267,6 +379,7 @@ static void matmul_i4g(float *y, const float *x, const uint8_t *p, const float *acc += part * sc[g]; /* scala chiusa per gruppo */}y[(int64_t)s * O + o] = acc;+#endif}}}@@ -285,7 +398,13 @@ static void matmul_i8r(float *y, const float *x, const int8_t *q, const float *s}}+#ifdef COLI_INKLING_SHARED_BATCH_TEST+static uint64_t g_matmul_w_calls;+#endifstatic void matmul_w(float *y, const float *x, Wt W, int S, int I, int O) {+#ifdef COLI_INKLING_SHARED_BATCH_TEST+ g_matmul_w_calls++;+#endif#ifdef COLI_CUDAif (W.dev) {if (ink_cuda_matmul_bf16(y, x, W.dev, S, I, O) == 0) return;@@ -1273,24 +1392,70 @@ static void dense_mlp(Model *m, Layer *l, float *x, int S, float *out) {* (3) compute. *//* shared experts for all S positions: gamma inside (before down_proj is* linear, so applied at the end). Factored out so the Metal path can run it- * on the CPU while the last routed-expert round is in flight on the GPU. */+ * on the CPU while the last routed-expert round is in flight on the GPU.+ *+ * Prefill batches positions so each shared matrix is traversed once per chunk,+ * rather than once per token. Decode (S=1) deliberately stays on the original+ * scalar path: its scratch remains tiny and the compiler sees a one-row GEMV.+ * Bound the batch scratch to 64 MiB so a long prompt cannot turn this speedup+ * into a memory spike. INK_SHARED_BATCH=0 restores the scalar path; a positive+ * value sets a smaller maximum row count for deterministic A/B tests. */+static int shared_batch_rows(int S, int D, int I) {+ if (S <= 1) return 1;+ const char *env = getenv("INK_SHARED_BATCH");+ if (env) {+ int requested = atoi(env);+ if (requested <= 0) return 1;+ if (requested < S) S = requested;+ }+ int64_t row_bytes = ((int64_t)2*I + D) * (int64_t)sizeof(float);+ int64_t bounded = (64LL << 20) / (row_bytes > 0 ? row_bytes : 1);+ if (bounded < 1) bounded = 1;+ if (S > bounded) S = (int)bounded;+ return S;+}static void shared_experts_cpu(Model *m, Layer *l, const float *x, int S,float *out, const float *wgt,float *g, float *u, float *hh) {Cfg *c = &m->c;int D = c->hidden, K = c->topk, I = c->moe_inter, ns = c->n_shared;- for (int s = 0; s < S; s++) {- const float *xs = x + (int64_t)s*D;- float *os = out + (int64_t)s*D;- const float *w = wgt + (int64_t)s*(K+ns);+ if (ns <= 0 || S <= 0) return;+ int B = shared_batch_rows(S, D, I);+ if (B == 1) {+ for (int s = 0; s < S; s++) {+ const float *xs = x + (int64_t)s*D;+ float *os = out + (int64_t)s*D;+ const float *w = wgt + (int64_t)s*(K+ns);+ for (int j = 0; j < ns; j++) {+ matmul_w(g, xs, wt_off_i(l->sh_g, (int64_t)j*I*D, D), 1, D, I);+ matmul_w(u, xs, wt_off_i(l->sh_u, (int64_t)j*I*D, D), 1, D, I);+ for (int i = 0; i < I; i++) g[i] = siluf(g[i]) * u[i];+ matmul_w(hh, g, wt_off_i(l->sh_d, (int64_t)j*D*I, I), 1, I, D);+ for (int d = 0; d < D; d++) os[d] += w[K+j] * hh[d];+ }+ }+ return;+ }+ float *bg = falloc((int64_t)2*B*I), *bu = bg + (int64_t)B*I;+ float *bh = falloc((int64_t)B*D);+ for (int base = 0; base < S; base += B) {+ int rows = S - base < B ? S - base : B;for (int j = 0; j < ns; j++) {- matmul_w(g, xs, wt_off_i(l->sh_g, (int64_t)j*I*D, D), 1, D, I);- matmul_w(u, xs, wt_off_i(l->sh_u, (int64_t)j*I*D, D), 1, D, I);- for (int i = 0; i < I; i++) g[i] = siluf(g[i]) * u[i];- matmul_w(hh, g, wt_off_i(l->sh_d, (int64_t)j*D*I, I), 1, I, D);- for (int d = 0; d < D; d++) os[d] += w[K+j] * hh[d];+ matmul_w(bg, x + (int64_t)base*D,+ wt_off_i(l->sh_g, (int64_t)j*I*D, D), rows, D, I);+ matmul_w(bu, x + (int64_t)base*D,+ wt_off_i(l->sh_u, (int64_t)j*I*D, D), rows, D, I);+ for (int64_t q = 0; q < (int64_t)rows*I; q++) bg[q] = siluf(bg[q]) * bu[q];+ matmul_w(bh, bg, wt_off_i(l->sh_d, (int64_t)j*D*I, I), rows, I, D);+ for (int s = 0; s < rows; s++) {+ float *os = out + (int64_t)(base+s)*D;+ const float *w = wgt + (int64_t)(base+s)*(K+ns);+ const float *hs = bh + (int64_t)s*D;+ for (int d = 0; d < D; d++) os[d] += w[K+j] * hs[d];+ }}}+ free(bg); free(bh);}static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {diff --git a/c/tests/bench_inkling_shared_batch.c b/c/tests/bench_inkling_shared_batch.cnew file mode 100644index 00000000..e7a22a8b--- /dev/null+++ b/c/tests/bench_inkling_shared_batch.c@@ -0,0 +1,118 @@+/* Optional A/B for Inkling shared-expert prefill batching. The dimensions+ * make the bf16 shared weights larger than the LLC of an ordinary desktop,+ * while keeping the whole process comfortably below 100 MiB. Correctness is+ * a bit-exact precondition; timing is reported but never used as a CI gate.+ *+ * make tests/bench_inkling_shared_batch ARCH=native+ * OMP_NUM_THREADS=<physical cores> ./tests/bench_inkling_shared_batch+ */+#define COLI_INKLING_SHARED_BATCH_TEST 1+#define main inkling_main_unused+#include "../inkling.c"+#undef main++enum { S = 16, D = 2048, I = 1024, K = 2, NS = 2, REPS = 3 };++static uint16_t to_bf16(float x) {+ union { float f; uint32_t u; } v = { x };+ return (uint16_t)(v.u >> 16);+}++static float val(int64_t i, int salt) {+ int v = (int)((i * 29 + salt * 43) % 127);+ return (float)(v - 63) / (float)(256 + salt);+}++static void fill_f32(float *p, int64_t n, int salt) {+ for (int64_t i = 0; i < n; i++) p[i] = val(i, salt);+}++static void fill_bf16(uint16_t *p, int64_t n, int salt) {+ for (int64_t i = 0; i < n; i++) p[i] = to_bf16(val(i, salt));+}++static void init_q4(Wt *w, int rows, int cols, int salt) {+ const int gs=64, rb=(cols+1)/2, ng=(cols+gs-1)/gs;+ w->qbits=4; w->gs=gs; w->qn=(int64_t)rows*rb;+ w->q4=malloc((size_t)w->qn); w->qs=malloc((size_t)rows*ng*sizeof(float));+ if(!w->q4||!w->qs){fputs("OOM q4 weights\n",stderr);exit(2);}+ for(int64_t i=0;i<w->qn;i++){+ int lo=(int)((i*5+salt)%16),hi=(int)((i*11+salt*3)%16);+ w->q4[i]=(uint8_t)(lo|(hi<<4));+ }+ for(int64_t i=0;i<(int64_t)rows*ng;i++)w->qs[i]=0.01f*(float)(1+(i+salt)%7);+}++static double one(Model *m, Layer *l, const float *x, const float *wgt,+ const float *seed, float *out, const char *mode) {+ float *g = falloc(2 * I), *u = g + I, *hh = falloc(D);+ if (mode) setenv("INK_SHARED_BATCH", mode, 1);+ else unsetenv("INK_SHARED_BATCH");+ memcpy(out, seed, (size_t)S * D * sizeof(float));+ double t0 = now_s();+ shared_experts_cpu(m, l, x, S, out, wgt, g, u, hh);+ double dt = now_s() - t0;+ free(g); free(hh);+ return dt;+}++static int measure(const char *format, double weight_mib, Model *m, Layer *l,+ const float *x, const float *wgt, const float *seed,+ float *scalar, float *batch) {+ one(m,l,x,wgt,seed,scalar,"0"); one(m,l,x,wgt,seed,batch,NULL);+ if(memcmp(scalar,batch,(size_t)S*D*sizeof(float))){+ int shown=0;float worst=0.f;+ for(int q=0;q<S*D;q++)if(scalar[q]!=batch[q]){+ float d=fabsf(scalar[q]-batch[q]);if(d>worst)worst=d;+ if(shown++<4)fprintf(stderr,"%s diff[%d] scalar=%a batch=%a delta=%g\n",+ format,q,scalar[q],batch[q],d);+ }+ fprintf(stderr,"FAIL %s: %d values differ, worst=%g\n",format,shown,worst);return 1;+ }+ double ts=0.0,tb=0.0;+ for(int r=0;r<REPS;r++){+ if(r&1){tb+=one(m,l,x,wgt,seed,batch,NULL);ts+=one(m,l,x,wgt,seed,scalar,"0");}+ else{ts+=one(m,l,x,wgt,seed,scalar,"0");tb+=one(m,l,x,wgt,seed,batch,NULL);}+ }+ ts/=REPS;tb/=REPS;+ printf("inkling shared %s: S=%d D=%d I=%d shared=%d bytes=%.1f MiB\n",+ format,S,D,I,NS,weight_mib);+ printf("scalar %.6f s batch %.6f s speedup %.2fx calls %d -> %d\n",+ ts,tb,ts/tb,S*NS*3,NS*3);+ return 0;+}++int main(void) {+ Model m; memset(&m, 0, sizeof(m));+ m.c.hidden = D; m.c.moe_inter = I; m.c.topk = K; m.c.n_shared = NS;+ Layer l; memset(&l, 0, sizeof(l));+ l.sh_g.h = malloc((size_t)NS * I * D * sizeof(uint16_t));+ l.sh_u.h = malloc((size_t)NS * I * D * sizeof(uint16_t));+ l.sh_d.h = malloc((size_t)NS * D * I * sizeof(uint16_t));+ float *x = falloc((int64_t)S * D);+ float *wgt = falloc((int64_t)S * (K + NS));+ float *seed = falloc((int64_t)S * D);+ float *scalar = falloc((int64_t)S * D), *batch = falloc((int64_t)S * D);+ if (!l.sh_g.h || !l.sh_u.h || !l.sh_d.h) { fputs("OOM weights\n", stderr); return 2; }+ fill_bf16(l.sh_g.h, (int64_t)NS * I * D, 1);+ fill_bf16(l.sh_u.h, (int64_t)NS * I * D, 2);+ fill_bf16(l.sh_d.h, (int64_t)NS * D * I, 3);+ fill_f32(x, (int64_t)S * D, 4);+ fill_f32(wgt, (int64_t)S * (K + NS), 5);+ fill_f32(seed, (int64_t)S * D, 6);++ int failed=measure("bf16",3.0*NS*D*I*sizeof(uint16_t)/1048576.0,+ &m,&l,x,wgt,seed,scalar,batch);+ free(l.sh_g.h);free(l.sh_u.h);free(l.sh_d.h);+ memset(&l,0,sizeof(l));+ init_q4(&l.sh_g,NS*I,D,1);init_q4(&l.sh_u,NS*I,D,2);init_q4(&l.sh_d,NS*D,I,3);+ double qbytes=(double)(l.sh_g.qn+l.sh_u.qn+l.sh_d.qn);+ qbytes+=(double)(NS*I*((D+63)/64)*2+NS*D*((I+63)/64))*sizeof(float);+ failed|=measure("int4-g64",qbytes/1048576.0,&m,&l,x,wgt,seed,scalar,batch);++ unsetenv("INK_SHARED_BATCH");+ free(l.sh_g.q4);free(l.sh_g.qs);free(l.sh_u.q4);free(l.sh_u.qs);+ free(l.sh_d.q4);free(l.sh_d.qs);+ free(x); free(wgt); free(seed); free(scalar); free(batch);+ return failed;+}diff --git a/c/tests/test_inkling_shared_batch.c b/c/tests/test_inkling_shared_batch.cnew file mode 100644index 00000000..505e2af2--- /dev/null+++ b/c/tests/test_inkling_shared_batch.c@@ -0,0 +1,106 @@+/* Model-free oracle for Inkling shared-expert prefill batching. It drives the+ * production helper in all resident formats (f32 oracle, bf16 checkpoint,+ * int4-g64 sidecar), compares against the old scalar path bit for bit, and+ * proves that the number of weight traversals is per chunk rather than per+ * prompt position. */+#define COLI_INKLING_SHARED_BATCH_TEST 1+#define main inkling_main_unused+#include "../inkling.c"+#undef main++static int failures;+#define CHECK(cond, ...) do { if (!(cond)) { \+ fprintf(stderr,"FAIL %s:%d: ",__FILE__,__LINE__); \+ fprintf(stderr,__VA_ARGS__); fputc('\n',stderr); failures++; } } while (0)++static float value(int64_t i, int salt) {+ int v = (int)((i * 37 + salt * 17) % 101);+ return (float)(v - 50) / (float)(71 + salt);+}+static void fill(float *p, int64_t n, int salt) {+ for (int64_t i=0;i<n;i++) p[i]=value(i,salt);+}+static uint16_t bf16(float x) {+ union { float f; uint32_t u; } v={x}; return (uint16_t)(v.u>>16);+}+static void fill_h(uint16_t *p, int64_t n, int salt) {+ for (int64_t i=0;i<n;i++) p[i]=bf16(value(i,salt));+}+static void init_q4(Wt *w, int rows, int cols, int salt) {+ int rb=(cols+1)/2,ng=(cols+63)/64;+ w->qbits=4;w->gs=64;w->qn=(int64_t)rows*rb;+ w->q4=malloc((size_t)w->qn);w->qs=malloc((size_t)rows*ng*sizeof(float));+ CHECK(w->q4&&w->qs,"q4 allocation failed");if(!w->q4||!w->qs)exit(2);+ for(int64_t i=0;i<w->qn;i++){+ int lo=(int)((i*5+salt)%16),hi=(int)((i*11+salt*3)%16);+ w->q4[i]=(uint8_t)(lo|(hi<<4));+ }+ for(int64_t i=0;i<(int64_t)rows*ng;i++)w->qs[i]=0.01f*(float)(1+(i+salt)%7);+}+static void free_w(Wt *w){free(w->f);free(w->h);free(w->q4);free(w->qs);}++static void compare_paths(const char *format,Model *m,Layer *l,int S,+ const float *x,const float *wgt,const float *seed){+ Cfg *c=&m->c;int D=c->hidden,I=c->moe_inter,ns=c->n_shared;+ size_t out_bytes=(size_t)S*D*sizeof(float);+ float *scalar=falloc((int64_t)S*D),*out=falloc((int64_t)S*D);+ float *g=falloc((int64_t)2*I),*u=g+I,*hh=falloc(D);+ memcpy(scalar,seed,out_bytes);setenv("INK_SHARED_BATCH","0",1);+ g_matmul_w_calls=0;shared_experts_cpu(m,l,x,S,scalar,wgt,g,u,hh);+ CHECK(g_matmul_w_calls==(uint64_t)S*ns*3,+ "%s scalar calls=%llu expected=%d",format,+ (unsigned long long)g_matmul_w_calls,S*ns*3);++ const char *modes[]={NULL,"3"};+ for(int z=0;z<2;z++){+ memcpy(out,seed,out_bytes);+ if(modes[z])setenv("INK_SHARED_BATCH",modes[z],1);else unsetenv("INK_SHARED_BATCH");+ g_matmul_w_calls=0;shared_experts_cpu(m,l,x,S,out,wgt,g,u,hh);+ int chunks=modes[z]?(S+2)/3:1,expected=chunks*ns*3;+ CHECK(!memcmp(out,scalar,out_bytes),"%s mode=%s is not scalar bit-exact",+ format,modes[z]?modes[z]:"default");+ CHECK(g_matmul_w_calls==(uint64_t)expected,+ "%s mode=%s calls=%llu expected=%d",format,modes[z]?modes[z]:"default",+ (unsigned long long)g_matmul_w_calls,expected);+ printf("inkling shared: format=%s S=%d mode=%s calls=%d -> %llu\n",+ format,S,modes[z]?modes[z]:"default",S*ns*3,+ (unsigned long long)g_matmul_w_calls);+ }++ /* Decode must remain the original one-row GEMV path. */+ memcpy(out,seed,(size_t)D*sizeof(float));unsetenv("INK_SHARED_BATCH");+ g_matmul_w_calls=0;shared_experts_cpu(m,l,x,1,out,wgt,g,u,hh);+ CHECK(!memcmp(out,scalar,(size_t)D*sizeof(float)),"%s decode row changed",format);+ CHECK(g_matmul_w_calls==(uint64_t)ns*3,"%s decode used batch shape",format);+ free(g);free(hh);free(scalar);free(out);+}++static void run_format(const char *format,int kind,int S,int D,int I){+ enum{K=2,NS=2};Model m;memset(&m,0,sizeof(m));+ m.c.hidden=D;m.c.moe_inter=I;m.c.topk=K;m.c.n_shared=NS;+ Layer l;memset(&l,0,sizeof(l));int64_t gi=(int64_t)NS*I*D,di=(int64_t)NS*D*I;+ if(kind==0){+ l.sh_g.f=falloc(gi);l.sh_u.f=falloc(gi);l.sh_d.f=falloc(di);+ fill(l.sh_g.f,gi,1);fill(l.sh_u.f,gi,2);fill(l.sh_d.f,di,3);+ }else if(kind==1){+ l.sh_g.h=malloc((size_t)gi*2);l.sh_u.h=malloc((size_t)gi*2);l.sh_d.h=malloc((size_t)di*2);+ CHECK(l.sh_g.h&&l.sh_u.h&&l.sh_d.h,"bf16 allocation failed");+ if(!l.sh_g.h||!l.sh_u.h||!l.sh_d.h)exit(2);+ fill_h(l.sh_g.h,gi,1);fill_h(l.sh_u.h,gi,2);fill_h(l.sh_d.h,di,3);+ }else{+ init_q4(&l.sh_g,NS*I,D,1);init_q4(&l.sh_u,NS*I,D,2);init_q4(&l.sh_d,NS*D,I,3);+ }+ float *x=falloc((int64_t)S*D),*w=falloc((int64_t)S*(K+NS)),*seed=falloc((int64_t)S*D);+ fill(x,(int64_t)S*D,4);fill(w,(int64_t)S*(K+NS),5);fill(seed,(int64_t)S*D,6);+ compare_paths(format,&m,&l,S,x,w,seed);+ free_w(&l.sh_g);free_w(&l.sh_u);free_w(&l.sh_d);free(x);free(w);free(seed);+}++int main(void){+ run_format("f32",0,12,17,13);+ run_format("bf16",1,12,32,16);+ run_format("int4-g64",2,12,64,64);+ unsetenv("INK_SHARED_BATCH");+ if(failures){fprintf(stderr,"inkling shared batch: %d failure(s)\n",failures);return 1;}+ puts("inkling shared batch: ok");return 0;+}diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex bc45e51d..747099a3 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -353,6 +353,7 @@ Read **only** by `c/inkling.c`.| `PIN_N` | `cap / 2` | Experts pinned per layer. Measured on the 975B: `cap/4` (19/layer) gave 83.6% hit / 0.32 tok/s, 40/layer gave 95.6% / 0.80 tok/s — decode fills run at queue depth ~1, so every pinned expert removes a ~35 ms stall. Clamped to `cap - 8`. || `REP_PEN` | `1.1` | Repetition penalty over a 128-token history (prompt tail + emitted). || `INK_DENSE_Q4` | auto | Use the `dense-int4g64/` sidecar for dense weights when that directory exists. `=0` forces the unquantized dense path. |+| `INK_SHARED_BATCH` | auto | Prefill rows per shared-expert batch, bounded to 64 MiB of scratch. `=0` restores the scalar per-token path for A/B/debugging; a positive value caps the chunk size. Decode (`S=1`) is unchanged. || `INK_METAL_MIN_S` | `1` | Minimum batch S to send the MoE block to Metal. `=2` restores the prefill-only gate (which mattered when the residency set was absent and per-block `useResource` churn cost ~135 ms). || `INK_PREFIX_LOG` | unset | Log the KV-prefix reuse decision and its reason, as `K3_PREFIX_LOG` does for K3. || `GPU_DEV` | `0` | CUDA device index for the inkling CUDA backend. |diff --git a/docs/inkling.md b/docs/inkling.mdindex d5fae47b..3ecedaaa 100644--- a/docs/inkling.md+++ b/docs/inkling.md@@ -191,6 +191,7 @@ SNAP=~/Models/inkling_i4 ./c/inkling -f warmup_prompts.txt -n 32| `USAGE_SAVE=0` | don't rewrite the history (benchmark runs) || `NOGPU=1` / `GPU_DEV=<n>` | disable CUDA / select device || `IDOT=0` | byte-exact scalar int kernels (debugging) |+| `INK_SHARED_BATCH=0` | disable shared-expert prefill batching for a controlled A/B; positive values cap rows per chunk. The default uses up to 64 MiB of bounded scratch and never changes decode (`S=1`) || `TOPP=<p>` | adaptive routing: keep routed experts up to cumulative weight `p`, drop the tail. **Trims the routing** — fewer experts read per token (the lever that matters on a disk-bound host), but a different computation from the declared top-k. Off by default; the run reports `[topp] … N/M routed used (X% trimmed)` so the trade is measurable. Same semantics as `TOPP` in `colibri.c` and `K3_TOPP` in `kimi_k3.c` || First positional arg | expert-cache cap per layer (`0` = auto-size from free RAM) |