Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
inkling: the routed experts on the Vulkan expert tier
JustVugge98ac550b9bd
diff --git a/c/inkling.c b/c/inkling.cindex 4a3eb8af..6f1d33a2 100644--- a/c/inkling.c+++ b/c/inkling.c@@ -72,10 +72,13 @@ static int g_metal = 0;/* Vulkan (opt-in, VK=1 build + COLI_VULKAN=1): the RESIDENT matrices, in the form* they are kept in RAM, uploaded on first use -- the dense-int4g64 container's int8* (fmt 1) and int4-g64 (fmt 4), and the f32 (fmt 10) and bf16 (fmt 11) residents of- * an unconverted snapshot. Routed experts, which arrive from disk, and the embedding- * row lookups stay on the CPU; CUDA keeps its own bf16 residents. */+ * an unconverted snapshot -- and the routed experts through the shared expert tier+ * (vk_tier.h, ink_vk_tier_start). The embedding row lookups stay on the CPU; CUDA+ * keeps its own bf16 residents, and with CUDA (or Metal) on the tier stays off. */#include "backend_vulkan.h"+#include "vk_tier.h"static int g_vk_ready = 0;+static int g_vk_tier_try = 0; /* the device opened with the expert tier to be tried */#endif#if defined(__APPLE__)#include <mach/mach.h>@@ -994,9 +997,7 @@ static void ink_vk_mark(Wt *w, int views, int *by, int *cpu) {int fmt = ink_vk_attach(w, views);if (fmt) by[fmt]++; else (*cpu)++;}-static void ink_vk_init_model(Model *m, int layer_begin, int layer_end) {- if (!g_vk_ready) g_vk_ready = coli_vk_init_env("inkling");- if (!g_vk_ready || !coli_vk_dense()) return; /* COLI_VK_DENSE=0: nothing marked, the CPU computes */+static void ink_vk_mark_dense(Model *m, int layer_begin, int layer_end) {int by[13] = {0}, cpu = 0;ink_vk_mark(&m->lm_head, 1, by, &cpu);for (int i = layer_begin; i < layer_end; i++) {@@ -1009,8 +1010,30 @@ static void ink_vk_init_model(Model *m, int layer_begin, int layer_end) {}fprintf(stderr, "[VK] inkling: %d resident matrices go to the GPU on first use ""(%d int8, %d int4-g64, %d f32, %d bf16), %d stay on the CPU; "- "routed experts and embedding lookups stay on the CPU\n",- by[1] + by[4] + by[10] + by[11], by[1], by[4], by[10], by[11], cpu);+ "%s and embedding lookups stay on the CPU\n",+ by[1] + by[4] + by[10] + by[11], by[1], by[4], by[10], by[11], cpu,+ g_vk_tier_try ? "routed experts go to the expert tier (its own line)" : "routed experts");+}+/* Opens the device saying whether the routed-expert tier will be tried (vk_tier.h+ * step 0): a whole model with routed experts, and neither CUDA nor Metal on, either+ * of which wins. The dense matrices then go where coli_vk_dense() says: on a device+ * sharing the CPU's RAM with the tier on, they stay on the CPU unless COLI_VK_DENSE=1;+ * if the tier does not start after all, ink_vk_tier_start marks them then. */+static void ink_vk_init_model(Model *m, int layer_begin, int layer_end) {+ int sparse = 0, other_gpu = 0;+ for (int i = layer_begin; i < layer_end; i++) sparse += m->c.sparse[i];+#ifdef COLI_CUDA+ other_gpu |= g_cuda;+#endif+#ifdef COLI_METAL+ other_gpu |= g_metal;+#endif+ g_vk_tier_try = vkt_wanted() && m->c.n_experts > 0 && sparse > 0 && !other_gpu &&+ layer_begin == 0 && layer_end == m->c.n_layers;+ if (!g_vk_ready) g_vk_ready = coli_vk_init_env_tier("inkling", g_vk_tier_try);+ if (!g_vk_ready) { g_vk_tier_try = 0; return; }+ if (!coli_vk_dense()) return; /* nothing marked: the CPU computes the dense matrices */+ ink_vk_mark_dense(m, layer_begin, layer_end);}#endif@@ -1620,6 +1643,263 @@ static void shared_experts_cpu(Model *m, Layer *l, const float *x, int S,free(bg); free(bh);}+#ifdef COLI_VULKAN+/* ---- the Vulkan routed-expert tier (vk_tier.c) ---------------------------------+ * With the tier on, one MoE layer runs per block of rows: the block's resident+ * experts go to the device as one batch (vkt_issue), the CPU computes the other+ * (row, rank) pairs into rows of their own -- the same cache rounds and kernels as+ * moe() -- and the shared experts into a buffer of their own while the batch runs.+ * Then every rank of every row joins `out` in rank order, the device's and the CPU's+ * alike, and the shared experts after them, as moe() adds them; so the order of the+ * sum never depends on which experts were resident. Every expert the CPU computed+ * passes its RAM bytes to the tier (vkt_note), which may promote it. */+#define INK_VK_ROWS 64+/* How the slots hold an expert: the container's packed int4 or int8 rows with f32+ * row scales (gate and up in one fused [2I, D] tensor: up starts I rows in), the+ * runtime-quantized int8 rows (int4 values when bits <= 4), or f32 at bits=0.+ * moe() reads down with gate/up's kernel, so the two share one kind. 0: none. */+static int ink_vk_fmt_of(const Model *m, VktFmt *f) {+ int D = m->c.hidden, I = m->c.moe_inter;+ if (m->xq) {+ int q4 = m->rb13 * 2 == D;+ if (q4 ? m->rb2 * 2 != I : m->rb2 != I) return 0;+ *f = (VktFmt){q4 ? VKT_SRC_I4U_PAIRS_ROW : VKT_SRC_I8_ROW, 0};+ } else if (m->quant_bits) *f = (VktFmt){m->quant_bits <= 4 ? VKT_SRC_I8_AS_I4_ROW : VKT_SRC_I8_ROW, 0};+ else *f = (VktFmt){VKT_SRC_F32, 0};+ return 1;+}+static VktExpertSrc ink_vk_slot_src(const Model *m, const Slot *e) {+ int64_t D = m->c.hidden, I = m->c.moe_inter;+ if (m->xq) return (VktExpertSrc){e->p13, e->p13 + I * m->rb13, e->p2, e->s13, e->s13 + I, e->s2};+ if (m->quant_bits) return (VktExpertSrc){e->q13, e->q13 + I * D, e->q2, e->s13, e->s13 + I, e->s2};+ return (VktExpertSrc){e->f13, e->f13 + I * D, e->f2, NULL, NULL, NULL};+}+/* Is the expert in this layer's RAM cache now (the tier's balance asks)? */+static int ink_vk_in_ram(void *ctx, int layer, int e) {+ Slot *s = slot_indexed((Model *)ctx, layer, e);+ return s && s->filled;+}+/* Bytes the dense matrices marked for the device will take there (the budget leaves+ * them room), from the geometry: int8 1 byte, int4-g64 0.5625, f32 4, bf16 2. */+static size_t ink_vk_dense_bytes(const Model *m) {+ const Cfg *c = &m->c;+ double b = 0;+ #define INK_VKB(w, n) do { const Wt *w_ = (w); int f_ = w_->vk ? ink_vk_fmt(w_) : 0; \+ b += (double)(n) * (f_ == 1 ? 1.0 : f_ == 4 ? 0.5625 : f_ == 10 ? 4.0 : f_ == 11 ? 2.0 : 0.0); } while (0)+ INK_VKB(&m->lm_head, (double)c->vocab * c->hidden);+ for (int i = 0; i < c->n_layers; i++) {+ const Layer *l = &m->L[i];+ double D = c->hidden, q = (double)L_HEADS(c, i) * L_HD(c, i), kv = (double)L_KV(c, i) * L_HD(c, i);+ INK_VKB(&l->q, D * q); INK_VKB(&l->k, D * kv); INK_VKB(&l->v, D * kv);+ INK_VKB(&l->r, D * L_HEADS(c, i) * c->d_rel); INK_VKB(&l->o, q * D);+ INK_VKB(&l->dg, D * c->dense_inter); INK_VKB(&l->du, D * c->dense_inter); INK_VKB(&l->dd, D * c->dense_inter);+ double sh = (double)c->n_shared * c->moe_inter * D;+ INK_VKB(&l->sh_g, sh); INK_VKB(&l->sh_u, sh); INK_VKB(&l->sh_d, sh);+ }+ #undef INK_VKB+ return (size_t)b;+}+/* A slot of its own for the warm start's reads (slot_fill's buffers), and its free. */+static int ink_vk_tmp_slot(const Model *m, Slot *s, int eid) {+ int64_t D = m->c.hidden, I = m->c.moe_inter;+ memset(s, 0, sizeof *s);+ s->eid = eid;+ if (m->xq) { s->p13 = malloc((size_t)(m->rb13 * 2 * I)); s->p2 = malloc((size_t)(m->rb2 * D)); }+ else if (m->quant_bits) { s->q13 = malloc((size_t)(2 * I * D)); s->q2 = malloc((size_t)(D * I)); }+ else { s->f13 = malloc((size_t)(2 * I * D) * sizeof(float)); s->f2 = malloc((size_t)(D * I) * sizeof(float)); }+ s->s13 = malloc((size_t)(2 * I + D) * sizeof(float));+ if (s->s13) s->s2 = s->s13 + 2 * I;+ return s->s13 && (m->xq ? s->p13 && s->p2 : m->quant_bits ? s->q13 && s->q2 : s->f13 && s->f2);+}+static void ink_vk_tmp_free(Slot *s) {+ free(s->p13); free(s->p2); free(s->q13); free(s->q2); free(s->f13); free(s->f2); free(s->s13);+}+/* COLI_VULKAN=1: describe the experts to the tier, after the weights and the history+ * (pins_load reads it in the generate and serve modes; the oracle reads none), and+ * warm it from that history: the hottest experts first, read from disk in parallel. */+static void ink_vk_tier_start(Model *m) {+ Cfg *c = &m->c;+ if (!g_vk_ready) return;+#ifdef COLI_CUDA+ if (g_cuda) {+ if (vkt_wanted()) fprintf(stderr, "[VK] tier inkling: the CUDA backend is on and wins; the Vulkan tier stays off\n");+ return;+ }+#endif+#ifdef COLI_METAL+ if (g_metal) {+ if (vkt_wanted()) fprintf(stderr, "[VK] tier inkling: the Metal expert path is on and wins; the Vulkan tier stays off\n");+ return;+ }+#endif+ VktFmt f;+ if (g_vk_tier_try && !ink_vk_fmt_of(m, &f))+ fprintf(stderr, "[VK] tier inkling: container rows of %lld/%lld bytes are no expert format the tier reads; the experts stay on the CPU\n",+ (long long)m->rb13, (long long)m->rb2);+ else if (g_vk_tier_try) {+ int nsp = 0;+ for (int i = 0; i < c->n_layers; i++) nsp += c->sparse[i];+ int64_t D = c->hidden, I = c->moe_inter;+ int64_t slotb = m->xq ? m->rb13*2*I + m->rb2*D + (2*I+D)*4+ : m->quant_bits ? 3*I*D + (2*I+D)*4 : 3*I*D*4;+ VktConfig vc = {.engine = "inkling", .layers = c->n_layers, .experts = c->n_experts,+ .hidden = c->hidden, .inter = c->moe_inter, .topk = c->topk,+ .gate_up = f, .down = f, .act = VKT_ACT_SWIGLU,+ .max_rows = INK_VK_ROWS * c->topk,+ .ram_reserve = (size_t)slotb * (size_t)m->cache[0].cap * (size_t)nsp,+ .dense_bytes = ink_vk_dense_bytes(m),+ .in_ram = ink_vk_in_ram, .ram_ctx = m};+ if (vkt_init(&vc, m->eusage)) {+ atexit(coli_vk_shutdown); /* runs after the tier's teardown: the device goes before the drivers unload */+ atexit(vkt_shutdown);+ int all = c->n_layers * c->n_experts;+ int *pl = malloc((size_t)all * sizeof(int)), *pe = malloc((size_t)all * sizeof(int));+ const char *warm = getenv("COLI_VK_TIER_WARM"); /* 0: no warm start, the tier fills as experts pass by */+ int n = pl && pe && !(warm && *warm == '0') ? vkt_plan(pl, pe, all) : 0;+ if (n > 0) {+ double t0 = now_s();+ #pragma omp parallel for schedule(dynamic, 4)+ for (int i = 0; i < n; i++) {+ Slot tmp;+ if (ink_vk_tmp_slot(m, &tmp, pe[i])) {+ slot_fill(m, pl[i], &tmp);+ VktExpertSrc src = ink_vk_slot_src(m, &tmp);+ vkt_put(pl[i], pe[i], &src);+ } else vkt_put(pl[i], pe[i], NULL); /* no memory: the planned slot is released */+ ink_vk_tmp_free(&tmp);+ }+ vkt_put_done();+ fprintf(stderr, "[VK] tier inkling: warm start, %d experts from the history in %.1fs\n", n, now_s() - t0);+ }+ free(pl); free(pe);+ }+ }+ if (!vkt_ready() && !coli_vk_dense() && coli_vk_dense_decide("inkling", 0, 1)) { /* no tier after all */+ g_vk_tier_try = 0;+ ink_vk_mark_dense(m, 0, c->n_layers);+ }+ g_vk_tier_try = vkt_ready();+}+static void ink_vk_tier_report(const Model *m, const char *scope) {+ vkt_report(scope, m->hits, m->miss); /* nothing when the tier never started */+}+/* The CPU's pairs of a block (want[i] set) into ctb[i]: moe()'s rounds of at most+ * `cap` pairs -- acquire, fill in parallel, check, compute -- with its kernels. The+ * pairs' history and HITS were counted by the caller. */+static void ink_vk_cpu_pairs(Model *m, int layer, const float *x, int n, const int *ib,+ const uint8_t *want, float *ctb, float *g) {+ Cfg *c = &m->c;+ int D = c->hidden, K = c->topk, I = c->moe_inter;+ float *u = g + I;+ int cap = m->cache[layer].cap; if (cap < 1) cap = 1;+ int q4 = m->xq && m->rb13*2 == D;+ Slot **use = malloc((size_t)cap * sizeof(Slot *)), **fill = malloc((size_t)cap * sizeof(Slot *));+ int *pi = malloc((size_t)cap * sizeof(int));+ if (!use || !fill || !pi) { fprintf(stderr, "OOM ink_vk_cpu_pairs\n"); exit(1); }+ for (int i = 0; i < n; ) {+ int nr = 0, nfill = 0;+ for (; i < n && nr < cap; i++) {+ if (!want[i] || ib[i] < 0) continue;+ Slot *e = slot_find(m, layer, ib[i]);+ if (e) m->hits++;+ else { m->miss++; e = slot_acquire(m, layer, ib[i]); fill[nfill++] = e; }+ use[nr] = e; pi[nr] = i; nr++;+ }+ if (nfill) {+ double tf = now_s();+ #pragma omp parallel for schedule(dynamic,1)+ for (int j = 0; j < nfill; j++) slot_fill(m, layer, fill[j]);+ m->t_fill += now_s() - tf;+ }+ for (int r = 0; r < nr; r++) {+ if (use[r]->eid != ib[pi[r]]) {+ fprintf(stderr, "layer %d: cache served expert %d for requested expert %d\n",+ layer, use[r]->eid, ib[pi[r]]);+ exit(1);+ }+ }+ for (int r = 0; r < nr; r++) {+ Slot *e = use[r];+ const float *xs = x + (int64_t)(pi[r] / K) * D;+ float *hh = ctb + (int64_t)pi[r] * D;+ if (m->xq) {+ if (q4) {+ matmul_q4(g, xs, e->p13, e->s13, D, 2*I); /* gate rows then up rows */+ for (int k = 0; k < I; k++) g[k] = siluf(g[k]) * u[k];+ matmul_q4(hh, g, e->p2, e->s2, I, D);+ } else {+ matmul_q(g, xs, (int8_t*)e->p13, e->s13, D, 2*I);+ for (int k = 0; k < I; k++) g[k] = siluf(g[k]) * u[k];+ matmul_q(hh, g, (int8_t*)e->p2, e->s2, I, D);+ }+ } else if (m->quant_bits) {+ matmul_q(g, xs, e->q13, e->s13, D, 2*I);+ for (int k = 0; k < I; k++) g[k] = siluf(g[k]) * u[k];+ matmul_q(hh, g, e->q2, e->s2, I, D);+ } else {+ matmul(g, xs, e->f13, 1, D, 2*I);+ for (int k = 0; k < I; k++) g[k] = siluf(g[k]) * u[k];+ matmul(hh, g, e->f2, 1, I, D);+ }+ VktExpertSrc vs = ink_vk_slot_src(m, e);+ vkt_note(layer, e->eid, &vs);+ }+ }+ free(use); free(fill); free(pi);+}+/* moe() with the tier on, after its routing pass: idx/keff/wgt as moe() made them. */+static void ink_vk_moe(Model *m, Layer *l, int layer, const float *x, int S, float *out,+ const int *idx, const int *keff, const float *wgt) {+ Cfg *c = &m->c;+ int D = c->hidden, K = c->topk, I = c->moe_inter, ns = c->n_shared;+ int B = S < INK_VK_ROWS ? S : INK_VK_ROWS;+ float *ctb = falloc((int64_t)B * K * D), *shb = ns > 0 ? falloc((int64_t)B * D) : NULL;+ float *g = falloc(2*I), *u = g + I, *hh = falloc(D);+ int *ib = malloc((size_t)B * K * sizeof(int));+ uint8_t *taken = malloc((size_t)B * K), *want = malloc((size_t)B * K);+ const float **dev = malloc((size_t)B * K * sizeof(*dev));+ if (!ib || !taken || !want || !dev) { fprintf(stderr, "OOM ink_vk_moe\n"); exit(1); }+ for (int s0 = 0; s0 < S; s0 += B) {+ int rows = S - s0 < B ? S - s0 : B, n = rows * K;+ const float *xb = x + (int64_t)s0 * D;+ double te = now_s(), f0 = m->t_fill, sh = 0;+ for (int i = 0; i < n; i++) { /* TOPP's trimmed ranks: -1, unused */+ int s = s0 + i / K, kk = i % K;+ ib[i] = kk < keff[s] ? idx[(int64_t)s * K + kk] : -1;+ if (ib[i] < 0) continue;+ if (m->eusage && m->eusage[layer]) m->eusage[layer][ib[i]]++;+ ehit_mark(m, layer, ib[i]);+ }+ int ndev = vkt_issue(layer, xb, rows, K, ib, taken);+ for (int i = 0; i < n; i++) want[i] = ib[i] >= 0 && !taken[i];+ ink_vk_cpu_pairs(m, layer, xb, n, ib, want, ctb, g);+ if (shb) {+ double ts = now_s();+ memset(shb, 0, (size_t)rows * D * sizeof(float));+ shared_experts_cpu(m, l, xb, rows, shb, wgt + (int64_t)s0 * (K + ns), g, u, hh);+ sh = now_s() - ts; m->t_shared += sh;+ }+ if (ndev && !vkt_join(dev)) { /* the batch failed (the tier stops): those pairs here */+ ink_vk_cpu_pairs(m, layer, xb, n, ib, taken, ctb, g);+ memset(taken, 0, (size_t)n);+ }+ for (int s = 0; s < rows; s++) {+ float *os = out + (int64_t)(s0 + s) * D;+ const float *w = wgt + (int64_t)(s0 + s) * (K + ns);+ for (int kk = 0; kk < K; kk++) {+ int i = s * K + kk;+ if (ib[i] < 0) continue;+ const float *cp = taken[i] ? dev[i] : ctb + (int64_t)i * D;+ for (int d = 0; d < D; d++) os[d] += w[kk] * cp[d];+ }+ if (shb) { const float *sp = shb + (int64_t)s * D; for (int d = 0; d < D; d++) os[d] += sp[d]; }+ }+ m->t_expert += now_s() - te - sh - (m->t_fill - f0);+ }+ free(ctb); free(shb); free(g); free(hh); free(ib); free(taken); free(want); free(dev);+}+#endif+static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {Cfg *c = &m->c;int D = c->hidden, E = c->n_experts, K = c->topk, I = c->moe_inter, ns = c->n_shared;@@ -1679,6 +1959,13 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {}m->euse += keff[s];}+#ifdef COLI_VULKAN+ if (vkt_ready()) { /* the Vulkan expert tier takes the layer from here (ink_vk_moe) */+ ink_vk_moe(m, l, layer, x, S, out, idx, keff, wgt);+ free(logits); free(idx); free(keff); free(wgt); free(use); free(fill); free(fl);+ return;+ }+#endif/* Il ciclo sotto ACQUISISCE uno slot per ogni coppia (token, esperto) e ne tiene* il puntatore fino al calcolo. slot_acquire evince l'LRU quando la cache e'* piena β anche uno slot gia' consegnato in QUESTA chiamata: il puntatore resta@@ -2284,7 +2571,7 @@ static void generate_stream(Model *m, Tok *T, const char *prompt, int n_new,m->t_fill, m->t_expert, m->t_shared, m->t_attn,wall - m->t_fill - m->t_expert - m->t_shared - m->t_attn);#ifdef COLI_VULKAN- fflush(stdout); ink_vk_report();+ fflush(stdout); ink_vk_report(); ink_vk_tier_report(m, "run");#endiffree(ids);}@@ -2617,7 +2904,7 @@ static int serve_one(Model *m, Tok *T, SReq *q) {m->t_fill - f0, m->t_shared - s0, m->t_expert - e0, m->t_attn - a0, 0.0, forwards);fflush(stdout);#ifdef COLI_VULKAN- ink_vk_report();+ ink_vk_report(); ink_vk_tier_report(m, "turn");#endifserve_hits(m);free(ids);@@ -2682,13 +2969,16 @@ static void serve_tiers_emap(Model *m) {int64_t slotb = m->xq ? m->rb13*2*I + m->rb2*D + (2*I+D)*4: m->quant_bits ? 3*I*D + (2*I+D)*4 : 3*I*D*4;printf("TIERS 0 %d %d 0.00 %.2f\n", filled, nsp*E - filled, filled*(double)slotb/1e9);- /* EMAP: 1 byte/expert hex β tier(2b: 0=disk 1=RAM)<<6 | heat(6b: log2 usage) */+ /* EMAP: 1 byte/expert hex β tier(2b: 0=disk 1=RAM 2=Vulkan device)<<6 | heat(6b: log2 usage) */char *hex = malloc((size_t)nsp*E*2 + 1); int w = 0;for (int i = 0; i < c->n_layers; i++) {if (!c->sparse[i]) continue;for (int e = 0; e < E; e++) {Slot *resident = slot_indexed(m, i, e);int tier = resident && resident->filled;+#ifdef COLI_VULKAN+ if (vkt_resident(i, e)) tier = 2; /* on the Vulkan device */+#endifuint32_t u = m->eusage[i] ? m->eusage[i][e] : 0;int heat = 0; while (u) { heat++; u >>= 1; } if (heat > 63) heat = 63;int b = (tier << 6) | heat;@@ -2847,6 +3137,9 @@ int main(int argc, char **argv) {if (getenv("SERVE") && getenv("SERVE")[0] == '1') {Model m; model_init(&m, snap, cap, bits);pins_load(&m, snap);+#ifdef COLI_VULKAN+ ink_vk_tier_start(&m); /* after the history: the warm start reads it */+#endifchar tkp[2048]; snprintf(tkp, sizeof(tkp), "%s/tokenizer.json", snap);Tok T; tok_load(&T, tkp);coli_rt_term_arm(); /* SIGTERM must reach the save below (#1629) */@@ -2860,6 +3153,9 @@ int main(int argc, char **argv) {printf("== Inkling C engine, %d layers, experts @ %s, cache %d/layer ==\n",m.c.n_layers, m.xq ? "container" : bits ? "int" : "f32", m.cache[0].cap);pins_load(&m, snap);+#ifdef COLI_VULKAN+ ink_vk_tier_start(&m);+#endifchar tkp[2048]; snprintf(tkp, sizeof(tkp), "%s/tokenizer.json", snap);Tok T; tok_load(&T, tkp);if (prompt) {@@ -2910,6 +3206,9 @@ int main(int argc, char **argv) {int ngen = nfull - np;Model m; model_init(&m, snap, cap, bits);+#ifdef COLI_VULKAN+ ink_vk_tier_start(&m); /* the oracle reads no history: the tier fills as experts pass by */+#endifprintf("== Inkling C engine (Stage A), cache = %d experts/layer, experts @ %s ==\n",cap, m.xq ? "container (int4/int8 + .qs)" : bits ? "int (runtime quant)" : "f32");printf("cfg: D=%d L=%d V=%d(%d) heads=%d/%d kv=%d/%d hd=%d win=%d d_rel=%d ext=%d E=%d+%d topk=%d\n",@@ -2960,7 +3259,7 @@ int main(int argc, char **argv) {printf("PEAK RSS: %.2f GB | expert cache hit %.1f%% | %.2f tok/s\n",rss_gb(), tot?100.0*m.hits/tot:0.0, ngen/dt);#ifdef COLI_VULKAN- fflush(stdout); ink_vk_report();+ fflush(stdout); ink_vk_report(); ink_vk_tier_report(&m, "run");#endiffree(buf); free(arena);return (match == ngen) ? 0 : 1;