Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

inkling: the dense chain on the device (COLI_VK_CHAIN=1)

JustVuggb686209c03ac
diff --git a/c/inkling.c b/c/inkling.c
index e92b58e5..81a9de31 100644
--- a/c/inkling.c
+++ b/c/inkling.c
@@ -79,6 +79,7 @@ static int g_metal = 0;
#include "vk_tier.h"
static int g_vk_ready = 0;
static int g_vk_tier_try = 0; /* the device opened with the expert tier to be tried */
+static int g_vk_chain = 0; /* COLI_VK_CHAIN decided on, and the chain's pipelines are up (inkling_chain.h) */
#endif
#if defined(__APPLE__)
#include <mach/mach.h>
@@ -188,6 +189,9 @@ typedef struct {
/* KV prefix reuse: what the current K/V and conv states were built from.
* See kv_prefix.h — recorded where the tokens are fed, never derived. */
kv_prefix kvp;
+#ifdef COLI_VULKAN
+ void *vkchain; /* the dense chain's device state (inkling_chain.h), NULL until it runs */
+#endif
} Model;
/* ---------- utility ---------- */
@@ -1847,13 +1851,14 @@ static void ink_vk_cpu_pairs(Model *m, int layer, const float *x, int n, const i
}
free(use); free(fill); free(pi);
}
-/* moe() with the tier on, after its routing pass: idx/keff/wgt as moe() made them. */
+/* moe() with the tier on, after its routing pass: idx/keff/wgt as moe() made them;
+ * routed_only leaves the shared experts out (the dense chain runs them). */
static void ink_vk_moe(Model *m, Layer *l, int layer, const float *x, int S, float *out,
- const int *idx, const int *keff, const float *wgt) {
+ const int *idx, const int *keff, const float *wgt, int routed_only) {
Cfg *c = &m->c;
int D = c->hidden, K = c->topk, I = c->moe_inter, ns = c->n_shared;
int B = S < INK_VK_ROWS ? S : INK_VK_ROWS;
- float *ctb = falloc((int64_t)B * K * D), *shb = ns > 0 ? falloc((int64_t)B * D) : NULL;
+ float *ctb = falloc((int64_t)B * K * D), *shb = ns > 0 && !routed_only ? falloc((int64_t)B * D) : NULL;
float *g = falloc(2*I), *u = g + I, *hh = falloc(D);
int *ib = malloc((size_t)B * K * sizeof(int));
uint8_t *taken = malloc((size_t)B * K), *want = malloc((size_t)B * K);
@@ -1900,12 +1905,17 @@ static void ink_vk_moe(Model *m, Layer *l, int layer, const float *x, int S, flo
}
#endif
-static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
+/* moe() with two doors for the dense chain (inkling_chain.h): lg_in, the router's
+ * logits [S][E+ns] the device computed (NULL: computed here); shw, when not NULL, asks
+ * for the routed experts only, and receives the shared experts' combine weights,
+ * shw[j*S + s] (the device runs the shared experts and adds them in moe()'s order).
+ * Under Metal the shared experts' GPU block ignores shw: the chain does not run there. */
+static void moe_ex(Model *m, Layer *l, int layer, float *x, int S, float *out, const float *lg_in, float *shw) {
Cfg *c = &m->c;
int D = c->hidden, E = c->n_experts, K = c->topk, I = c->moe_inter, ns = c->n_shared;
int ET = E + ns;
- float *logits = falloc((int64_t)S*ET);
- matmul(logits, x, l->router, S, D, ET);
+ float *logits = lg_in ? (float *)lg_in : falloc((int64_t)S*ET);
+ if (!lg_in) matmul(logits, x, l->router, S, D, ET);
memset(out, 0, (int64_t)S*D*sizeof(float));
int *idx = malloc((size_t)S*K*sizeof(int));
int *keff = malloc((size_t)S*sizeof(int)); /* routed effettivi per token (TOPP) */
@@ -1959,10 +1969,12 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
}
m->euse += keff[s];
}
+ if (shw) for (int s = 0; s < S; s++) for (int j = 0; j < ns; j++) shw[(int64_t)j*S + s] = wgt[(int64_t)s*(K+ns) + K + j];
#ifdef COLI_VULKAN
if (vkt_ready()) { /* the Vulkan expert tier takes the layer from here (ink_vk_moe) */
- ink_vk_moe(m, l, layer, x, S, out, idx, keff, wgt);
- free(logits); free(idx); free(keff); free(wgt); free(use); free(fill); free(fl);
+ ink_vk_moe(m, l, layer, x, S, out, idx, keff, wgt, shw != NULL);
+ if (!lg_in) free(logits);
+ free(idx); free(keff); free(wgt); free(use); free(fill); free(fl);
return;
}
#endif
@@ -2179,12 +2191,13 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
* Under Metal these run on the GPU as an fmt=5 block overlapped with the
* routed rounds (or on the CPU during the last GPU round); either path
* sets shared_done. */
- if (!shared_done) {
+ if (!shared_done && !shw) {
double ts = now_s();
shared_experts_cpu(m, l, x, S, out, wgt, g, u, hh);
m->t_shared += now_s() - ts;
}
- free(logits); free(idx); free(keff); free(wgt); free(use); free(fill); free(fl);
+ if (!lg_in) free(logits);
+ free(idx); free(keff); free(wgt); free(use); free(fill); free(fl);
free(g); free(hh); /* u aliases g+I */
#ifdef COLI_METAL
if (mxg) {
@@ -2195,6 +2208,9 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
free(sxg); free(srw);
#endif
}
+static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
+ moe_ex(m, l, layer, x, S, out, NULL, NULL);
+}
/* ---------- DMel audio embedding ----------
* One frame = mel_bins u8 levels in [0, mel_vocab). Its decoder embedding is
@@ -2246,6 +2262,10 @@ static void inkling_layers_forward_range(Model *m, float *x, int S, int pos0,
free(nrm); free(tmp);
}
+#ifdef COLI_VULKAN
+#include "inkling_chain.h" /* COLI_VK_CHAIN: every layer's dense chain on the device */
+#endif
+
/* ---------- one forward pass over S new tokens ----------
* Returns malloc'd logits of the last token (unpadded vocab). If tf_out is
* non-NULL also writes the per-position argmax (teacher-forcing check).
@@ -2331,6 +2351,9 @@ static void ink_pin_state_restore(Model *m, const InkPinState *st){
for (int i = 0; i < c->n_layers; i++)
memcpy(m->cs[j][i], st->cs[j][i],
(size_t)ink_cs_cells(c,j,i) * sizeof(float));
+#ifdef COLI_VULKAN
+ inkc_host_wrote(m, 0); /* the dense chain's copy goes up again before its next step */
+#endif
}
static float *step_mm(Model *m, const int *ids, int S, int pos0, int *tf_out,
@@ -2349,6 +2372,19 @@ static float *step_mm(Model *m, const int *ids, int S, int pos0, int *tf_out,
if (m->embed_norm) rmsnorm_row(x + (int64_t)s*D, x + (int64_t)s*D, m->embed_norm, D, c->eps);
}
free(arow);
+#ifdef COLI_VULKAN
+ /* COLI_VK_CHAIN: the layers, the final norm and lm_head on the device; x comes
+ * back only when the per-position heads below read every row */
+ float *chain_logit = NULL;
+ if (g_vk_chain) {
+ chain_logit = falloc(c->unpad_vocab);
+ if (!inkc_forward(m, x, S, pos0, (g_echo_k > 0 && g_echo_id && S > 1) || tf_out != NULL, chain_logit)) {
+ free(chain_logit); chain_logit = NULL;
+ inkc_cpu_step(m, pos0, S);
+ }
+ }
+ if (!chain_logit)
+#endif
inkling_layers_forward_range(m, x, S, pos0, 0, c->n_layers);
m->kv_len = pos0 + S;
/* record what was just fed, at the positions it went to (kv_prefix.h).
@@ -2405,6 +2441,9 @@ static float *step_mm(Model *m, const int *ids, int S, int pos0, int *tf_out,
}
free(normed); free(logits);
}
+#ifdef COLI_VULKAN
+ if (chain_logit) { free(x); free(last); return chain_logit; } /* the chain's last frame ran the head */
+#endif
rmsnorm_row(last, x + (int64_t)(S-1)*D, m->final_norm, D, c->eps);
for (int d = 0; d < D; d++) last[d] /= c->mup;
matmul_w(logit, last, m->lm_head, 1, D, c->unpad_vocab);
@@ -2425,6 +2464,9 @@ static void state_reset(Model *m) {
for (int j = 0; j < 4; j++)
memset(m->cs[j][i], 0, (int64_t)((j < 2) ? kvdim : c->hidden) * (c->conv_k-1) * sizeof(float));
}
+#ifdef COLI_VULKAN
+ inkc_host_wrote(m, 1); /* zeros: the dense chain fills its copy with zeros */
+#endif
}
static void kv_alloc(Model *m, int max_t) {
@@ -2497,10 +2539,15 @@ static void kv_alloc(Model *m, int max_t) {
static void generate(Model *m, const int *prompt, int np, int n_new, int *out,
const uint8_t *dmel, int naud) {
for (int i = 0; i < np; i++) out[i] = prompt[i];
+ /* DUMP=<path>: every forward's logits (unpadded vocab raw float32 each, in order),
+ * for a comparison of two runs (tests/vulkan_engines.sh compares the dense chain's) */
+ const char *dp = getenv("DUMP");
+ FILE *df = dp && *dp ? fopen(dp, "wb") : NULL;
float *logit = step_mm(m, prompt, np, 0, NULL, dmel, naud);
int len = np;
Cfg *c = &m->c;
for (int s = 0; s < n_new; s++) {
+ if (df) fwrite(logit, sizeof(float), (size_t)c->unpad_vocab, df);
int best = 0; float bv = logit[0];
for (int i = 1; i < c->unpad_vocab; i++) if (logit[i] > bv) { bv = logit[i]; best = i; }
free(logit);
@@ -2509,6 +2556,7 @@ static void generate(Model *m, const int *prompt, int np, int n_new, int *out,
int one = best;
logit = step(m, &one, 1, len - 1, NULL);
}
+ if (df) fclose(df);
}
/* ---------- interactive prompt mode: greedy, streaming, stop on eos ---------- */
@@ -2571,7 +2619,7 @@ static void generate_stream(Model *m, Tok *T, const char *prompt, int n_new,
m->t_fill, m->t_expert, m->t_shared, m->t_attn,
wall - m->t_fill - m->t_expert - m->t_shared - m->t_attn);
#ifdef COLI_VULKAN
- fflush(stdout); ink_vk_report(); ink_vk_tier_report(m, "run");
+ fflush(stdout); ink_vk_report(); ink_vk_tier_report(m, "run"); inkc_report(m);
#endif
free(ids);
}
@@ -2904,7 +2952,7 @@ static int serve_one(Model *m, Tok *T, SReq *q) {
m->t_fill - f0, m->t_shared - s0, m->t_expert - e0, m->t_attn - a0, 0.0, forwards);
fflush(stdout);
#ifdef COLI_VULKAN
- ink_vk_report(); ink_vk_tier_report(m, "turn");
+ ink_vk_report(); ink_vk_tier_report(m, "turn"); inkc_report(m);
#endif
serve_hits(m);
free(ids);
@@ -3139,6 +3187,7 @@ int main(int argc, char **argv) {
pins_load(&m, snap);
#ifdef COLI_VULKAN
ink_vk_tier_start(&m); /* after the history: the warm start reads it */
+ inkc_start(&m); /* COLI_VK_CHAIN, after the tier */
#endif
char tkp[2048]; snprintf(tkp, sizeof(tkp), "%s/tokenizer.json", snap);
Tok T; tok_load(&T, tkp);
@@ -3155,6 +3204,7 @@ int main(int argc, char **argv) {
pins_load(&m, snap);
#ifdef COLI_VULKAN
ink_vk_tier_start(&m);
+ inkc_start(&m); /* COLI_VK_CHAIN, after the tier */
#endif
char tkp[2048]; snprintf(tkp, sizeof(tkp), "%s/tokenizer.json", snap);
Tok T; tok_load(&T, tkp);
@@ -3208,6 +3258,7 @@ int main(int argc, char **argv) {
Model m; model_init(&m, snap, cap, bits);
#ifdef COLI_VULKAN
ink_vk_tier_start(&m); /* the oracle reads no history: the tier fills as experts pass by */
+ inkc_start(&m); /* COLI_VK_CHAIN, after the tier */
#endif
printf("== Inkling C engine (Stage A), cache = %d experts/layer, experts @ %s ==\n",
cap, m.xq ? "container (int4/int8 + .qs)" : bits ? "int (runtime quant)" : "f32");
@@ -3259,7 +3310,7 @@ int main(int argc, char **argv) {
printf("PEAK RSS: %.2f GB | expert cache hit %.1f%% | %.2f tok/s\n",
rss_gb(), tot?100.0*m.hits/tot:0.0, ngen/dt);
#ifdef COLI_VULKAN
- fflush(stdout); ink_vk_report(); ink_vk_tier_report(&m, "run");
+ fflush(stdout); ink_vk_report(); ink_vk_tier_report(&m, "run"); inkc_report(&m);
#endif
free(buf); free(arena);
return (match == ngen) ? 0 : 1;
diff --git a/c/inkling_chain.h b/c/inkling_chain.h
new file mode 100644
index 00000000..9548e2d9
--- /dev/null
+++ b/c/inkling_chain.h
@@ -0,0 +1,564 @@
+/* inkling_chain.h -- Inkling's layers as a dense chain on the Vulkan device (vk_chain.h).
+ * Included once by inkling.c in a COLI_VULKAN build, after the CPU forward it stands in
+ * for; COLI_VK_CHAIN decides (backend_vulkan.c, coli_vk_chain_decide).
+ *
+ * What runs where, per layer, for one block of rows (decode: one row):
+ * device, frame A1: the MoE output of the layer before joins the residual (the routed
+ * sum from the host, plus each shared expert times its combine weight, in moe()'s
+ * order; then that layer's MLP short convolution and the residual add), the input
+ * RMSNorm, q/k/v and the relative-bias projection r, the short convolutions on K and
+ * V with their rings, the per-head q/k RMSNorms, the attention (chain_relattn.comp:
+ * the bias bank mixed per row and head, tau, the sliding window over a ring cache),
+ * the step's K/V rows into the device ring, o_proj, the attention's short
+ * convolution, the residual add, the post-attention RMSNorm and the router logits
+ * (E routed and the shared experts' logits). Then the frame is waited for.
+ * A dense-MLP layer has no host step: its MLP (gate, up, SwiGLU, down, the global
+ * scale), its short convolution and the residual add follow in the same frame.
+ * host: the sigmoid router with its bias, top-k and the joint combine weights (TOPP
+ * included), the routed experts (the expert tier's device batch and the CPU's share,
+ * joined in rank order: moe_ex, the code moe() runs), the K/V rows copied into the
+ * host's cache.
+ * device, frame A2 (not waited for): the shared experts, while the host computes the
+ * routed experts; their combine weights come up with the routed sum.
+ * Crossing per MoE layer: the normalized rows (D floats a row), the router logits
+ * (E + n_shared a row) and the K/V rows down; the routed sum (D a row) and the shared
+ * experts' weights up. The last frame normalizes the last row, divides it by the logits'
+ * width multiplier and runs lm_head; its logits come back. The per-position heads (the
+ * logprobs channel, teacher forcing) read the final rows on the host, as before.
+ *
+ * State and who owns it:
+ * - the residual stream: on the device for the whole forward;
+ * - the K/V cache: the host's stays canonical (every chain step copies its new rows
+ * back). The device holds a mirror per layer with the host's layout, a ring of
+ * `cap` rows on a sliding layer (position t at row t % cap) and the whole context
+ * on a global one. What the host wrote alone (a CPU step) is a range of positions
+ * per layer, [dlo, dhi): the next chain step uploads the rows of its last `cap`
+ * positions first, whatever positions they now hold, so that every row of the
+ * device ring is the host's even after a rewind (a pinned snapshot) over a wrapped
+ * ring. A cache allocated anew (kv_alloc) is mirrored again;
+ * - the four short-convolution states per layer: on the device while the chain runs,
+ * and copied back to the host at the end of every chain step (a few KB a layer), so
+ * the host's are always current; anything that writes the host's (a reset, a pinned
+ * snapshot restored, a CPU step) has the next chain step upload them (or fill zeros).
+ *
+ * The chain declines (the per-matrix path runs, the state marked as the host's) under
+ * CUDA or Metal (they keep their priority), where a matrix is in a form the shaders do
+ * not read as this CPU does (bf16 on a CPU whose bf16 dot rounds the activations, as
+ * the per-matrix path keeps it on the CPU there), and for a geometry outside its shaders
+ * (head dim above 256, a bias bank wider than 64, a convolution above 9 taps).
+ * A device lost mid-step: the engine rebuilds the K/V cache and the convolution states
+ * on the CPU from the prefix record (the ids the state was built from), a prefill's
+ * worth of CPU work, and runs on the CPU from there. Only a state the ids do not
+ * describe (audio) cannot be rebuilt: that stops the engine with a message.
+ * COLI_VK_CHAIN_FAULT=n (vk_chain.c) fakes the loss at the n-th frame, for tests. */
+#include "vk_chain.h"
+
+/* COLI_VK_CHAIN unset on an integrated GPU with the expert tier (coli_vk_chain_decide):
+ * no Inkling checkpoint has been timed on one */
+#ifndef INKLING_CHAIN_IGPU
+#define INKLING_CHAIN_IGPU COLI_VK_CHAIN_UNMEASURED
+#endif
+#define INKC_HOST 0 /* the host's convolution states are newer than the device's */
+#define INKC_BOTH 1
+
+typedef struct {
+ int ok, failed;
+ int rows; /* scratch capacity in rows */
+ int max_t; float **hostK; /* the host cache the mirrors copy */
+ VkcBuf *prm; /* norms, bias banks, convolution taps */
+ size_t *o_in, *o_post, *o_qn, *o_kn, *o_relp, *o_cw[4], o_final;
+ ColiVkTensor *t_lm, **t_q, **t_k, **t_v, **t_r, **t_o, **t_router, **t_dg, **t_du, **t_dd, **t_sg, **t_su, **t_sd;
+ VkcBuf **kc, **vc, **ring[4];
+ int *cap, *dlo, *dhi; /* per layer: ring rows; positions the host wrote alone */
+ int *slot, nslot; /* per layer: its K/V region in kvd, within its frame */
+ int kvo_max, qo_max, ro_max, mi_max;
+ size_t vs_off; /* the V rows' offset in kvs */
+ size_t *o_csd, csd_n; /* per (bank, layer): the state's offset in csd */
+ int cs_where, host_zero;
+ VkcBuf *x, *nrm, *tmp, *q, *kvs, *r, *ctx, *h2, *lg, *gs, *us, *sh, *mlp, *fin;
+ VkcBuf *h2d, *lgd, *kvd, *csd, *outd, *xd, *routed, *shw, *tau;
+ float *host_routed, *host_w;
+ unsigned long long forwards;
+ double host_ms;
+} InkChain;
+
+static int inkc_chunk_rows(void) {
+ const char *e = getenv("COLI_VK_CHAIN_ROWS");
+ int v = e && *e ? atoi(e) : 512;
+ return v < 1 ? 1 : v > 65535 ? 65535 : v;
+}
+static void inkc_fatal(const char *what) {
+ fprintf(stderr, "[VK] inkling chain: %s -- stopping (COLI_VK_CHAIN=0 keeps the state on the CPU)\n", what);
+ exit(1);
+}
+static int64_t inkc_cs_cells(const Cfg *c, int bank, int i) {
+ return (int64_t)(bank < 2 ? L_KV(c, i) * L_HD(c, i) : c->hidden) * (c->conv_k - 1);
+}
+
+/* The device copy of a resident matrix or of a view of one (wt_off_i: a shared expert
+ * of the fused [ns][R][I] tensors): the per-matrix path's copy when it keeps a table
+ * for the tensor (ink_vk_matmul, keyed by the view's first weight), else the chain's
+ * own. NULL: not a form the shaders read as this CPU does (ink_vk_fmt). */
+static ColiVkTensor *inkc_tensor(Wt view, int I, int O) {
+ int fmt = ink_vk_fmt(&view);
+ const void *data = view.q4 ? (const void *)view.q4 : view.f ? (const void *)view.f : (const void *)view.h;
+ const float *sc = view.q4 ? view.qs : NULL;
+ int gs = view.q4 ? view.gs : 0;
+ if (!fmt || !data) return NULL;
+ InkVk *v = view.vk;
+ if (v) for (int k = 0; k < v->n; k++) {
+ InkVkView *e = &v->e[k];
+ if (e->q && e->q != data) continue;
+ if (e->dead) return NULL;
+ e->q = data;
+ if (!e->t && !coli_vk_tensor_ensure(&e->t, data, sc, fmt, I, O, gs)) { e->dead = 1; return NULL; }
+ return e->t;
+ }
+ ColiVkTensor *t = NULL;
+ return coli_vk_tensor_ensure(&t, data, sc, fmt, I, O, gs) ? t : NULL;
+}
+
+/* The model's parameters on the device, its tensors resolved; NULL = the chain cannot run. */
+static InkChain *inkc_setup(Model *m) {
+ InkChain *ch = (InkChain *)m->vkchain;
+ if (ch) return ch->ok ? ch : NULL;
+ ch = (InkChain *)calloc(1, sizeof *ch);
+ if (!ch) return NULL;
+ m->vkchain = ch;
+ Cfg *c = &m->c; int L = c->n_layers, D = c->hidden, E = c->n_experts, ns = c->n_shared, CK = c->conv_k, dr = c->d_rel;
+ for (int i = 0; i < L; i++) {
+ int H = L_HEADS(c, i), KV = L_KV(c, i), hd = L_HD(c, i);
+ if (hd > 256 || hd < 1 || KV < 1 || H % KV || dr < 1 || dr > 64 || L_EXT(c, i) < 1 || CK < 1 || CK > 9 ||
+ (c->local[i] && c->window < 1)) {
+ fprintf(stderr, "[VK] inkling chain: a geometry its shaders do not take (layer %d: head dim %d, %d/%d heads, "
+ "bias bank %d x %d, %d taps); per-matrix path\n", i, hd, H, KV, dr, L_EXT(c, i), CK);
+ return NULL;
+ }
+ }
+ if (!vkc_sconv_ready() || !vkc_relattn_ready()) {
+ fprintf(stderr, "[VK] inkling chain: chain_sconv.spv or chain_relattn.spv is missing; per-matrix path\n");
+ return NULL;
+ }
+#define INKC_ARR(f, T) (ch->f = calloc((size_t)L, sizeof(T)))
+ if (!INKC_ARR(o_in, size_t) || !INKC_ARR(o_post, size_t) || !INKC_ARR(o_qn, size_t) || !INKC_ARR(o_kn, size_t) ||
+ !INKC_ARR(o_relp, size_t) || !INKC_ARR(o_cw[0], size_t) || !INKC_ARR(o_cw[1], size_t) || !INKC_ARR(o_cw[2], size_t) ||
+ !INKC_ARR(o_cw[3], size_t) || !INKC_ARR(t_q, void *) || !INKC_ARR(t_k, void *) || !INKC_ARR(t_v, void *) ||
+ !INKC_ARR(t_r, void *) || !INKC_ARR(t_o, void *) || !INKC_ARR(t_router, void *) || !INKC_ARR(t_dg, void *) ||
+ !INKC_ARR(t_du, void *) || !INKC_ARR(t_dd, void *) || !INKC_ARR(kc, void *) || !INKC_ARR(vc, void *) ||
+ !INKC_ARR(ring[0], void *) || !INKC_ARR(ring[1], void *) || !INKC_ARR(ring[2], void *) || !INKC_ARR(ring[3], void *) ||
+ !INKC_ARR(cap, int) || !INKC_ARR(dlo, int) || !INKC_ARR(dhi, int) || !INKC_ARR(slot, int)) return NULL;
+#undef INKC_ARR
+ int nsl = ns > 0 ? ns : 1;
+ ch->t_sg = calloc((size_t)L * nsl, sizeof(void *)); ch->t_su = calloc((size_t)L * nsl, sizeof(void *));
+ ch->t_sd = calloc((size_t)L * nsl, sizeof(void *)); ch->o_csd = calloc((size_t)4 * L, sizeof(size_t));
+ if (!ch->t_sg || !ch->t_su || !ch->t_sd || !ch->o_csd) return NULL;
+ /* the parameter arena: offsets, then one upload */
+ size_t n = 0;
+ for (int i = 0; i < L; i++) {
+ int hd = L_HD(c, i), kvo = L_KV(c, i) * hd;
+ ch->o_in[i] = n; n += D; ch->o_post[i] = n; n += D;
+ ch->o_qn[i] = n; n += hd; ch->o_kn[i] = n; n += hd;
+ ch->o_relp[i] = n; n += (size_t)dr * L_EXT(c, i);
+ ch->o_cw[0][i] = n; n += (size_t)kvo * CK; ch->o_cw[1][i] = n; n += (size_t)kvo * CK;
+ ch->o_cw[2][i] = n; n += (size_t)D * CK; ch->o_cw[3][i] = n; n += (size_t)D * CK;
+ }
+ ch->o_final = n; n += D;
+ float *arena = calloc(n, sizeof(float));
+ if (!arena) return NULL;
+ for (int i = 0; i < L; i++) {
+ Layer *l = &m->L[i];
+ int hd = L_HD(c, i), kvo = L_KV(c, i) * hd;
+ memcpy(arena + ch->o_in[i], l->in_ln, D * sizeof(float));
+ memcpy(arena + ch->o_post[i], l->post_ln, D * sizeof(float));
+ memcpy(arena + ch->o_qn[i], l->qn, hd * sizeof(float));
+ memcpy(arena + ch->o_kn[i], l->kn, hd * sizeof(float));
+ memcpy(arena + ch->o_relp[i], l->relp, (size_t)dr * L_EXT(c, i) * sizeof(float));
+ memcpy(arena + ch->o_cw[0][i], l->k_cw, (size_t)kvo * CK * sizeof(float));
+ memcpy(arena + ch->o_cw[1][i], l->v_cw, (size_t)kvo * CK * sizeof(float));
+ memcpy(arena + ch->o_cw[2][i], l->a_cw, (size_t)D * CK * sizeof(float));
+ memcpy(arena + ch->o_cw[3][i], l->m_cw, (size_t)D * CK * sizeof(float));
+ }
+ memcpy(arena + ch->o_final, m->final_norm, D * sizeof(float));
+ ch->prm = vkc_buf(n * sizeof(float), VKC_DEV);
+ int ok = ch->prm && vkc_begin() && vkc_write(ch->prm, 0, arena, n * sizeof(float)) && vkc_submit(1);
+ free(arena);
+ if (!ok) return NULL;
+ /* the tensors: the per-matrix path's device copies where it keeps them */
+ const char *what = NULL;
+ for (int i = 0; i < L && !what; i++) {
+ Layer *l = &m->L[i];
+ int H = L_HEADS(c, i), hd = L_HD(c, i), kvo = L_KV(c, i) * hd, I = c->moe_inter;
+ if (!(ch->t_q[i] = inkc_tensor(l->q, D, H * hd)) || !(ch->t_k[i] = inkc_tensor(l->k, D, kvo)) ||
+ !(ch->t_v[i] = inkc_tensor(l->v, D, kvo)) || !(ch->t_r[i] = inkc_tensor(l->r, D, H * dr)) ||
+ !(ch->t_o[i] = inkc_tensor(l->o, H * hd, D))) what = "an attention matrix";
+ else if (!c->sparse[i]) {
+ if (!(ch->t_dg[i] = inkc_tensor(l->dg, D, c->dense_inter)) || !(ch->t_du[i] = inkc_tensor(l->du, D, c->dense_inter)) ||
+ !(ch->t_dd[i] = inkc_tensor(l->dd, c->dense_inter, D))) what = "a dense MLP matrix";
+ } else {
+ if (!coli_vk_tensor_ensure(&ch->t_router[i], l->router, NULL, 10, D, E + ns, 0)) what = "the router";
+ for (int j = 0; j < ns && !what; j++)
+ if (!(ch->t_sg[(size_t)i * nsl + j] = inkc_tensor(wt_off_i(l->sh_g, (int64_t)j * I * D, D), D, I)) ||
+ !(ch->t_su[(size_t)i * nsl + j] = inkc_tensor(wt_off_i(l->sh_u, (int64_t)j * I * D, D), D, I)) ||
+ !(ch->t_sd[(size_t)i * nsl + j] = inkc_tensor(wt_off_i(l->sh_d, (int64_t)j * D * I, I), I, D)))
+ what = "a shared expert";
+ }
+ }
+ if (!what && !(ch->t_lm = inkc_tensor(m->lm_head, D, c->unpad_vocab))) what = "lm_head";
+ if (what) {
+ int bf16 = m->L[0].q.h && !m->L[0].q.q4 && !ink_vk_fmt(&m->L[0].q);
+ fprintf(stderr, "[VK] inkling chain: %s did not reach the device%s; per-matrix path\n", what,
+ bf16 ? " (bf16: this CPU's bf16 dot rounds the activations, the device would not, so bf16 stays on the CPU)" : "");
+ return NULL;
+ }
+ /* geometry of the scratch; the frames: a MoE layer ends one (its host step) */
+ int fr = 0;
+ for (int i = 0; i < L; i++) {
+ int H = L_HEADS(c, i), hd = L_HD(c, i), kvo = L_KV(c, i) * hd;
+ if (kvo > ch->kvo_max) ch->kvo_max = kvo;
+ if (H * hd > ch->qo_max) ch->qo_max = H * hd;
+ if (H * dr > ch->ro_max) ch->ro_max = H * dr;
+ int mi = c->sparse[i] ? c->moe_inter : c->dense_inter;
+ if (mi > ch->mi_max) ch->mi_max = mi;
+ ch->slot[i] = fr++;
+ if (fr > ch->nslot) ch->nslot = fr;
+ if (c->sparse[i]) fr = 0;
+ for (int b = 0; b < 4; b++) {
+ ch->o_csd[(size_t)b * L + i] = ch->csd_n; ch->csd_n += (size_t)inkc_cs_cells(c, b, i);
+ if (!(ch->ring[b][i] = vkc_buf((size_t)(inkc_cs_cells(c, b, i) > 0 ? inkc_cs_cells(c, b, i) : 1) * sizeof(float), VKC_DEV)))
+ return NULL;
+ }
+ }
+ ch->cs_where = INKC_HOST; ch->host_zero = 0;
+ ch->ok = 1;
+ int nsp = 0; for (int i = 0; i < L; i++) nsp += c->sparse[i];
+ fprintf(stderr, "[VK] inkling chain: %d layers on the device (%d MoE), %.1f MiB of parameters\n",
+ L, nsp, n * 4 / 1048576.0);
+ return ch;
+}
+
+static int inkc_res(VkcBuf **b, size_t floats, int kind) { return vkc_reserve(b, (floats ? floats : 1) * sizeof(float), kind); }
+
+/* scratch for `rows` rows, and the K/V mirrors at the host's layout */
+static int inkc_scratch(InkChain *ch, Model *m, int rows) {
+ Cfg *c = &m->c; int D = c->hidden, ET = c->n_experts + c->n_shared, ns = c->n_shared > 0 ? c->n_shared : 1;
+ size_t r = (size_t)rows;
+ ch->vs_off = (r * ch->kvo_max + 63) & ~(size_t)63;
+ int ok = inkc_res(&ch->x, r * D, VKC_DEV) && inkc_res(&ch->nrm, r * D, VKC_DEV) && inkc_res(&ch->tmp, r * D, VKC_DEV) &&
+ inkc_res(&ch->q, r * ch->qo_max, VKC_DEV) && inkc_res(&ch->kvs, ch->vs_off + r * ch->kvo_max, VKC_DEV) &&
+ inkc_res(&ch->r, r * ch->ro_max, VKC_DEV) && inkc_res(&ch->ctx, r * ch->qo_max, VKC_DEV) &&
+ inkc_res(&ch->h2, r * D, VKC_DEV) && inkc_res(&ch->lg, r * ET, VKC_DEV) &&
+ inkc_res(&ch->gs, r * ch->mi_max, VKC_DEV) && inkc_res(&ch->us, r * ch->mi_max, VKC_DEV) &&
+ inkc_res(&ch->sh, (size_t)ns * r * D, VKC_DEV) && inkc_res(&ch->mlp, r * D, VKC_DEV) && inkc_res(&ch->fin, D, VKC_DEV) &&
+ inkc_res(&ch->h2d, r * D, VKC_DOWN) && inkc_res(&ch->lgd, r * ET, VKC_DOWN) &&
+ inkc_res(&ch->kvd, (size_t)ch->nslot * 2 * r * ch->kvo_max, VKC_DOWN) && inkc_res(&ch->csd, ch->csd_n, VKC_DOWN) &&
+ inkc_res(&ch->outd, c->unpad_vocab, VKC_DOWN) && inkc_res(&ch->routed, r * D, VKC_UP) &&
+ inkc_res(&ch->shw, (size_t)ns * r, VKC_UP) && inkc_res(&ch->tau, 2 * r, VKC_UP);
+ if (!ok) return 0;
+ if (ch->rows < rows) {
+ float *hr = realloc(ch->host_routed, r * D * sizeof(float)), *hw = hr ? realloc(ch->host_w, (size_t)ns * r * sizeof(float)) : NULL;
+ if (hr) ch->host_routed = hr;
+ if (!hr || !hw) return 0;
+ ch->host_w = hw; ch->rows = rows;
+ }
+ if (ch->max_t != m->max_t || ch->hostK != m->K) { /* a cache allocated anew: mirror it again */
+ for (int i = 0; i < c->n_layers; i++) {
+ vkc_free(ch->kc[i]); vkc_free(ch->vc[i]); ch->kc[i] = ch->vc[i] = NULL;
+ int cap = kv_ring_rows(c, i, m->max_t), hd = L_HD(c, i), KV = L_KV(c, i);
+ ch->cap[i] = cap;
+ /* a ring: every row, whatever positions it holds; a whole cache: the positions held */
+ ch->dlo[i] = 0; ch->dhi[i] = cap < m->max_t ? m->max_t : m->kv_len;
+ ch->kc[i] = vkc_buf((size_t)KV * cap * hd * sizeof(float), VKC_DEV);
+ ch->vc[i] = vkc_buf((size_t)KV * cap * hd * sizeof(float), VKC_DEV);
+ if (!ch->kc[i] || !ch->vc[i]) { ch->max_t = 0; ch->hostK = NULL; return 0; }
+ }
+ ch->max_t = m->max_t; ch->hostK = m->K;
+ }
+ return 1;
+}
+
+/* ---- state between the host and the device ----------------------------------- */
+/* The host wrote its convolution states (zero: state_reset's zeros). */
+static void inkc_host_wrote(Model *m, int zero) {
+ InkChain *ch = (InkChain *)m->vkchain;
+ if (!ch || !ch->ok) return;
+ ch->cs_where = INKC_HOST; ch->host_zero = zero;
+}
+/* A CPU step of S rows from pos_base: its K/V rows and states are the host's alone. */
+static void inkc_cpu_step(Model *m, int pos_base, int S) {
+ InkChain *ch = (InkChain *)m->vkchain;
+ if (!ch || !ch->ok) return;
+ ch->cs_where = INKC_HOST; ch->host_zero = 0;
+ for (int i = 0; i < m->c.n_layers; i++) {
+ if (ch->dlo[i] >= ch->dhi[i]) { ch->dlo[i] = pos_base; ch->dhi[i] = pos_base + S; continue; }
+ if (pos_base < ch->dlo[i]) ch->dlo[i] = pos_base;
+ if (pos_base + S > ch->dhi[i]) ch->dhi[i] = pos_base + S;
+ }
+}
+/* Record the uploads that make the device state the host's: the convolution states if
+ * the host's are newer, and the rows of the last `cap` positions the host wrote alone. */
+static int inkc_push_state(InkChain *ch, Model *m) {
+ Cfg *c = &m->c; int ok = 1;
+ if (ch->cs_where == INKC_HOST) {
+ for (int i = 0; i < c->n_layers && ok; i++) for (int b = 0; b < 4 && ok; b++) {
+ size_t nc = (size_t)inkc_cs_cells(c, b, i);
+ ok = ch->host_zero ? vkc_zero(ch->ring[b][i], 0, nc) : vkc_write(ch->ring[b][i], 0, m->cs[b][i], nc * sizeof(float));
+ }
+ ch->cs_where = INKC_BOTH;
+ }
+ for (int i = 0; i < c->n_layers && ok; i++) {
+ if (ch->dlo[i] >= ch->dhi[i]) continue;
+ int cap = ch->cap[i], hd = L_HD(c, i), KV = L_KV(c, i);
+ int t0 = ch->dhi[i] - cap > ch->dlo[i] ? ch->dhi[i] - cap : ch->dlo[i];
+ for (int t = t0; t < ch->dhi[i] && ok; ) { /* a run of rows up to the ring's end */
+ int row = t % cap, run = ch->dhi[i] - t < cap - row ? ch->dhi[i] - t : cap - row;
+ for (int h = 0; h < KV && ok; h++) {
+ size_t off = ((size_t)h * cap + row) * hd;
+ ok = vkc_write(ch->kc[i], off, m->K[i] + off, (size_t)run * hd * sizeof(float)) &&
+ vkc_write(ch->vc[i], off, m->V[i] + off, (size_t)run * hd * sizeof(float));
+ }
+ t += run;
+ }
+ ch->dlo[i] = ch->dhi[i] = 0;
+ }
+ return ok;
+}
+
+/* The device was lost with the newest state on it. The host's K/V rows below pos_base
+ * may already hold some of the lost step's (a ring wrapped by it), and its convolution
+ * states are the step's start: rebuild both on the CPU from the `upto` positions the
+ * prefix record names, and leave the chain off. */
+static void inkc_recover(Model *m, int upto) {
+ InkChain *ch = (InkChain *)m->vkchain;
+ Cfg *c = &m->c; int D = c->hidden;
+ g_vk_chain = 0;
+ if (ch) ch->failed = 1;
+ for (int i = 0; i < c->n_layers; i++)
+ for (int b = 0; b < 4; b++) memset(m->cs[b][i], 0, (size_t)inkc_cs_cells(c, b, i) * sizeof(float));
+ if (upto <= 0) return;
+ if (m->kvp.tainted || m->kvp.len < upto || !m->kvp.fed)
+ inkc_fatal("the device was lost with a state its token ids do not describe (audio)");
+ fprintf(stderr, "[VK] inkling chain: the device was lost; rebuilding the state of %d positions on the CPU, "
+ "which runs from here on\n", upto);
+ int *ids = malloc((size_t)upto * sizeof(int));
+ float *x = falloc((int64_t)upto * D);
+ if (!ids) { fprintf(stderr, "OOM rebuilding the state\n"); exit(1); }
+ memcpy(ids, m->kvp.fed, (size_t)upto * sizeof(int));
+ for (int s = 0; s < upto; s++) {
+ wt_row_f32(m->embed, (int64_t)ids[s] * D, x + (int64_t)s * D, D);
+ if (m->embed_norm) rmsnorm_row(x + (int64_t)s * D, x + (int64_t)s * D, m->embed_norm, D, c->eps);
+ }
+ inkling_layers_forward_range(m, x, upto, 0, 0, c->n_layers);
+ free(ids); free(x);
+}
+
+/* ---- one layer's pieces ---------------------------------------------------------- */
+static int inkc_norm(VkcBuf *x, size_t xo, VkcBuf *w, size_t wo, VkcBuf *y, size_t yo, int rows, int D, float eps) {
+ VkcNorm p = {rows, D, 1, (int)xo, D, D, (int)yo, D, D, (int)wo, 0, 0, eps, 1.f};
+ return vkc_norm(x, w, y, &p);
+}
+static int inkc_conv(InkChain *ch, VkcBuf *x, size_t xo, int n, int C, int bank, int i, const Cfg *c) {
+ VkcSconv p = {0, n, C, c->conv_k, (int)xo, C, (int)ch->o_cw[bank][i], 0, 0, 1.f};
+ return vkc_sconv(x, ch->prm, ch->ring[bank][i], &p);
+}
+/* nrm in; the attention block's o_proj rows in tmp; the new K/V rows into the device
+ * ring and into the frame's region of kvd */
+static int inkc_attention(InkChain *ch, Model *m, int i, int n, int pb) {
+ Cfg *c = &m->c;
+ int H = L_HEADS(c, i), KV = L_KV(c, i), hd = L_HD(c, i), kvo = KV * hd, dr = c->d_rel;
+ int cap = ch->cap[i], local = c->local[i];
+ size_t vs = ch->vs_off;
+ int ok = vkc_matmul(ch->t_q[i], ch->nrm, 0, ch->q, 0, n) && vkc_matmul(ch->t_k[i], ch->nrm, 0, ch->kvs, 0, n) &&
+ vkc_matmul(ch->t_v[i], ch->nrm, 0, ch->kvs, vs, n) && vkc_matmul(ch->t_r[i], ch->nrm, 0, ch->r, 0, n) &&
+ inkc_conv(ch, ch->kvs, 0, n, kvo, 0, i, c) && inkc_conv(ch, ch->kvs, vs, n, kvo, 1, i, c);
+ VkcNorm qn = {n * H, hd, H, 0, H * hd, hd, 0, H * hd, hd, (int)ch->o_qn[i], 0, 0, c->eps, 1.f};
+ VkcNorm kn = {n * KV, hd, KV, 0, kvo, hd, 0, kvo, hd, (int)ch->o_kn[i], 0, 0, c->eps, 1.f};
+ ok = ok && vkc_norm(ch->q, ch->prm, ch->q, &qn) && vkc_norm(ch->kvs, ch->prm, ch->kvs, &kn);
+ VkcRelAttn a = {n, H, KV, hd, pb, cap, local ? c->window : 0, L_EXT(c, i), dr,
+ 0, H * hd, 0, H * hd, 0, 0, 0, (int)vs, kvo, 0, H * dr, (int)ch->o_relp[i], local ? ch->rows : 0,
+ 1.f / (float)hd};
+ ok = ok && vkc_relattn(ch->q, ch->kc[i], ch->vc[i], ch->ctx, ch->kvs, ch->r, ch->prm, ch->tau, &a);
+ if (!ok) return 0;
+ /* the step's rows into the ring after the attention read them: the last `cap` of them */
+ int s0 = n > cap ? n - cap : 0, nr = (n - s0) * KV;
+ VkcRegion *rk = malloc(sizeof *rk * (size_t)nr * 2), *rv = rk ? rk + nr : NULL;
+ if (!rk) return 0;
+ for (int s = s0; s < n; s++) for (int h = 0; h < KV; h++) {
+ size_t dst = ((size_t)h * cap + (size_t)(pb + s) % cap) * hd, src = (size_t)s * kvo + (size_t)h * hd;
+ rk[(s - s0) * KV + h] = (VkcRegion){dst, src, (size_t)hd};
+ rv[(s - s0) * KV + h] = (VkcRegion){dst, vs + src, (size_t)hd};
+ }
+ size_t ko = (size_t)ch->slot[i] * 2 * ch->rows * ch->kvo_max;
+ ok = vkc_copy_regions(ch->kc[i], ch->kvs, rk, nr) && vkc_copy_regions(ch->vc[i], ch->kvs, rv, nr) &&
+ vkc_copy(ch->kvd, ko, ch->kvs, 0, (size_t)n * kvo) &&
+ vkc_copy(ch->kvd, ko + (size_t)ch->rows * ch->kvo_max, ch->kvs, vs, (size_t)n * kvo);
+ free(rk);
+ return ok && vkc_matmul(ch->t_o[i], ch->ctx, 0, ch->tmp, 0, n);
+}
+/* The rows a frame's layers [i0, i1] wrote, into the host's cache (attention()'s rows). */
+static void inkc_kv_down(InkChain *ch, Model *m, int i0, int i1, int n, int pb) {
+ Cfg *c = &m->c;
+ for (int i = i0; i <= i1; i++) {
+ int KV = L_KV(c, i), hd = L_HD(c, i), kvo = KV * hd, cap = ch->cap[i], s0 = n > cap ? n - cap : 0;
+ const float *kv = (const float *)vkc_ptr(ch->kvd) + (size_t)ch->slot[i] * 2 * ch->rows * ch->kvo_max;
+ const float *vv = kv + (size_t)ch->rows * ch->kvo_max;
+ for (int s = s0; s < n; s++) for (int h = 0; h < KV; h++) {
+ size_t dst = ((size_t)h * cap + (size_t)(pb + s) % cap) * hd, src = (size_t)s * kvo + (size_t)h * hd;
+ memcpy(m->K[i] + dst, kv + src, hd * sizeof(float));
+ memcpy(m->V[i] + dst, vv + src, hd * sizeof(float));
+ }
+ }
+}
+/* the MLP output in mlp: its short convolution, then the residual add */
+static int inkc_mlp_out(InkChain *ch, int i, int n, const Cfg *c) {
+ VkcEw add = {VKC_EW_ADD, n * c->hidden, c->hidden, 1, 0, 1, 0, 0, 0, 0, 0, 1.f};
+ return inkc_conv(ch, ch->mlp, 0, n, c->hidden, 3, i, c) && vkc_ew(ch->x, ch->x, ch->mlp, NULL, NULL, &add);
+}
+/* a dense-MLP layer's MLP from h2, as dense_mlp() */
+static int inkc_dense(InkChain *ch, Model *m, Layer *l, int i, int n) {
+ Cfg *c = &m->c; int DI = c->dense_inter;
+ VkcEw sw = {VKC_EW_SWIGLU, n * DI, DI, 1, 0, 1, 0, 0, 0, 0, 0, 1.f};
+ VkcSconv sc = {1, 0, 0, 0, 0, 0, 0, 0, n * c->hidden, l->dgs};
+ return vkc_matmul(ch->t_dg[i], ch->h2, 0, ch->gs, 0, n) && vkc_matmul(ch->t_du[i], ch->h2, 0, ch->us, 0, n) &&
+ vkc_ew(ch->gs, ch->gs, ch->us, NULL, NULL, &sw) && vkc_matmul(ch->t_dd[i], ch->gs, 0, ch->mlp, 0, n) &&
+ vkc_sconv(ch->mlp, NULL, NULL, &sc) && inkc_mlp_out(ch, i, n, c);
+}
+/* frame A2: each shared expert's output from h2, unweighted, into sh[j] */
+static int inkc_shared(InkChain *ch, Model *m, int i, int n) {
+ Cfg *c = &m->c; int I = c->moe_inter, ns = c->n_shared, nsl = ns > 0 ? ns : 1, ok = 1;
+ VkcEw sw = {VKC_EW_SWIGLU, n * I, I, 1, 0, 1, 0, 0, 0, 0, 0, 1.f};
+ for (int j = 0; j < ns && ok; j++)
+ ok = vkc_matmul(ch->t_sg[(size_t)i * nsl + j], ch->h2, 0, ch->gs, 0, n) &&
+ vkc_matmul(ch->t_su[(size_t)i * nsl + j], ch->h2, 0, ch->us, 0, n) &&
+ vkc_ew(ch->gs, ch->gs, ch->us, NULL, NULL, &sw) &&
+ vkc_matmul(ch->t_sd[(size_t)i * nsl + j], ch->gs, 0, ch->sh, (size_t)j * ch->rows * c->hidden, n);
+ return ok;
+}
+/* a MoE layer's output joining the residual, in moe()'s order: the routed sum, then each
+ * shared expert times its weight; then the MLP's short convolution and the add */
+static int inkc_join(InkChain *ch, Model *m, int i, int n) {
+ Cfg *c = &m->c; int D = c->hidden, ok = vkc_copy(ch->mlp, 0, ch->routed, 0, (size_t)n * D);
+ for (int j = 0; j < c->n_shared && ok; j++) {
+ VkcEw p = {VKC_EW_HC_APPLY, n * D, D, 1, 0, 1, 0, j * ch->rows, j * ch->rows * D, 0, 0, 1.f};
+ ok = vkc_ew(ch->mlp, ch->shw, ch->sh, NULL, NULL, &p);
+ }
+ return ok && inkc_mlp_out(ch, i, n, c);
+}
+
+/* Every layer for S rows from host rows xh, the last row's logits into `logit`; xh gets
+ * the final rows back when want_x. 0 = not taken: xh is as it was, and the caller runs
+ * the step on the CPU (after inkc_recover when the device was lost). */
+static int inkc_forward(Model *m, float *xh, int S, int pos_base, int want_x, float *logit) {
+ if (!g_vk_chain) return 0;
+ if (g_vk_chain == COLI_VK_CHAIN_PREFILL && S <= 2) return 0; /* prompts only: decode on the CPU */
+ InkChain *ch = inkc_setup(m);
+ if (!ch || ch->failed) return 0;
+ Cfg *c = &m->c; int D = c->hidden, L = c->n_layers, ET = c->n_experts + c->n_shared, ns = c->n_shared;
+ int CH = inkc_chunk_rows(), rows = S < CH ? S : CH;
+ if (pos_base + S > m->max_t || !inkc_scratch(ch, m, rows) || (want_x && !inkc_res(&ch->xd, (size_t)rows * D, VKC_DOWN))) {
+ fprintf(stderr, "[VK] inkling chain: device memory for %d rows refused; per-matrix path\n", rows);
+ ch->failed = 1;
+ return 0;
+ }
+ float *xfin = want_x ? malloc((size_t)S * D * sizeof(float)) : NULL;
+ if (want_x && !xfin) return 0;
+ vkc_gemm_rows(-1);
+ for (int c0 = 0; c0 < S; c0 += rows) {
+ int n = S - c0 < rows ? S - c0 : rows, pb = pos_base + c0;
+ if (!vkc_begin() || !vkc_write(ch->x, 0, xh + (size_t)c0 * D, (size_t)n * D * sizeof(float)) ||
+ !inkc_push_state(ch, m)) goto lost;
+ float *tau = (float *)vkc_ptr(ch->tau); /* attention()'s tau: global rows, then 1 for sliding ones */
+ for (int s = 0; s < n; s++) {
+ float t = 1.f;
+ if (c->log_floor > 0) { double en = (double)(pb + s + 1) / c->log_floor; if (en > 1.0) t = 1.f + c->log_alpha * (float)log(en); }
+ tau[s] = t; tau[ch->rows + s] = 1.f;
+ }
+ int ok = 1, pending = 0, f0 = 0;
+ for (int i = 0; i < L && ok; i++) {
+ Layer *l = &m->L[i];
+ VkcEw add = {VKC_EW_ADD, n * D, D, 1, 0, 1, 0, 0, 0, 0, 0, 1.f};
+ if (pending) ok = inkc_join(ch, m, i - 1, n);
+ ok = ok && inkc_norm(ch->x, 0, ch->prm, ch->o_in[i], ch->nrm, 0, n, D, c->eps) &&
+ inkc_attention(ch, m, i, n, pb) && inkc_conv(ch, ch->tmp, 0, n, D, 2, i, c) &&
+ vkc_ew(ch->x, ch->x, ch->tmp, NULL, NULL, &add) &&
+ inkc_norm(ch->x, 0, ch->prm, ch->o_post[i], ch->h2, 0, n, D, c->eps);
+ pending = 0;
+ if (ok && !c->sparse[i]) { ok = inkc_dense(ch, m, l, i, n); continue; }
+ ok = ok && vkc_matmul(ch->t_router[i], ch->h2, 0, ch->lg, 0, n) &&
+ vkc_copy(ch->lgd, 0, ch->lg, 0, (size_t)n * ET) && vkc_copy(ch->h2d, 0, ch->h2, 0, (size_t)n * D);
+ double t0 = now_s();
+ ok = ok && vkc_submit(1); /* A1 */
+ m->t_attn += now_s() - t0;
+ if (!ok) break;
+ inkc_kv_down(ch, m, f0, i, n, pb);
+ f0 = i + 1;
+ /* A2: the shared experts, while the host computes the routed experts */
+ ok = vkc_begin() && inkc_shared(ch, m, i, n) && vkc_submit(0);
+ double t1 = now_s();
+ moe_ex(m, l, i, (float *)vkc_ptr(ch->h2d), n, ch->host_routed, (const float *)vkc_ptr(ch->lgd), ch->host_w);
+ memcpy(vkc_ptr(ch->routed), ch->host_routed, (size_t)n * D * sizeof(float));
+ for (int j = 0; j < ns; j++)
+ memcpy((float *)vkc_ptr(ch->shw) + (size_t)j * ch->rows, ch->host_w + (size_t)j * n, (size_t)n * sizeof(float));
+ ch->host_ms += (now_s() - t1) * 1e3;
+ ok = ok && vkc_begin();
+ pending = 1;
+ }
+ if (ok && pending) ok = inkc_join(ch, m, L - 1, n);
+ if (ok && want_x) ok = vkc_copy(ch->xd, 0, ch->x, 0, (size_t)n * D);
+ for (int i = 0; i < L && ok; i++) for (int b = 0; b < 4 && ok; b++) /* the convolution states, for the host */
+ ok = vkc_copy(ch->csd, ch->o_csd[(size_t)b * L + i], ch->ring[b][i], 0, (size_t)inkc_cs_cells(c, b, i));
+ int last = c0 + n == S;
+ if (ok && last) {
+ VkcSconv dv = {2, 0, 0, 0, 0, 0, 0, 0, D, c->mup};
+ ok = inkc_norm(ch->x, (size_t)(n - 1) * D, ch->prm, ch->o_final, ch->fin, 0, 1, D, c->eps) &&
+ vkc_sconv(ch->fin, NULL, NULL, &dv) && vkc_matmul(ch->t_lm, ch->fin, 0, ch->outd, 0, 1);
+ }
+ double t0 = now_s();
+ ok = ok && vkc_submit(1);
+ m->t_attn += now_s() - t0;
+ if (!ok) goto lost;
+ if (f0 < L) inkc_kv_down(ch, m, f0, L - 1, n, pb);
+ const float *csd = (const float *)vkc_ptr(ch->csd);
+ for (int i = 0; i < L; i++) for (int b = 0; b < 4; b++)
+ memcpy(m->cs[b][i], csd + ch->o_csd[(size_t)b * L + i], (size_t)inkc_cs_cells(c, b, i) * sizeof(float));
+ ch->cs_where = INKC_BOTH; ch->host_zero = 0;
+ if (want_x) memcpy(xfin + (size_t)c0 * D, vkc_ptr(ch->xd), (size_t)n * D * sizeof(float));
+ if (last) memcpy(logit, vkc_ptr(ch->outd), (size_t)c->unpad_vocab * sizeof(float));
+ }
+ if (want_x) { memcpy(xh, xfin, (size_t)S * D * sizeof(float)); free(xfin); }
+ ch->forwards++;
+ return 1;
+lost: /* a frame failed: the device is gone (or would not take a command); the CPU takes over */
+ if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost(); }
+ free(xfin);
+ inkc_recover(m, pos_base);
+ return 0;
+}
+
+static void inkc_report(Model *m) {
+ InkChain *ch = (InkChain *)m->vkchain;
+ if (!ch || !ch->ok || !ch->forwards) return;
+ VkcStats st; vkc_stats(&st);
+ fprintf(stderr, "[VK] inkling chain: %llu forwards, %llu frames (%llu ops, %llu matmuls, %llu tiled GEMM), "
+ "%.1f ms waiting for the device, %.1f ms of routed experts on the host, %.1f MiB on the device\n",
+ ch->forwards, st.frames, st.ops, st.matmuls, st.gemms, st.wait_ms, ch->host_ms, st.dev_bytes / 1048576.0);
+ vkc_prof_print();
+}
+
+/* COLI_VK_CHAIN, decided after the tier (ink_vk_tier_start): CUDA and Metal keep their
+ * priority; the chain's pipelines, and its teardown registered after the tier's so
+ * that it runs first at exit. */
+static void inkc_start(Model *m) {
+ (void)m;
+ if (!g_vk_ready) return;
+ const char *e = getenv("COLI_VK_CHAIN");
+#ifdef COLI_CUDA
+ if (g_cuda) { if (e && *e && *e != '0') fprintf(stderr, "[VK] inkling: dense chain off, the CUDA backend is on\n"); return; }
+#endif
+#ifdef COLI_METAL
+ if (g_metal) { if (e && *e && *e != '0') fprintf(stderr, "[VK] inkling: dense chain off, the Metal expert path is on\n"); return; }
+#endif
+ (void)e;
+ g_vk_chain = coli_vk_chain_decide("inkling", vkt_ready(), INKLING_CHAIN_IGPU);
+ if (g_vk_chain && !vkc_init()) g_vk_chain = 0;
+ if (g_vk_chain) atexit(vkc_shutdown);
+}
diff --git a/c/tools/make_tiny_inkling.py b/c/tools/make_tiny_inkling.py
index 73205792..62f5b4a1 100644
--- a/c/tools/make_tiny_inkling.py
+++ b/c/tools/make_tiny_inkling.py
@@ -10,7 +10,12 @@ The tiny config exercises every architectural branch of the big model:
sliding + global layers, log-length tau scaling (n_floor set below the
sequence length), dense + sparse MLP layers, 2 shared experts, unpadded vocab.
-Usage: python3 make_tiny_inkling.py <outdir>
+Usage: python3 make_tiny_inkling.py <outdir> [--wide]
+
+--wide keeps the real model's hidden size, 6144, and head dim, 128, in two layers (a
+sliding one with a dense MLP, a global one with the experts): the size the Vulkan
+expert batch stages its input at, and the dense chain's matrices, norms and
+convolutions at the width they run at.
"""
import json
import sys
@@ -24,37 +29,69 @@ except ImportError:
def main():
- out = sys.argv[1] if len(sys.argv) > 1 else "tiny_inkling"
+ args = [a for a in sys.argv[1:] if not a.startswith("--")]
+ wide = "--wide" in sys.argv[1:]
+ out = args[0] if args else "tiny_inkling"
torch.manual_seed(0)
- cfg = InklingTextConfig(
- vocab_size=256,
- unpadded_vocab_size=250,
- hidden_size=64,
- num_hidden_layers=8,
- num_attention_heads=4,
- num_key_value_heads=2,
- head_dim=16,
- swa_num_attention_heads=4,
- swa_num_key_value_heads=4,
- swa_head_dim=16,
- sliding_window_size=8,
- d_rel=8,
- rel_extent=32,
- log_scaling_n_floor=8, # small, so tau != 1 is exercised
- log_scaling_alpha=0.1,
- local_layer_ids=[0, 1, 2, 3, 4, 6, 7], # global layer at 5
- dense_mlp_idx=2,
- dense_intermediate_size=96,
- moe_intermediate_size=32,
- n_routed_experts=8,
- num_experts_per_tok=2,
- n_shared_experts=2,
- route_scale=2.0,
- logits_mup_width_multiplier=4.0,
- max_position_embeddings=4096,
- eos_token_id=None,
- )
+ if wide:
+ cfg = InklingTextConfig(
+ vocab_size=256,
+ unpadded_vocab_size=250,
+ hidden_size=6144,
+ num_hidden_layers=2,
+ num_attention_heads=8,
+ num_key_value_heads=2,
+ head_dim=128,
+ swa_num_attention_heads=8,
+ swa_num_key_value_heads=4,
+ swa_head_dim=128,
+ sliding_window_size=8,
+ d_rel=16,
+ rel_extent=32,
+ log_scaling_n_floor=8,
+ log_scaling_alpha=0.1,
+ local_layer_ids=[0], # global layer at 1
+ dense_mlp_idx=1,
+ dense_intermediate_size=128,
+ moe_intermediate_size=64,
+ n_routed_experts=8,
+ num_experts_per_tok=2,
+ n_shared_experts=2,
+ route_scale=2.0,
+ logits_mup_width_multiplier=4.0,
+ max_position_embeddings=4096,
+ eos_token_id=None,
+ )
+ else:
+ cfg = InklingTextConfig(
+ vocab_size=256,
+ unpadded_vocab_size=250,
+ hidden_size=64,
+ num_hidden_layers=8,
+ num_attention_heads=4,
+ num_key_value_heads=2,
+ head_dim=16,
+ swa_num_attention_heads=4,
+ swa_num_key_value_heads=4,
+ swa_head_dim=16,
+ sliding_window_size=8,
+ d_rel=8,
+ rel_extent=32,
+ log_scaling_n_floor=8, # small, so tau != 1 is exercised
+ log_scaling_alpha=0.1,
+ local_layer_ids=[0, 1, 2, 3, 4, 6, 7], # global layer at 5
+ dense_mlp_idx=2,
+ dense_intermediate_size=96,
+ moe_intermediate_size=32,
+ n_routed_experts=8,
+ num_experts_per_tok=2,
+ n_shared_experts=2,
+ route_scale=2.0,
+ logits_mup_width_multiplier=4.0,
+ max_position_embeddings=4096,
+ eos_token_id=None,
+ )
model = InklingForCausalLM(cfg).eval().float()