Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(vulkan): inkling's partial dense chain, the first N layers on the device

JustVugg35ad8a4de8a5
diff --git a/c/inkling.c b/c/inkling.c
index 0b89b021..6cab3008 100644
--- a/c/inkling.c
+++ b/c/inkling.c
@@ -77,10 +77,14 @@ static int g_metal = 0;
* keeps its own bf16 residents, and with CUDA (or Metal) on the tier stays off. */
#include "backend_vulkan.h"
#include "vk_tier.h"
+#include "vk_chain.h"
static int g_vk_ready = 0;
static int g_vk_tier_try = 0; /* the device opened with the expert tier to be tried */
static int g_vk_chain = 0; /* COLI_VK_CHAIN decided on, and the chain's pipelines are up (inkling_chain.h) */
static int g_ink_dho = 0; /* COLI_VK_DENSE_HOST: the dense matrices on the device only (ink_dho_place) */
+/* The partial chain's fit (inkc_fit_start, inkling_chain.h): the first n of L layers on the
+ * device; L = 0 while there is none (the chain off, or one that declines as before). */
+static VkcFit g_inkc_fit;
#endif
#if defined(__APPLE__)
#include <mach/mach.h>
@@ -1050,9 +1054,10 @@ static void ink_vk_mark(Wt *w, int views, int *by, int *cpu) {
int fmt = ink_vk_attach(w, views);
if (fmt) by[fmt]++; else (*cpu)++;
}
-static void ink_vk_mark_dense(Model *m, int layer_begin, int layer_end) {
+/* head 0: lm_head stays on the CPU (a partial chain's, inkc_fit_start) */
+static void ink_vk_mark_dense(Model *m, int layer_begin, int layer_end, int head) {
int by[13] = {0}, cpu = 0;
- ink_vk_mark(&m->lm_head, 1, by, &cpu);
+ if (head) ink_vk_mark(&m->lm_head, 1, by, &cpu);
for (int i = layer_begin; i < layer_end; i++) {
Layer *l = &m->L[i];
Wt *one[] = { &l->q, &l->k, &l->v, &l->r, &l->o, &l->dg, &l->du, &l->dd };
@@ -1136,9 +1141,9 @@ static int ink_dho_drop(Wt *w, int views, int64_t stride, int I, int O, int *n)
return 1;
}
static size_t ink_dho_count(const Wt *w, int64_t n, int I) { return ink_vk_fmt(w) ? ink_wt_bytes(w, n, I) : 0; }
-static size_t ink_dho_bytes(const Model *m, int layer_begin, int layer_end) {
+static size_t ink_dho_bytes(const Model *m, int layer_begin, int layer_end, int head) {
const Cfg *c = &m->c; int64_t D = c->hidden;
- size_t b = ink_dho_count(&m->lm_head, (int64_t)c->unpad_vocab * D, (int)D);
+ size_t b = head ? ink_dho_count(&m->lm_head, (int64_t)c->unpad_vocab * D, (int)D) : 0;
for (int i = layer_begin; i < layer_end; i++) {
const Layer *l = &m->L[i];
int64_t q = (int64_t)L_HEADS(c, i) * L_HD(c, i), kv = (int64_t)L_KV(c, i) * L_HD(c, i), I = c->moe_inter;
@@ -1151,21 +1156,34 @@ static size_t ink_dho_bytes(const Model *m, int layer_begin, int layer_end) {
}
return b;
}
+/* Layer i's matrices up (each view of a fused tensor its own) and their host copies given back. */
+static void ink_dho_drop_layer(Model *m, int i, int *n) {
+ const Cfg *c = &m->c; int D = c->hidden;
+ Layer *l = &m->L[i];
+ int q = L_HEADS(c, i) * L_HD(c, i), kv = L_KV(c, i) * L_HD(c, i), I = c->moe_inter, ns = c->n_shared;
+ ink_dho_drop(&l->q, 1, 0, D, q, n); ink_dho_drop(&l->k, 1, 0, D, kv, n); ink_dho_drop(&l->v, 1, 0, D, kv, n);
+ ink_dho_drop(&l->r, 1, 0, D, L_HEADS(c, i) * c->d_rel, n); ink_dho_drop(&l->o, 1, 0, q, D, n);
+ if (!c->sparse[i]) {
+ ink_dho_drop(&l->dg, 1, 0, D, c->dense_inter, n); ink_dho_drop(&l->du, 1, 0, D, c->dense_inter, n);
+ ink_dho_drop(&l->dd, 1, 0, c->dense_inter, D, n);
+ } else if (ns > 0) {
+ ink_dho_drop(&l->sh_g, ns, (int64_t)I * D, D, I, n); ink_dho_drop(&l->sh_u, ns, (int64_t)I * D, D, I, n);
+ ink_dho_drop(&l->sh_d, ns, (int64_t)D * I, I, D, n);
+ }
+}
+/* inkling_chain.h: the partial chain's fit, and its layers placed at start-up */
+static void inkc_fit_start(Model *m);
+static void inkc_place_init(Model *m, int *dropped);
static void ink_dho_place(Model *m, int layer_begin, int layer_end) {
const Cfg *c = &m->c; int D = c->hidden, n = 0;
- ink_dho_drop(&m->lm_head, 1, 0, D, c->unpad_vocab, &n);
- for (int i = layer_begin; i < layer_end; i++) {
- Layer *l = &m->L[i];
- int q = L_HEADS(c, i) * L_HD(c, i), kv = L_KV(c, i) * L_HD(c, i), I = c->moe_inter, ns = c->n_shared;
- ink_dho_drop(&l->q, 1, 0, D, q, &n); ink_dho_drop(&l->k, 1, 0, D, kv, &n); ink_dho_drop(&l->v, 1, 0, D, kv, &n);
- ink_dho_drop(&l->r, 1, 0, D, L_HEADS(c, i) * c->d_rel, &n); ink_dho_drop(&l->o, 1, 0, q, D, &n);
- if (!c->sparse[i]) {
- ink_dho_drop(&l->dg, 1, 0, D, c->dense_inter, &n); ink_dho_drop(&l->du, 1, 0, D, c->dense_inter, &n);
- ink_dho_drop(&l->dd, 1, 0, c->dense_inter, D, &n);
- } else if (ns > 0) {
- ink_dho_drop(&l->sh_g, ns, (int64_t)I * D, D, I, &n); ink_dho_drop(&l->sh_u, ns, (int64_t)I * D, D, I, &n);
- ink_dho_drop(&l->sh_d, ns, (int64_t)D * I, I, D, &n);
- }
+ if (g_inkc_fit.L) {
+ /* the chain's fit: its layers whole, each one's host copies given back once all of
+ * it is on the device (inkc_place_init), then lm_head when the tail went up too */
+ inkc_place_init(m, &n);
+ if (g_inkc_fit.n == g_inkc_fit.L && g_inkc_fit.tail) ink_dho_drop(&m->lm_head, 1, 0, D, c->unpad_vocab, &n);
+ } else {
+ ink_dho_drop(&m->lm_head, 1, 0, D, c->unpad_vocab, &n);
+ for (int i = layer_begin; i < layer_end; i++) ink_dho_drop_layer(m, i, &n);
}
coli_vk_dense_host_placed("inkling", "the embedding and audio tables (row lookups), the routers, norms, convolution taps, bias banks"
#ifdef HAVE_BF16_DOT
@@ -1192,14 +1210,28 @@ static void ink_vk_init_model(Model *m, int layer_begin, int layer_end) {
* shaders read is marked and goes up now, its host copy given back */
int chain = !other_gpu && layer_begin == 0 && layer_end == m->c.n_layers &&
coli_vk_chain_decide(NULL, g_vk_tier_try, INKLING_CHAIN_IGPU) != COLI_VK_CHAIN_OFF;
- if (coli_vk_dense_host_decide("inkling", (coli_vk_dense() || chain) && !other_gpu, ink_dho_bytes(m, layer_begin, layer_end))) {
+ /* the chain's fit before any upload and before the expert cache is sized: the first n
+ * layers on the device, the others (and lm_head, unless the tail went up) on the CPU
+ * with their host copies, the per-matrix path leaving them there */
+ if (chain) inkc_fit_start(m);
+ int end = g_inkc_fit.L ? g_inkc_fit.n : layer_end, head = g_inkc_fit.L ? g_inkc_fit.tail : 1;
+ int on_device = (coli_vk_dense() || chain) && !other_gpu && !(g_inkc_fit.L && !g_inkc_fit.n);
+ if (coli_vk_dense_host_decide("inkling", on_device, ink_dho_bytes(m, layer_begin, end, head))) {
g_ink_dho = 1; g_ink_dho_model = m;
- ink_vk_mark_dense(m, layer_begin, layer_end);
- ink_dho_place(m, layer_begin, layer_end);
+ if (g_inkc_fit.L) coli_vk_dense_host_layers(g_inkc_fit.n, g_inkc_fit.L);
+ ink_vk_mark_dense(m, layer_begin, end, head);
+ ink_dho_place(m, layer_begin, end);
+ return;
+ }
+ if (g_inkc_fit.L && vkc_fit_partial(&g_inkc_fit)) {
+ /* a partial chain: its layers up now, before the expert cache and the tier size
+ * themselves; the per-matrix path shares their copies where it runs */
+ if (coli_vk_dense()) ink_vk_mark_dense(m, 0, end, 0);
+ inkc_place_init(m, NULL);
return;
}
if (!coli_vk_dense()) return; /* nothing marked: the CPU computes the dense matrices */
- ink_vk_mark_dense(m, layer_begin, layer_end);
+ ink_vk_mark_dense(m, layer_begin, layer_end, 1);
}
#endif
@@ -1859,9 +1891,12 @@ static int ink_vk_in_ram(void *ctx, int layer, int e) {
}
/* Bytes the dense matrices marked for the device will take there (the budget leaves
* them room), from the geometry: int8 1 byte, int4-g64 0.5625, f32 4, bf16 2. */
+static size_t inkc_fit_later(const Model *m); /* inkling_chain.h */
static size_t ink_vk_dense_bytes(const Model *m) {
const Cfg *c = &m->c;
double b = 0;
+ if (g_inkc_fit.L && vkc_fit_partial(&g_inkc_fit)) /* a partial chain: its layers placed already */
+ return inkc_fit_later(m);
if (g_ink_dho) return 0; /* placed already (ink_dho_place): the free memory the tier reads counts them */
#define INK_VKB(w, n) do { const Wt *w_ = (w); int f_ = w_->vk ? ink_vk_fmt(w_) : 0; \
b += (double)(n) * (f_ == 1 ? 1.0 : f_ == 4 ? 0.5625 : f_ == 10 ? 4.0 : f_ == 11 ? 2.0 : 0.0); } while (0)
@@ -1959,7 +1994,8 @@ static void ink_vk_tier_start(Model *m) {
}
if (!vkt_ready() && !coli_vk_dense() && coli_vk_dense_decide("inkling", 0, 1)) { /* no tier after all */
g_vk_tier_try = 0;
- ink_vk_mark_dense(m, 0, c->n_layers);
+ /* a partial chain's layers are up already (the chain's copies): the rest stays on the CPU */
+ if (!vkc_fit_partial(&g_inkc_fit)) ink_vk_mark_dense(m, 0, c->n_layers, 1);
}
g_vk_tier_try = vkt_ready();
}
@@ -2594,16 +2630,19 @@ static float *step_mm(Model *m, const int *ids, int S, int pos0, int *tf_out,
/* COLI_VK_CHAIN: the layers, the final norm and lm_head on the device; x comes
* back only when the per-position heads below read every row */
float *chain_logit = NULL;
+ int layer0 = 0; /* the first layer the CPU runs */
if (g_vk_chain) {
chain_logit = falloc(c->unpad_vocab);
- if (!inkc_forward(m, x, S, pos0, (g_echo_k > 0 && g_echo_id && S > 1) || tf_out != NULL, chain_logit)) {
- free(chain_logit); chain_logit = NULL;
- inkc_cpu_step(m, pos0, S);
- }
+ int took = inkc_forward(m, x, S, pos0, (g_echo_k > 0 && g_echo_id && S > 1) || tf_out != NULL, chain_logit);
+ if (took != 1) { free(chain_logit); chain_logit = NULL; }
+ if (!took) inkc_cpu_step(m, pos0, S);
+ else if (took == 2) layer0 = inkc_layers(m); /* a partial chain: x is the residual after its last layer */
}
if (!chain_logit)
-#endif
+ inkling_layers_forward_range(m, x, S, pos0, layer0, c->n_layers);
+#else
inkling_layers_forward_range(m, x, S, pos0, 0, c->n_layers);
+#endif
m->kv_len = pos0 + S;
/* record what was just fed, at the positions it went to (kv_prefix.h).
* Audio taints the record: every frame carries the same token id while the
diff --git a/c/inkling_chain.h b/c/inkling_chain.h
index cd074e06..103b0c55 100644
--- a/c/inkling_chain.h
+++ b/c/inkling_chain.h
@@ -58,7 +58,20 @@
* device's rows (chain_kvs.comp's form of chain_relattn) and on the CPU over the older
* positions from the host's canonical cache at once, the two merged through their
* softmax statistics. A split layer's mirror follows a watermark (vkc_kv_lower on a CPU
- * step) instead of the [dlo, dhi) ranges. Below the budget the mirrors are whole. */
+ * step) instead of the [dlo, dhi) ranges. Below the budget the mirrors are whole.
+ *
+ * A partial chain (vk_chain.h, vkc_fit; inkc_fit_start at start-up): when the dense
+ * layers do not all fit the device, the first n run here and the CPU runs the others and
+ * lm_head. Every chunk of a forward crosses the n layers; the residual rows then come down
+ * once (the frame that ends the chunk), and the CPU runs layers n.. over all the rows
+ * layer by layer, as its own forward does. The device's layers keep their K/V mirrors,
+ * rings and convolution states as above; the CPU's layers keep theirs on the host only,
+ * so a CPU step lowers and a lost device rebuilds the device's layers alone. A partial
+ * chain's matrices (and every layer's, with the host copies dropped) go up at start-up,
+ * before the expert cache and the tier size themselves; with every layer fitting and the
+ * host copies kept, at the first forward as before. Either way layer by layer: a layer
+ * that fails is freed whole and the chain keeps the layers before it. The per-matrix
+ * path never uploads a CPU layer's matrices or lm_head. */
#include "vk_chain.h"
#include "vk_kvsplit.h"
@@ -71,7 +84,8 @@
#define INKC_BOTH 1
typedef struct {
- int ok, failed;
+ int ok, failed, tried; /* tried: inkc_setup ran (ok: the chain runs) */
+ int nl, full; /* layers on the device (the first nl); lm_head there too */
int rows; /* scratch capacity in rows */
int max_t; float **hostK; /* the host cache the mirrors copy */
VkcBuf *prm; /* norms, bias banks, convolution taps */
@@ -101,6 +115,8 @@ static void inkc_fatal(const char *what) {
static int64_t inkc_cs_cells(const Cfg *c, int bank, int i) {
return (int64_t)(bank < 2 ? L_KV(c, i) * L_HD(c, i) : c->hidden) * (c->conv_k - 1);
}
+/* The layers the chain runs on the device: the first n of a partial chain's fit, else all. */
+static int inkc_layers(const Model *m) { return g_inkc_fit.L ? g_inkc_fit.n : m->c.n_layers; }
/* The device copy of a resident matrix or of a view of one (wt_off_i: a shared expert
* of the fused [ns][R][I] tensors): the per-matrix path's copy when it keeps a table
@@ -117,27 +133,48 @@ static ColiVkTensor *inkc_tensor(Wt view, int I, int O) {
return coli_vk_tensor_ensure(&t, data, sc, fmt, I, O, gs) ? t : NULL;
}
-/* The model's parameters on the device, its tensors resolved; NULL = the chain cannot run. */
-static InkChain *inkc_setup(Model *m) {
- InkChain *ch = (InkChain *)m->vkchain;
- if (ch) return ch->ok ? ch : NULL;
- ch = (InkChain *)calloc(1, sizeof *ch);
- if (!ch) return NULL;
- m->vkchain = ch;
- Cfg *c = &m->c; int L = c->n_layers, D = c->hidden, E = c->n_experts, ns = c->n_shared, CK = c->conv_k, dr = c->d_rel;
- for (int i = 0; i < L; i++) {
+/* A geometry the shaders take (head dim up to 256, a bias bank up to 64 wide, up to 9
+ * taps); say: the line when one does not. */
+static int inkc_geometry_ok(const Model *m, int say) {
+ const Cfg *c = &m->c; int dr = c->d_rel, CK = c->conv_k;
+ for (int i = 0; i < c->n_layers; i++) {
int H = L_HEADS(c, i), KV = L_KV(c, i), hd = L_HD(c, i);
if (hd > 256 || hd < 1 || KV < 1 || H % KV || dr < 1 || dr > 64 || L_EXT(c, i) < 1 || CK < 1 || CK > 9 ||
(c->local[i] && c->window < 1)) {
- fprintf(stderr, "[VK] inkling chain: a geometry its shaders do not take (layer %d: head dim %d, %d/%d heads, "
- "bias bank %d x %d, %d taps); per-matrix path\n", i, hd, H, KV, dr, L_EXT(c, i), CK);
- return NULL;
+ if (say)
+ fprintf(stderr, "[VK] inkling chain: a geometry its shaders do not take (layer %d: head dim %d, %d/%d heads, "
+ "bias bank %d x %d, %d taps); per-matrix path\n", i, hd, H, KV, dr, L_EXT(c, i), CK);
+ return 0;
}
}
- if (!vkc_sconv_ready() || !vkc_relattn_ready()) {
- fprintf(stderr, "[VK] inkling chain: chain_sconv.spv or chain_relattn.spv is missing; per-matrix path\n");
- return NULL;
+ return 1;
+}
+/* The scratch's geometry over every layer (the widest a frame may be) and the frames: a
+ * MoE layer ends one (its host step); ch->slot, when allocated, gets each layer's place. */
+static void inkc_geom(InkChain *ch, const Model *m) {
+ const Cfg *c = &m->c;
+ ch->kvo_max = ch->qo_max = ch->ro_max = ch->mi_max = ch->nslot = 0;
+ for (int i = 0, fr = 0; i < c->n_layers; i++) {
+ int H = L_HEADS(c, i), hd = L_HD(c, i), kvo = L_KV(c, i) * hd;
+ if (kvo > ch->kvo_max) ch->kvo_max = kvo;
+ if (H * hd > ch->qo_max) ch->qo_max = H * hd;
+ if (H * c->d_rel > ch->ro_max) ch->ro_max = H * c->d_rel;
+ int mi = c->sparse[i] ? c->moe_inter : c->dense_inter;
+ if (mi > ch->mi_max) ch->mi_max = mi;
+ if (ch->slot) ch->slot[i] = fr;
+ fr++;
+ if (fr > ch->nslot) ch->nslot = fr;
+ if (c->sparse[i]) fr = 0;
}
+}
+/* The chain's state, its tables empty (inkc_setup and inkc_place_init fill them). */
+static InkChain *inkc_new(Model *m) {
+ InkChain *ch = (InkChain *)m->vkchain;
+ if (ch) return ch;
+ ch = (InkChain *)calloc(1, sizeof *ch);
+ if (!ch) return NULL;
+ m->vkchain = ch;
+ int L = m->c.n_layers, nsl = m->c.n_shared > 0 ? m->c.n_shared : 1;
#define INKC_ARR(f, T) (ch->f = calloc((size_t)L, sizeof(T)))
if (!INKC_ARR(o_in, size_t) || !INKC_ARR(o_post, size_t) || !INKC_ARR(o_qn, size_t) || !INKC_ARR(o_kn, size_t) ||
!INKC_ARR(o_relp, size_t) || !INKC_ARR(o_cw[0], size_t) || !INKC_ARR(o_cw[1], size_t) || !INKC_ARR(o_cw[2], size_t) ||
@@ -148,13 +185,224 @@ static InkChain *inkc_setup(Model *m) {
!INKC_ARR(cap, int) || !INKC_ARR(dlo, int) || !INKC_ARR(dhi, int) || !INKC_ARR(slot, int) ||
!INKC_ARR(kli, int) || !INKC_ARR(drows, int)) return NULL;
#undef INKC_ARR
- int nsl = ns > 0 ? ns : 1;
ch->t_sg = calloc((size_t)L * nsl, sizeof(void *)); ch->t_su = calloc((size_t)L * nsl, sizeof(void *));
ch->t_sd = calloc((size_t)L * nsl, sizeof(void *)); ch->o_csd = calloc((size_t)4 * L, sizeof(size_t));
if (!ch->t_sg || !ch->t_su || !ch->t_sd || !ch->o_csd) return NULL;
- /* the parameter arena: offsets, then one upload */
+ return ch;
+}
+
+/* ---- a partial chain (vk_chain.h, vkc_fit): the first n layers on the device ----------
+ * Layer i's matrices, all of them or none: q, k, v, the relative-bias projection r and
+ * o_proj; a dense layer's MLP, or a MoE layer's router (the chain's own f32 copy) and
+ * its shared experts. what: the first that did not go up. */
+static void inkc_unplace_layer(Model *m, InkChain *ch, int i);
+static int inkc_place_layer(Model *m, InkChain *ch, int i, const char **what) {
+ Cfg *c = &m->c; Layer *l = &m->L[i];
+ int D = c->hidden, E = c->n_experts, ns = c->n_shared, nsl = ns > 0 ? ns : 1, dr = c->d_rel;
+ int H = L_HEADS(c, i), hd = L_HD(c, i), kvo = L_KV(c, i) * hd, I = c->moe_inter;
+ *what = NULL;
+ if (!(ch->t_q[i] = inkc_tensor(l->q, D, H * hd)) || !(ch->t_k[i] = inkc_tensor(l->k, D, kvo)) ||
+ !(ch->t_v[i] = inkc_tensor(l->v, D, kvo)) || !(ch->t_r[i] = inkc_tensor(l->r, D, H * dr)) ||
+ !(ch->t_o[i] = inkc_tensor(l->o, H * hd, D))) *what = "an attention matrix";
+ else if (!c->sparse[i]) {
+ if (!(ch->t_dg[i] = inkc_tensor(l->dg, D, c->dense_inter)) || !(ch->t_du[i] = inkc_tensor(l->du, D, c->dense_inter)) ||
+ !(ch->t_dd[i] = inkc_tensor(l->dd, c->dense_inter, D))) *what = "a dense MLP matrix";
+ } else {
+ if (!coli_vk_tensor_ensure(&ch->t_router[i], l->router, NULL, 10, D, E + ns, 0)) *what = "the router";
+ for (int j = 0; j < ns && !*what; j++)
+ if (!(ch->t_sg[(size_t)i * nsl + j] = inkc_tensor(wt_off_i(l->sh_g, (int64_t)j * I * D, D), D, I)) ||
+ !(ch->t_su[(size_t)i * nsl + j] = inkc_tensor(wt_off_i(l->sh_u, (int64_t)j * I * D, D), D, I)) ||
+ !(ch->t_sd[(size_t)i * nsl + j] = inkc_tensor(wt_off_i(l->sh_d, (int64_t)j * D * I, I), I, D)))
+ *what = "a shared expert";
+ }
+ if (*what && g_inkc_fit.L) inkc_unplace_layer(m, ch, i); /* without a fit the chain declines as before */
+ return *what == NULL;
+}
+/* Everything layer i placed, freed: the chain's own copies, and the per-matrix path's
+ * tables of its tensors (their copies with them: the CPU runs the layer). */
+static void inkc_unplace_layer(Model *m, InkChain *ch, int i) {
+ Cfg *c = &m->c; Layer *l = &m->L[i]; int nsl = c->n_shared > 0 ? c->n_shared : 1;
+ if (vkc_ready()) vkc_finish();
+ struct { ColiVkTensor **t; Wt *w; } own[] = {{&ch->t_q[i], &l->q}, {&ch->t_k[i], &l->k}, {&ch->t_v[i], &l->v},
+ {&ch->t_r[i], &l->r}, {&ch->t_o[i], &l->o}, {&ch->t_dg[i], &l->dg}, {&ch->t_du[i], &l->du}, {&ch->t_dd[i], &l->dd}};
+ for (size_t k = 0; k < sizeof own / sizeof own[0]; k++) {
+ if (*own[k].t && !own[k].w->vk) coli_vk_tensor_free(*own[k].t); /* a table's copy goes with its table */
+ *own[k].t = NULL;
+ }
+ for (int j = 0; j < nsl; j++) {
+ ColiVkTensor **t3[3] = {&ch->t_sg[(size_t)i * nsl + j], &ch->t_su[(size_t)i * nsl + j], &ch->t_sd[(size_t)i * nsl + j]};
+ Wt *w3[3] = {&l->sh_g, &l->sh_u, &l->sh_d};
+ for (int k = 0; k < 3; k++) { if (*t3[k] && !w3[k]->vk) coli_vk_tensor_free(*t3[k]); *t3[k] = NULL; }
+ }
+ if (ch->t_router[i]) coli_vk_tensor_free(ch->t_router[i]);
+ ch->t_router[i] = NULL;
+ Wt *tab[] = {&l->q, &l->k, &l->v, &l->r, &l->o, &l->dg, &l->du, &l->dd, &l->sh_g, &l->sh_u, &l->sh_d};
+ for (size_t k = 0; k < sizeof tab / sizeof tab[0]; k++) if (tab[k]->vk && !((InkVk *)tab[k]->vk)->gone) ink_vk_release(tab[k]);
+}
+/* Layers from..L-1 and lm_head refused to the device: their tables go (the per-matrix
+ * path computes them on the CPU, from their host copies). */
+static void inkc_refuse_from(Model *m, int from) {
+ for (int i = from; i < m->c.n_layers; i++) {
+ Layer *l = &m->L[i];
+ Wt *tab[] = {&l->q, &l->k, &l->v, &l->r, &l->o, &l->dg, &l->du, &l->dd, &l->sh_g, &l->sh_u, &l->sh_d};
+ for (size_t k = 0; k < sizeof tab / sizeof tab[0]; k++)
+ if (tab[k]->vk && !((InkVk *)tab[k]->vk)->gone) ink_vk_release(tab[k]);
+ }
+ if (m->lm_head.vk && !((InkVk *)m->lm_head.vk)->gone) ink_vk_release(&m->lm_head);
+}
+/* Layer i failed during setup: free it, refuse it and the layers after it, and go on with
+ * the layers before it (vkc_fit_shrink says so). */
+static void inkc_shrink(Model *m, InkChain *ch, int i, const char *what) {
+ char why[96];
+ snprintf(why, sizeof why, "%s did not go up", what ? what : "a matrix");
+ inkc_unplace_layer(m, ch, i);
+ inkc_refuse_from(m, i);
+ vkc_fit_shrink("inkling", &g_inkc_fit, i, why);
+}
+
+/* The fit (vkc_fit), at start-up before any upload and before the expert cache is sized.
+ * A layer's bytes: its matrices in the form each goes up in (the shared experts' views one
+ * tensor each), its share of the parameter arena (norms, bias bank, convolution taps), its
+ * four convolution rings and their place in the state's read-back, and its K/V mirror at
+ * the window's rows (a global layer's grows with the context: the KV split covers that).
+ * Fixed: the final norm and the scratch of one prompt chunk of vkc_fit_rows(256) rows, the
+ * residual's read-back included. Tail: lm_head. No fit when the chain would decline anyway
+ * (a geometry outside its shaders, a matrix in no form the shaders read as this CPU does,
+ * bf16 under a rounding bf16 dot included): it says so at its first forward, as before. */
+static long long g_inkc_count = -1;
+static int g_inkc_fitcount; /* the counting pass rounds each buffer as the chain's pools do */
+static int g_inkc_placed; /* vkc_fit_placed said it (at start-up, or at the chain's first forward) */
+static int inkc_bufs(InkChain *ch, Model *m, int rows);
+static void inkc_fit_start(Model *m) {
+ Cfg *c = &m->c; int L = c->n_layers, D = c->hidden, E = c->n_experts, ns = c->n_shared, CK = c->conv_k, dr = c->d_rel;
+ if (L < 1 || !inkc_geometry_ok(m, 0)) return;
+ size_t *per = calloc((size_t)L, sizeof *per), *mat = calloc((size_t)L, sizeof *mat);
+ int ok = per && mat;
+#define INKC_FITW(w, In, Out) do { const Wt *w_ = (w); int f_ = ink_vk_fmt(w_), g_ = w_->q4 ? w_->gs : 0; \
+ if (!f_) ok = 0; else { per[i] += vkc_fit_tensor(f_, (In), (Out), g_); mat[i] += coli_vk_tensor_payload(f_, (In), (Out), g_); } } while (0)
+ for (int i = 0; i < L && ok; i++) {
+ const Layer *l = &m->L[i];
+ int H = L_HEADS(c, i), KV = L_KV(c, i), hd = L_HD(c, i), kvo = KV * hd, I = c->moe_inter;
+ INKC_FITW(&l->q, D, H * hd); INKC_FITW(&l->k, D, kvo); INKC_FITW(&l->v, D, kvo);
+ INKC_FITW(&l->r, D, H * dr); INKC_FITW(&l->o, H * hd, D);
+ if (!c->sparse[i]) { INKC_FITW(&l->dg, D, c->dense_inter); INKC_FITW(&l->du, D, c->dense_inter); INKC_FITW(&l->dd, c->dense_inter, D); }
+ else {
+ per[i] += vkc_fit_tensor(10, D, E + ns, 0); mat[i] += coli_vk_tensor_payload(10, D, E + ns, 0);
+ for (int j = 0; j < ns; j++) { INKC_FITW(&l->sh_g, D, I); INKC_FITW(&l->sh_u, D, I); INKC_FITW(&l->sh_d, I, D); }
+ }
+ per[i] += ((size_t)2 * D + 2 * (size_t)hd + (size_t)dr * L_EXT(c, i) + (size_t)2 * kvo * CK + (size_t)2 * D * CK) * sizeof(float);
+ for (int b = 0; b < 4; b++) {
+ size_t cells = (size_t)inkc_cs_cells(c, b, i);
+ per[i] += vkc_fit_buf((cells ? cells : 1) * sizeof(float)) + cells * sizeof(float);
+ }
+ int rows0 = c->window > 0 ? c->window : 1;
+ per[i] += 2 * vkc_fit_buf((size_t)KV * rows0 * hd * sizeof(float));
+ }
+#undef INKC_FITW
+ if (!ink_vk_fmt(&m->lm_head)) ok = 0;
+ if (!ok) { /* a matrix the shaders do not read as this CPU does: the chain declines as before */
+ free(per); free(mat);
+ return;
+ }
+ const Wt *h = &m->lm_head;
+ int hf = ink_vk_fmt(h);
+ size_t tail = vkc_fit_tensor(hf, D, c->unpad_vocab, h->q4 ? h->gs : 0);
+ InkChain g; memset(&g, 0, sizeof g);
+ inkc_geom(&g, m);
+ int R = vkc_fit_rows(256);
+ g_inkc_count = 0; g_inkc_fitcount = 1;
+ inkc_bufs(&g, m, R);
+ size_t fixed = (size_t)g_inkc_count + vkc_fit_buf((size_t)R * D * sizeof(float)) + vkc_fit_buf((size_t)D * sizeof(float));
+ g_inkc_count = -1; g_inkc_fitcount = 0;
+ vkc_fit("inkling", L, per, mat, fixed, tail, &g_inkc_fit);
+ free(per); free(mat);
+}
+/* What a partial chain still allocates on the device after its layers went up (the
+ * tier's dense_bytes): their state and parameters, the layer bytes less the matrices. */
+static size_t inkc_fit_later(const Model *m) {
+ const Cfg *c = &m->c; size_t b = 0;
+ for (int i = 0; i < g_inkc_fit.n; i++) {
+ const Layer *l = &m->L[i];
+ int H = L_HEADS(c, i), hd = L_HD(c, i), kvo = L_KV(c, i) * hd, D = c->hidden, I = c->moe_inter;
+ size_t w = 0;
+#define INKC_LW(x, In, Out) do { const Wt *x_ = (x); InkVk *v_ = x_->vk; int f_ = v_ && v_->gone ? v_->fmt : ink_vk_fmt(x_); \
+ w += vkc_fit_tensor(f_, (In), (Out), v_ && v_->gone ? v_->gs : x_->q4 ? x_->gs : 0); } while (0)
+ INKC_LW(&l->q, D, H * hd); INKC_LW(&l->k, D, kvo); INKC_LW(&l->v, D, kvo);
+ INKC_LW(&l->r, D, H * c->d_rel); INKC_LW(&l->o, H * hd, D);
+ if (!c->sparse[i]) { INKC_LW(&l->dg, D, c->dense_inter); INKC_LW(&l->du, D, c->dense_inter); INKC_LW(&l->dd, c->dense_inter, D); }
+ else {
+ w += vkc_fit_tensor(10, D, c->n_experts + c->n_shared, 0);
+ for (int j = 0; j < c->n_shared; j++) { INKC_LW(&l->sh_g, D, I); INKC_LW(&l->sh_u, D, I); INKC_LW(&l->sh_d, I, D); }
+ }
+#undef INKC_LW
+ b += g_inkc_fit.per[i] > w ? g_inkc_fit.per[i] - w : 0;
+ }
+ return b;
+}
+/* A fit's layers up at start-up (a partial chain, or the dense weights on the device only):
+ * layer by layer, each whole or not at all; dropped: with COLI_VK_DENSE_HOST, each layer's
+ * host copies given back once all of it is on the device (counted there). */
+static void inkc_place_init(Model *m, int *dropped) {
+ InkChain *ch = g_inkc_fit.n ? inkc_new(m) : NULL;
+ if (g_inkc_fit.n && !ch) {
+ inkc_refuse_from(m, 0);
+ vkc_fit_shrink("inkling", &g_inkc_fit, 0, "no host memory for the chain's tables");
+ }
+ for (int i = 0; ch && i < g_inkc_fit.n; i++) {
+ const char *what = NULL;
+ if (!inkc_place_layer(m, ch, i, &what)) { inkc_shrink(m, ch, i, what); break; }
+ if (dropped) ink_dho_drop_layer(m, i, dropped);
+ vkc_fit_mark(&g_inkc_fit, i);
+ }
+ if (vkc_fit_partial(&g_inkc_fit)) inkc_refuse_from(m, g_inkc_fit.n); /* the CPU's layers and lm_head stay there */
+ vkc_fit_placed("inkling", &g_inkc_fit);
+ g_inkc_placed = 1;
+}
+
+/* The model's parameters on the device, its tensors resolved; NULL = the chain cannot run.
+ * With a fit the first n layers only (placed at start-up when the chain is partial or the
+ * host copies go; here at the first forward otherwise), and lm_head when the tail is on. */
+static void inkc_free_own(Model *m, InkChain *ch);
+static InkChain *inkc_setup(Model *m) {
+ InkChain *ch = (InkChain *)m->vkchain;
+ if (ch && ch->tried) return ch->ok ? ch : NULL;
+ if (!ch && !(ch = inkc_new(m))) return NULL;
+ ch->tried = 1;
+ Cfg *c = &m->c; int L = c->n_layers, D = c->hidden, CK = c->conv_k, dr = c->d_rel;
+ if (!inkc_geometry_ok(m, 1)) return NULL;
+ if (!vkc_sconv_ready() || !vkc_relattn_ready()) {
+ fprintf(stderr, "[VK] inkling chain: chain_sconv.spv or chain_relattn.spv is missing; per-matrix path\n");
+ inkc_free_own(m, ch);
+ return NULL;
+ }
+ /* the tensors: the per-matrix path's device copies where it keeps them */
+ const char *what = NULL;
+ int fit = g_inkc_fit.L > 0;
+ for (int i = 0; i < inkc_layers(m) && !what; i++) {
+ if (ch->t_q[i]) continue; /* placed at start-up */
+ if (!inkc_place_layer(m, ch, i, &what)) {
+ if (fit) { inkc_shrink(m, ch, i, what); what = NULL; break; }
+ continue;
+ }
+ if (fit) vkc_fit_mark(&g_inkc_fit, i);
+ }
+ int nl = inkc_layers(m);
+ if (fit && !g_inkc_placed) { vkc_fit_placed("inkling", &g_inkc_fit); g_inkc_placed = 1; } /* placed here, lm_head next */
+ if (fit && vkc_fit_partial(&g_inkc_fit)) inkc_refuse_from(m, nl);
+ if (!what && nl == L && (!fit || g_inkc_fit.tail) && !(ch->t_lm = inkc_tensor(m->lm_head, D, c->unpad_vocab))) what = "lm_head";
+ if (what || !nl) {
+ if (what) {
+ int bf16 = m->L[0].q.h && !m->L[0].q.q4 && !ink_vk_fmt(&m->L[0].q);
+ fprintf(stderr, "[VK] inkling chain: %s did not reach the device%s; per-matrix path\n", what,
+ bf16 ? " (bf16: this CPU's bf16 dot rounds the activations, the device would not, so bf16 stays on the CPU)" : "");
+ }
+ inkc_free_own(m, ch);
+ return NULL;
+ }
+ ch->nl = nl; ch->full = ch->t_lm != NULL;
+ /* the parameter arena of the device's layers: offsets, then one upload */
size_t n = 0;
- for (int i = 0; i < L; i++) {
+ for (int i = 0; i < nl; i++) {
int hd = L_HD(c, i), kvo = L_KV(c, i) * hd;
ch->o_in[i] = n; n += D; ch->o_post[i] = n; n += D;
ch->o_qn[i] = n; n += hd; ch->o_kn[i] = n; n += hd;
@@ -164,8 +412,8 @@ static InkChain *inkc_setup(Model *m) {
}
ch->o_final = n; n += D;
float *arena = calloc(n, sizeof(float));
- if (!arena) return NULL;
- for (int i = 0; i < L; i++) {
+ if (!arena) { inkc_free_own(m, ch); return NULL; }
+ for (int i = 0; i < nl; i++) {
Layer *l = &m->L[i];
int hd = L_HD(c, i), kvo = L_KV(c, i) * hd;
memcpy(arena + ch->o_in[i], l->in_ln, D * sizeof(float));
@@ -182,68 +430,59 @@ static InkChain *inkc_setup(Model *m) {
ch->prm = vkc_buf(n * sizeof(float), VKC_DEV);
int ok = ch->prm && vkc_begin() && vkc_write(ch->prm, 0, arena, n * sizeof(float)) && vkc_submit(1);
free(arena);
- if (!ok) return NULL;
- /* the tensors: the per-matrix path's device copies where it keeps them */
- const char *what = NULL;
- for (int i = 0; i < L && !what; i++) {
- Layer *l = &m->L[i];
- int H = L_HEADS(c, i), hd = L_HD(c, i), kvo = L_KV(c, i) * hd, I = c->moe_inter;
- if (!(ch->t_q[i] = inkc_tensor(l->q, D, H * hd)) || !(ch->t_k[i] = inkc_tensor(l->k, D, kvo)) ||
- !(ch->t_v[i] = inkc_tensor(l->v, D, kvo)) || !(ch->t_r[i] = inkc_tensor(l->r, D, H * dr)) ||
- !(ch->t_o[i] = inkc_tensor(l->o, H * hd, D))) what = "an attention matrix";
- else if (!c->sparse[i]) {
- if (!(ch->t_dg[i] = inkc_tensor(l->dg, D, c->dense_inter)) || !(ch->t_du[i] = inkc_tensor(l->du, D, c->dense_inter)) ||
- !(ch->t_dd[i] = inkc_tensor(l->dd, c->dense_inter, D))) what = "a dense MLP matrix";
- } else {
- if (!coli_vk_tensor_ensure(&ch->t_router[i], l->router, NULL, 10, D, E + ns, 0)) what = "the router";
- for (int j = 0; j < ns && !what; j++)
- if (!(ch->t_sg[(size_t)i * nsl + j] = inkc_tensor(wt_off_i(l->sh_g, (int64_t)j * I * D, D), D, I)) ||
- !(ch->t_su[(size_t)i * nsl + j] = inkc_tensor(wt_off_i(l->sh_u, (int64_t)j * I * D, D), D, I)) ||
- !(ch->t_sd[(size_t)i * nsl + j] = inkc_tensor(wt_off_i(l->sh_d, (int64_t)j * D * I, I), I, D)))
- what = "a shared expert";
- }
- }
- if (!what && !(ch->t_lm = inkc_tensor(m->lm_head, D, c->unpad_vocab))) what = "lm_head";
- if (what) {
- int bf16 = m->L[0].q.h && !m->L[0].q.q4 && !ink_vk_fmt(&m->L[0].q);
- fprintf(stderr, "[VK] inkling chain: %s did not reach the device%s; per-matrix path\n", what,
- bf16 ? " (bf16: this CPU's bf16 dot rounds the activations, the device would not, so bf16 stays on the CPU)" : "");
- return NULL;
- }
+ if (!ok) { inkc_free_own(m, ch); return NULL; }
/* geometry of the scratch; the frames: a MoE layer ends one (its host step) */
- int fr = 0;
- for (int i = 0; i < L; i++) {
- int H = L_HEADS(c, i), hd = L_HD(c, i), kvo = L_KV(c, i) * hd;
- if (kvo > ch->kvo_max) ch->kvo_max = kvo;
- if (H * hd > ch->qo_max) ch->qo_max = H * hd;
- if (H * dr > ch->ro_max) ch->ro_max = H * dr;
- int mi = c->sparse[i] ? c->moe_inter : c->dense_inter;
- if (mi > ch->mi_max) ch->mi_max = mi;
- ch->slot[i] = fr++;
- if (fr > ch->nslot) ch->nslot = fr;
- if (c->sparse[i]) fr = 0;
+ inkc_geom(ch, m);
+ for (int i = 0; i < nl; i++)
for (int b = 0; b < 4; b++) {
ch->o_csd[(size_t)b * L + i] = ch->csd_n; ch->csd_n += (size_t)inkc_cs_cells(c, b, i);
- if (!(ch->ring[b][i] = vkc_buf((size_t)(inkc_cs_cells(c, b, i) > 0 ? inkc_cs_cells(c, b, i) : 1) * sizeof(float), VKC_DEV)))
+ if (!(ch->ring[b][i] = vkc_buf((size_t)(inkc_cs_cells(c, b, i) > 0 ? inkc_cs_cells(c, b, i) : 1) * sizeof(float), VKC_DEV))) {
+ inkc_free_own(m, ch);
return NULL;
+ }
}
- }
- for (int i = 0; i < L; i++) ch->kli[i] = c->local[i] ? -1 : ch->nglob++;
+ for (int i = 0; i < L; i++) ch->kli[i] = i < nl && !c->local[i] ? ch->nglob++ : -1;
ch->cs_where = INKC_HOST; ch->host_zero = 0;
ch->ok = 1;
- int nsp = 0; for (int i = 0; i < L; i++) nsp += c->sparse[i];
+ int nsp = 0; for (int i = 0; i < nl; i++) nsp += c->sparse[i];
fprintf(stderr, "[VK] inkling chain: %d layers on the device (%d MoE), %.1f MiB of parameters\n",
- L, nsp, n * 4 / 1048576.0);
+ nl, nsp, n * 4 / 1048576.0);
return ch;
}
+/* The chain declined after its layers went up: the copies only it holds (its own, not the
+ * per-matrix path's tables) are freed, with whatever device memory it took. */
+static void inkc_free_own(Model *m, InkChain *ch) {
+ if (vkc_ready()) vkc_finish();
+ int L = m->c.n_layers, nsl = m->c.n_shared > 0 ? m->c.n_shared : 1;
+ for (int i = 0; i < L; i++) {
+ Layer *l = &m->L[i];
+ ColiVkTensor **t[] = {&ch->t_q[i], &ch->t_k[i], &ch->t_v[i], &ch->t_r[i], &ch->t_o[i], &ch->t_dg[i], &ch->t_du[i], &ch->t_dd[i]};
+ Wt *w[] = {&l->q, &l->k, &l->v, &l->r, &l->o, &l->dg, &l->du, &l->dd};
+ for (size_t k = 0; k < sizeof t / sizeof t[0]; k++) { if (*t[k] && !w[k]->vk) coli_vk_tensor_free(*t[k]); *t[k] = NULL; }
+ for (int j = 0; j < nsl; j++) {
+ ColiVkTensor **s[3] = {&ch->t_sg[(size_t)i * nsl + j], &ch->t_su[(size_t)i * nsl + j], &ch->t_sd[(size_t)i * nsl + j]};
+ Wt *sw[3] = {&l->sh_g, &l->sh_u, &l->sh_d};
+ for (int k = 0; k < 3; k++) { if (*s[k] && !sw[k]->vk) coli_vk_tensor_free(*s[k]); *s[k] = NULL; }
+ }
+ if (ch->t_router[i]) coli_vk_tensor_free(ch->t_router[i]);
+ ch->t_router[i] = NULL;
+ for (int b = 0; b < 4; b++) { vkc_free(ch->ring[b][i]); ch->ring[b][i] = NULL; }
+ }
+ if (ch->t_lm && !m->lm_head.vk) coli_vk_tensor_free(ch->t_lm);
+ ch->t_lm = NULL;
+ vkc_free(ch->prm); ch->prm = NULL;
+}
-/* inkc_res counts instead of reserving while g_inkc_count >= 0 (the chunk's sizing) */
-static long long g_inkc_count = -1;
+/* inkc_res counts instead of reserving while g_inkc_count >= 0 (the chunk's sizing; the
+ * fit's, g_inkc_fitcount, each buffer as the pools round it) */
static int inkc_res(VkcBuf **b, size_t floats, int kind) {
- if (g_inkc_count >= 0) { g_inkc_count += (long long)(floats ? floats : 1) * (long long)sizeof(float); return 1; }
+ if (g_inkc_count >= 0) {
+ size_t by = (floats ? floats : 1) * sizeof(float);
+ g_inkc_count += (long long)(g_inkc_fitcount ? vkc_fit_buf(by) : by);
+ return 1;
+ }
return vkc_reserve(b, (floats ? floats : 1) * sizeof(float), kind);
}
-static int inkc_bufs(InkChain *ch, Model *m, int rows);
/* Prompt rows per chunk (vkc_chunk_rows): the chain's scratch a row, counted from the
* reservations, and the routed experts' outputs (the tier's rows, the CPU's
* contributions, and the host's sum). */
@@ -293,7 +532,7 @@ static int inkc_mirror(InkChain *ch, Model *m) {
}
int plan = ch->nglob ? vkc_kv_plan(&ch->ks, "inkling", ch->nglob, row, m->max_t, 1, 0, held) : 1;
if (!plan) { ch->max_t = 0; ch->hostK = NULL; return 0; }
- for (int i = 0; i < c->n_layers; i++) {
+ for (int i = 0; i < ch->nl; i++) {
vkc_free(ch->kc[i]); vkc_free(ch->vc[i]); ch->kc[i] = ch->vc[i] = NULL;
int cap = kv_ring_rows(c, i, m->max_t), hd = L_HD(c, i), KV = L_KV(c, i);
int split = ch->ks.on && ch->kli[i] >= 0;
@@ -345,7 +584,7 @@ static void inkc_cpu_step(Model *m, int pos_base, int S) {
InkChain *ch = (InkChain *)m->vkchain;
if (!ch || !ch->ok) return;
ch->cs_where = INKC_HOST; ch->host_zero = 0;
- for (int i = 0; i < m->c.n_layers; i++) {
+ for (int i = 0; i < ch->nl; i++) { /* the device's layers: the CPU's keep their state on the host */
if (ch->ks.on && ch->kli[i] >= 0) { vkc_kv_lower(&ch->ks, ch->kli[i], pos_base); continue; }
if (ch->dlo[i] >= ch->dhi[i]) { ch->dlo[i] = pos_base; ch->dhi[i] = pos_base + S; continue; }
if (pos_base < ch->dlo[i]) ch->dlo[i] = pos_base;
@@ -359,13 +598,13 @@ static void inkc_cpu_step(Model *m, int pos_base, int S) {
static int inkc_push_state(InkChain *ch, Model *m, int pos_base, int n) {
Cfg *c = &m->c; int ok = 1;
if (ch->cs_where == INKC_HOST) {
- for (int i = 0; i < c->n_layers && ok; i++) for (int b = 0; b < 4 && ok; b++) {
+ for (int i = 0; i < ch->nl && ok; i++) for (int b = 0; b < 4 && ok; b++) {
size_t nc = (size_t)inkc_cs_cells(c, b, i);
ok = ch->host_zero ? vkc_zero(ch->ring[b][i], 0, nc) : vkc_write(ch->ring[b][i], 0, m->cs[b][i], nc * sizeof(float));
}
ch->cs_where = INKC_BOTH;
}
- for (int i = 0; i < c->n_layers && ok; i++) {
+ for (int i = 0; i < ch->nl && ok; i++) {
if (ch->ks.on && ch->kli[i] >= 0) {
int KV = L_KV(c, i), hd = L_HD(c, i);
VkcKvPart pt[2] = {{KV, hd, m->K[i], (size_t)ch->cap[i] * hd, ch->kc[i], 0},
@@ -397,16 +636,18 @@ static int inkc_push_state(InkChain *ch, Model *m, int pos_base, int n) {
* prefix record names, and leave the chain off. */
static void inkc_recover(Model *m, int upto) {
InkChain *ch = (InkChain *)m->vkchain;
- Cfg *c = &m->c; int D = c->hidden;
+ Cfg *c = &m->c; int D = c->hidden, nl = ch ? ch->nl : c->n_layers;
g_vk_chain = 0;
if (ch) ch->failed = 1;
- for (int i = 0; i < c->n_layers; i++)
+ /* the device's layers only: a partial chain's CPU layers hold their state on the host,
+ * whole up to the lost step (they run after the device's, once per forward) */
+ for (int i = 0; i < nl; i++)
for (int b = 0; b < 4; b++) memset(m->cs[b][i], 0, (size_t)inkc_cs_cells(c, b, i) * sizeof(float));
if (upto <= 0) return;
if (m->kvp.tainted || m->kvp.len < upto || !m->kvp.fed)
inkc_fatal("the device was lost with a state its token ids do not describe (audio)");
fprintf(stderr, "[VK] inkling chain: the device was lost; rebuilding the state of %d positions on the CPU, "
- "which runs from here on\n", upto);
+ "which runs from here on%s\n", upto, nl < c->n_layers ? " (the device's layers; the CPU's have theirs)" : "");
int *ids = malloc((size_t)upto * sizeof(int));
float *x = falloc((int64_t)upto * D);
if (!ids) { fprintf(stderr, "OOM rebuilding the state\n"); exit(1); }
@@ -415,7 +656,7 @@ static void inkc_recover(Model *m, int upto) {
wt_row_f32(m->embed, (int64_t)ids[s] * D, x + (int64_t)s * D, D);
if (m->embed_norm) rmsnorm_row(x + (int64_t)s * D, x + (int64_t)s * D, m->embed_norm, D, c->eps);
}
- inkling_layers_forward_range(m, x, upto, 0, 0, c->n_layers);
+ inkling_layers_forward_range(m, x, upto, 0, 0, nl);
free(ids); free(x);
}
@@ -530,13 +771,18 @@ static int inkc_join(InkChain *ch, Model *m, int i, int n) {
/* Every layer for S rows from host rows xh, the last row's logits into `logit`; xh gets
* the final rows back when want_x. 0 = not taken: xh is as it was, and the caller runs
- * the step on the CPU (after inkc_recover when the device was lost). */
+ * the step on the CPU (after inkc_recover when the device was lost). 1 = taken. 2 = a
+ * partial chain's share taken: its first ch->nl layers ran here for every row, xh holds
+ * the residual after the last of them, and the caller runs the CPU's layers (from
+ * inkc_layers) and lm_head on it. */
static int inkc_forward(Model *m, float *xh, int S, int pos_base, int want_x, float *logit) {
if (!g_vk_chain) return 0;
if (g_vk_chain == COLI_VK_CHAIN_PREFILL && S <= 2) return 0; /* prompts only: decode on the CPU */
InkChain *ch = inkc_setup(m);
if (!ch || ch->failed) return 0;
- Cfg *c = &m->c; int D = c->hidden, L = c->n_layers, ET = c->n_experts + c->n_shared, ns = c->n_shared;
+ Cfg *c = &m->c; int D = c->hidden, L = ch->nl, ET = c->n_experts + c->n_shared, ns = c->n_shared;
+ int full = ch->full, LT = c->n_layers; /* full: every layer and lm_head here */
+ if (!full) want_x = 1; /* the CPU's layers read the residual */
int mirror_ok = inkc_mirror(ch, m);
int CH = mirror_ok ? inkc_chunk_rows(ch, m) : 1, rows = S < CH ? S : CH;
if (ch->ks.on && rows > ch->ks.chunk) rows = ch->ks.chunk; /* a step's rows fit the split's window */
@@ -593,11 +839,11 @@ static int inkc_forward(Model *m, float *xh, int S, int pos_base, int want_x, fl
pending = 1;
}
if (ok && pending) ok = inkc_join(ch, m, L - 1, n);
- if (ok && want_x) ok = vkc_copy(ch->xd, 0, ch->x, 0, (size_t)n * D);
+ if (ok && want_x) ok = vkc_copy(ch->xd, 0, ch->x, 0, (size_t)n * D); /* a partial chain: the handoff */
for (int i = 0; i < L && ok; i++) for (int b = 0; b < 4 && ok; b++) /* the convolution states, for the host */
- ok = vkc_copy(ch->csd, ch->o_csd[(size_t)b * L + i], ch->ring[b][i], 0, (size_t)inkc_cs_cells(c, b, i));
+ ok = vkc_copy(ch->csd, ch->o_csd[(size_t)b * LT + i], ch->ring[b][i], 0, (size_t)inkc_cs_cells(c, b, i));
int last = c0 + n == S;
- if (ok && last) {
+ if (ok && last && full) {
VkcSconv dv = {2, 0, 0, 0, 0, 0, 0, 0, D, c->mup};
ok = inkc_norm(ch->x, (size_t)(n - 1) * D, ch->prm, ch->o_final, ch->fin, 0, 1, D, c->eps) &&
vkc_sconv(ch->fin, NULL, NULL, &dv) && vkc_matmul(ch->t_lm, ch->fin, 0, ch->outd, 0, 1);
@@ -609,15 +855,15 @@ static int inkc_forward(Model *m, float *xh, int S, int pos_base, int want_x, fl
if (f0 < L) inkc_kv_down(ch, m, f0, L - 1, n, pb);
const float *csd = (const float *)vkc_ptr(ch->csd);
for (int i = 0; i < L; i++) for (int b = 0; b < 4; b++)
- memcpy(m->cs[b][i], csd + ch->o_csd[(size_t)b * L + i], (size_t)inkc_cs_cells(c, b, i) * sizeof(float));
+ memcpy(m->cs[b][i], csd + ch->o_csd[(size_t)b * LT + i], (size_t)inkc_cs_cells(c, b, i) * sizeof(float));
ch->cs_where = INKC_BOTH; ch->host_zero = 0;
for (int i = 0; i < L; i++) if (ch->kli[i] >= 0) vkc_kv_done(&ch->ks, ch->kli[i], pb + n);
if (want_x) memcpy(xfin + (size_t)c0 * D, vkc_ptr(ch->xd), (size_t)n * D * sizeof(float));
- if (last) memcpy(logit, vkc_ptr(ch->outd), (size_t)c->unpad_vocab * sizeof(float));
+ if (last && full) memcpy(logit, vkc_ptr(ch->outd), (size_t)c->unpad_vocab * sizeof(float));
}
if (want_x) { memcpy(xh, xfin, (size_t)S * D * sizeof(float)); free(xfin); }
ch->forwards++;
- return 1;
+ return full ? 1 : 2;
lost: /* a frame failed: the device is gone (or would not take a command); the CPU takes over */
if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost(); }
free(xfin);
@@ -651,6 +897,7 @@ static void inkc_start(Model *m) {
#endif
(void)e;
g_vk_chain = coli_vk_chain_decide("inkling", vkt_ready(), INKLING_CHAIN_IGPU);
+ if (g_inkc_fit.L && !g_inkc_fit.n) g_vk_chain = 0; /* the fit left no layer to the device (its line said so) */
if (g_vk_chain && !vkc_init()) g_vk_chain = 0;
if (g_vk_chain) atexit(vkc_shutdown);
}