Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

inkling: the chain's layers on two devices (COLI_VK_DEV2)

JustVugg1beae0acd4e8
diff --git a/c/inkling.c b/c/inkling.c
index bf6db7cc..590286db 100644
--- a/c/inkling.c
+++ b/c/inkling.c
@@ -203,6 +203,7 @@ typedef struct {
kv_prefix kvp;
#ifdef COLI_VULKAN
void *vkchain; /* the dense chain's device state (inkling_chain.h), NULL until it runs */
+ void *vkchain2; /* its layers on COLI_VK_DEV2's device, after the primary's (inkling_chain.h) */
#endif
} Model;
@@ -2646,7 +2647,7 @@ static float *step_mm(Model *m, const int *ids, int S, int pos0, int *tf_out,
int took = inkc_forward(m, x, S, pos0, (g_echo_k > 0 && g_echo_id && S > 1) || tf_out != NULL, chain_logit);
if (took != 1) { free(chain_logit); chain_logit = NULL; }
if (!took) inkc_cpu_step(m, pos0, S);
- else if (took == 2) layer0 = inkc_layers(m); /* a partial chain: x is the residual after its last layer */
+ else if (took == 2) layer0 = inkc_ran(); /* a partial chain: x is the residual after the last layer it ran */
}
if (!chain_logit)
inkling_layers_forward_range(m, x, S, pos0, layer0, c->n_layers);
diff --git a/c/inkling_chain.h b/c/inkling_chain.h
index b1c4a070..7ee6f938 100644
--- a/c/inkling_chain.h
+++ b/c/inkling_chain.h
@@ -85,7 +85,8 @@
typedef struct {
int ok, failed, tried; /* tried: inkc_setup ran (ok: the chain runs) */
- int nl, full; /* layers on the device (the first nl); lm_head there too */
+ int lo, d; /* its layers start at lo, on device d (vkc_device) */
+ int nl, full; /* layers on the device (lo..lo+nl-1); lm_head there too */
int rows; /* scratch capacity in rows */
int max_t; float **hostK; /* the host cache the mirrors copy */
VkcBuf *prm; /* norms, bias banks, convolution taps */
@@ -117,20 +118,32 @@ static int64_t inkc_cs_cells(const Cfg *c, int bank, int i) {
}
/* The layers the chain runs on the device: the first n of a partial chain's fit, else all. */
static int inkc_layers(const Model *m) { return g_inkc_fit.L ? g_inkc_fit.n : m->c.n_layers; }
+/* The layers after the primary's on COLI_VK_DEV2's device (docs/vulkan.md, "Layers on two
+ * devices"): a second chain from layer g_inkc_fit.n, its own fit (inkc_fit_start), its own
+ * copies of their matrices there (the per-matrix path leaves those layers to the CPU). A
+ * forward runs the primary's layers, brings the residual rows back and runs these; the CPU
+ * runs what is left. */
+static VkcFit g_inkc_fit2;
+static int g_inkc_fit2_on;
+static int g_inkc_ran; /* the layers the last forward ran on the devices, from layer 0 */
+static int inkc_ran(void) { return g_inkc_ran; }
+static InkChain *inkc_of(Model *m, int d) { return (InkChain *)(d ? m->vkchain2 : m->vkchain); }
+static const char *inkc_name(const InkChain *ch) { return ch && ch->d ? "inkling dev2" : "inkling"; }
+static int inkc_dev2_wanted(void) { const char *e = getenv("COLI_VK_CHAIN_DEV2"); return !(e && *e == '0'); }
/* The device copy of a resident matrix or of a view of one (wt_off_i: a shared expert
* of the fused [ns][R][I] tensors): the per-matrix path's copy when it keeps a table
* for the tensor (ink_vk_view_tensor, keyed by the view's first element in the tensor),
* else the chain's own. NULL: not a form the shaders read as this CPU does (ink_vk_fmt). */
-static ColiVkTensor *inkc_tensor(Wt view, int I, int O) {
- if (view.vk) return ink_vk_view_tensor(&view, I, O); /* the per-matrix path's table, keyed by the view */
+static ColiVkTensor *inkc_tensor(int d, Wt view, int I, int O) {
+ if (view.vk && !d) return ink_vk_view_tensor(&view, I, O); /* the per-matrix path's table, keyed by the view */
int fmt = ink_vk_fmt(&view);
const void *data = view.q4 ? (const void *)view.q4 : view.f ? (const void *)view.f : (const void *)view.h;
const float *sc = view.q4 ? view.qs : NULL;
int gs = view.q4 ? view.gs : 0;
if (!fmt || !data) return NULL;
- ColiVkTensor *t = NULL;
- return coli_vk_tensor_ensure(&t, data, sc, fmt, I, O, gs) ? t : NULL;
+ ColiVkTensor *t = NULL; /* the second device's: its own copy there */
+ return (d ? coli_vk_tensor_ensure2(&t, data, sc, fmt, I, O, gs) : coli_vk_tensor_ensure(&t, data, sc, fmt, I, O, gs)) ? t : NULL;
}
/* A geometry the shaders take (head dim up to 256, a bias bank up to 64 wide, up to 9
@@ -168,12 +181,13 @@ static void inkc_geom(InkChain *ch, const Model *m) {
}
}
/* The chain's state, its tables empty (inkc_setup and inkc_place_init fill them). */
-static InkChain *inkc_new(Model *m) {
- InkChain *ch = (InkChain *)m->vkchain;
+static InkChain *inkc_new_dev(Model *m, int d) {
+ InkChain *ch = inkc_of(m, d);
if (ch) return ch;
ch = (InkChain *)calloc(1, sizeof *ch);
if (!ch) return NULL;
- m->vkchain = ch;
+ if (d) m->vkchain2 = ch; else m->vkchain = ch;
+ ch->d = d; ch->lo = d ? g_inkc_fit.n : 0;
int L = m->c.n_layers, nsl = m->c.n_shared > 0 ? m->c.n_shared : 1;
#define INKC_ARR(f, T) (ch->f = calloc((size_t)L, sizeof(T)))
if (!INKC_ARR(o_in, size_t) || !INKC_ARR(o_post, size_t) || !INKC_ARR(o_qn, size_t) || !INKC_ARR(o_kn, size_t) ||
@@ -190,6 +204,7 @@ static InkChain *inkc_new(Model *m) {
if (!ch->t_sg || !ch->t_su || !ch->t_sd || !ch->o_csd) return NULL;
return ch;
}
+static InkChain *inkc_new(Model *m) { return inkc_new_dev(m, 0); }
/* ---- a partial chain (vk_chain.h, vkc_fit): the first n layers on the device ----------
* Layer i's matrices, all of them or none: q, k, v, the relative-bias projection r and
@@ -201,18 +216,20 @@ static int inkc_place_layer(Model *m, InkChain *ch, int i, const char **what) {
int D = c->hidden, E = c->n_experts, ns = c->n_shared, nsl = ns > 0 ? ns : 1, dr = c->d_rel;
int H = L_HEADS(c, i), hd = L_HD(c, i), kvo = L_KV(c, i) * hd, I = c->moe_inter;
*what = NULL;
- if (!(ch->t_q[i] = inkc_tensor(l->q, D, H * hd)) || !(ch->t_k[i] = inkc_tensor(l->k, D, kvo)) ||
- !(ch->t_v[i] = inkc_tensor(l->v, D, kvo)) || !(ch->t_r[i] = inkc_tensor(l->r, D, H * dr)) ||
- !(ch->t_o[i] = inkc_tensor(l->o, H * hd, D))) *what = "an attention matrix";
+ int d = ch->d;
+ if (!(ch->t_q[i] = inkc_tensor(d, l->q, D, H * hd)) || !(ch->t_k[i] = inkc_tensor(d, l->k, D, kvo)) ||
+ !(ch->t_v[i] = inkc_tensor(d, l->v, D, kvo)) || !(ch->t_r[i] = inkc_tensor(d, l->r, D, H * dr)) ||
+ !(ch->t_o[i] = inkc_tensor(d, l->o, H * hd, D))) *what = "an attention matrix";
else if (!c->sparse[i]) {
- if (!(ch->t_dg[i] = inkc_tensor(l->dg, D, c->dense_inter)) || !(ch->t_du[i] = inkc_tensor(l->du, D, c->dense_inter)) ||
- !(ch->t_dd[i] = inkc_tensor(l->dd, c->dense_inter, D))) *what = "a dense MLP matrix";
+ if (!(ch->t_dg[i] = inkc_tensor(d, l->dg, D, c->dense_inter)) || !(ch->t_du[i] = inkc_tensor(d, l->du, D, c->dense_inter)) ||
+ !(ch->t_dd[i] = inkc_tensor(d, l->dd, c->dense_inter, D))) *what = "a dense MLP matrix";
} else {
- if (!coli_vk_tensor_ensure(&ch->t_router[i], l->router, NULL, 10, D, E + ns, 0)) *what = "the router";
+ if (!(d ? coli_vk_tensor_ensure2(&ch->t_router[i], l->router, NULL, 10, D, E + ns, 0)
+ : coli_vk_tensor_ensure(&ch->t_router[i], l->router, NULL, 10, D, E + ns, 0))) *what = "the router";
for (int j = 0; j < ns && !*what; j++)
- if (!(ch->t_sg[(size_t)i * nsl + j] = inkc_tensor(wt_off_i(l->sh_g, (int64_t)j * I * D, D), D, I)) ||
- !(ch->t_su[(size_t)i * nsl + j] = inkc_tensor(wt_off_i(l->sh_u, (int64_t)j * I * D, D), D, I)) ||
- !(ch->t_sd[(size_t)i * nsl + j] = inkc_tensor(wt_off_i(l->sh_d, (int64_t)j * D * I, I), I, D)))
+ if (!(ch->t_sg[(size_t)i * nsl + j] = inkc_tensor(d, wt_off_i(l->sh_g, (int64_t)j * I * D, D), D, I)) ||
+ !(ch->t_su[(size_t)i * nsl + j] = inkc_tensor(d, wt_off_i(l->sh_u, (int64_t)j * I * D, D), D, I)) ||
+ !(ch->t_sd[(size_t)i * nsl + j] = inkc_tensor(d, wt_off_i(l->sh_d, (int64_t)j * D * I, I), I, D)))
*what = "a shared expert";
}
if (*what && g_inkc_fit.L) inkc_unplace_layer(m, ch, i); /* without a fit the chain declines as before */
@@ -225,14 +242,14 @@ static void inkc_unplace_layer(Model *m, InkChain *ch, int i) {
if (vkc_ready()) vkc_finish();
struct { ColiVkTensor **t; Wt *w; } own[] = {{&ch->t_q[i], &l->q}, {&ch->t_k[i], &l->k}, {&ch->t_v[i], &l->v},
{&ch->t_r[i], &l->r}, {&ch->t_o[i], &l->o}, {&ch->t_dg[i], &l->dg}, {&ch->t_du[i], &l->du}, {&ch->t_dd[i], &l->dd}};
- for (size_t k = 0; k < sizeof own / sizeof own[0]; k++) {
- if (*own[k].t && !own[k].w->vk) coli_vk_tensor_free(*own[k].t); /* a table's copy goes with its table */
+ for (size_t k = 0; k < sizeof own / sizeof own[0]; k++) { /* a table's copy goes with its table; the second device's are own */
+ if (*own[k].t && (ch->d || !own[k].w->vk)) coli_vk_tensor_free(*own[k].t);
*own[k].t = NULL;
}
for (int j = 0; j < nsl; j++) {
ColiVkTensor **t3[3] = {&ch->t_sg[(size_t)i * nsl + j], &ch->t_su[(size_t)i * nsl + j], &ch->t_sd[(size_t)i * nsl + j]};
Wt *w3[3] = {&l->sh_g, &l->sh_u, &l->sh_d};
- for (int k = 0; k < 3; k++) { if (*t3[k] && !w3[k]->vk) coli_vk_tensor_free(*t3[k]); *t3[k] = NULL; }
+ for (int k = 0; k < 3; k++) { if (*t3[k] && (ch->d || !w3[k]->vk)) coli_vk_tensor_free(*t3[k]); *t3[k] = NULL; }
}
if (ch->t_router[i]) coli_vk_tensor_free(ch->t_router[i]);
ch->t_router[i] = NULL;
@@ -257,7 +274,7 @@ static void inkc_shrink(Model *m, InkChain *ch, int i, const char *what) {
snprintf(why, sizeof why, "%s did not go up", what ? what : "a matrix");
inkc_unplace_layer(m, ch, i);
inkc_refuse_from(m, i);
- vkc_fit_shrink("inkling", &g_inkc_fit, i, why);
+ vkc_fit_shrink(inkc_name(ch), ch->d ? &g_inkc_fit2 : &g_inkc_fit, i - ch->lo, why);
}
/* The fit (vkc_fit), at start-up before any upload and before the expert cache is sized.
@@ -315,6 +332,22 @@ static void inkc_fit_start(Model *m) {
size_t fixed = (size_t)g_inkc_count + vkc_fit_buf((size_t)R * D * sizeof(float)) + vkc_fit_buf((size_t)D * sizeof(float));
g_inkc_count = -1; g_inkc_fitcount = 0;
vkc_fit("inkling", L, per, mat, fixed, tail, &g_inkc_fit);
+ /* the layers the primary leaves, on COLI_VK_DEV2's device: a fit of its own from layer
+ * n0 with that device's free memory, its pipelines up now (inkc_place_init places its
+ * layers; the head goes with the last one when there is room) */
+ int n0 = g_inkc_fit.n;
+ if (vkc_fit_partial(&g_inkc_fit) && n0 > 0 && n0 < L && inkc_dev2_wanted() && getenv("COLI_VK_DEV2") &&
+ coli_vk_dev2_open_env()) {
+ vkc_device(1);
+ int n2 = vkc_fit("inkling dev2", L - n0, per + n0, mat + n0, fixed, tail, &g_inkc_fit2);
+ if (n2 > 0 && !vkc_init()) {
+ fprintf(stderr, "[VK] inkling chain: the second device's pipelines did not come up; its layers stay on the CPU\n");
+ n2 = 0;
+ }
+ g_inkc_fit2_on = n2 > 0;
+ if (!g_inkc_fit2_on) g_inkc_fit2.tail = 0;
+ vkc_device(0);
+ }
free(per); free(mat);
}
/* What a partial chain still allocates on the device after its layers went up (the
@@ -348,14 +381,36 @@ static void inkc_place_init(Model *m, int *dropped) {
inkc_refuse_from(m, 0);
vkc_fit_shrink("inkling", &g_inkc_fit, 0, "no host memory for the chain's tables");
}
+ int n0 = g_inkc_fit.n;
for (int i = 0; ch && i < g_inkc_fit.n; i++) {
const char *what = NULL;
if (!inkc_place_layer(m, ch, i, &what)) { inkc_shrink(m, ch, i, what); break; }
if (dropped) ink_dho_drop_layer(m, i, dropped);
vkc_fit_mark(&g_inkc_fit, i);
}
- if (vkc_fit_partial(&g_inkc_fit)) inkc_refuse_from(m, g_inkc_fit.n); /* the CPU's layers and lm_head stay there */
+ int end = g_inkc_fit.n;
+ if (vkc_fit_partial(&g_inkc_fit)) inkc_refuse_from(m, end); /* the CPU's layers and lm_head stay there */
vkc_fit_placed("inkling", &g_inkc_fit);
+ if (g_inkc_fit2_on && g_inkc_fit.n < n0) {
+ /* the primary placed fewer layers than its fit: the second device's would not follow
+ * them, so they stay on the CPU too */
+ fprintf(stderr, "[VK] inkling chain: the primary device stopped before layer %d; layers %d..%d stay on the CPU, "
+ "not on the second device\n", n0, n0, n0 + g_inkc_fit2.n - 1);
+ g_inkc_fit2_on = 0;
+ vkc_device(1); vkc_shutdown(); vkc_device(0);
+ } else if (g_inkc_fit2_on) { /* the second device's layers: their own copies there, host copies kept */
+ vkc_device(1);
+ InkChain *c2 = inkc_new_dev(m, 1);
+ for (int i = n0; c2 && i < n0 + g_inkc_fit2.n; i++) {
+ const char *what = NULL;
+ if (!inkc_place_layer(m, c2, i, &what)) { inkc_shrink(m, c2, i, what); break; }
+ vkc_fit_mark(&g_inkc_fit2, i - n0);
+ }
+ if (!c2) g_inkc_fit2.n = 0;
+ vkc_fit_placed("inkling dev2", &g_inkc_fit2);
+ if (!g_inkc_fit2.n) { g_inkc_fit2_on = 0; vkc_shutdown(); }
+ vkc_device(0);
+ }
g_inkc_placed = 1;
}
@@ -363,10 +418,12 @@ static void inkc_place_init(Model *m, int *dropped) {
* With a fit the first n layers only (placed at start-up when the chain is partial or the
* host copies go; here at the first forward otherwise), and lm_head when the tail is on. */
static void inkc_free_own(Model *m, InkChain *ch);
-static InkChain *inkc_setup(Model *m) {
- InkChain *ch = (InkChain *)m->vkchain;
+/* d = 1: the second device's chain (its layers placed by inkc_place_init; device 1 current). */
+static InkChain *inkc_setup_dev(Model *m, int d) {
+ InkChain *ch = inkc_of(m, d);
if (ch && ch->tried) return ch->ok ? ch : NULL;
- if (!ch && !(ch = inkc_new(m))) return NULL;
+ if (d && !g_inkc_fit2_on) return NULL;
+ if (!ch && !(ch = inkc_new_dev(m, d))) return NULL;
ch->tried = 1;
Cfg *c = &m->c; int L = c->n_layers, D = c->hidden, CK = c->conv_k, dr = c->d_rel;
if (!inkc_geometry_ok(m, 1)) return NULL;
@@ -377,22 +434,23 @@ static InkChain *inkc_setup(Model *m) {
}
/* the tensors: the per-matrix path's device copies where it keeps them */
const char *what = NULL;
- int fit = g_inkc_fit.L > 0;
- for (int i = 0; i < inkc_layers(m) && !what; i++) {
+ VkcFit *fitp = d ? &g_inkc_fit2 : &g_inkc_fit;
+ int fit = fitp->L > 0, lo = ch->lo;
+ for (int i = lo; i < lo + (d ? g_inkc_fit2.n : inkc_layers(m)) && !what; i++) {
if (ch->t_q[i]) continue; /* placed at start-up */
if (!inkc_place_layer(m, ch, i, &what)) {
if (fit) { inkc_shrink(m, ch, i, what); what = NULL; break; }
continue;
}
- if (fit) vkc_fit_mark(&g_inkc_fit, i);
+ if (fit) vkc_fit_mark(fitp, i - lo);
}
- int nl = inkc_layers(m);
- if (fit && !g_inkc_placed) { vkc_fit_placed("inkling", &g_inkc_fit); g_inkc_placed = 1; } /* placed here, lm_head next */
- if (fit && vkc_fit_partial(&g_inkc_fit)) inkc_refuse_from(m, nl);
- if (!what && nl == L && (!fit || g_inkc_fit.tail) && !(ch->t_lm = inkc_tensor(m->lm_head, D, c->unpad_vocab))) {
+ int nl = d ? g_inkc_fit2.n : inkc_layers(m);
+ if (fit && !d && !g_inkc_placed) { vkc_fit_placed("inkling", &g_inkc_fit); g_inkc_placed = 1; } /* placed here, lm_head next */
+ if (fit && !d && vkc_fit_partial(&g_inkc_fit)) inkc_refuse_from(m, lo + nl);
+ if (!what && lo + nl == L && (!fit || fitp->tail) && !(ch->t_lm = inkc_tensor(d, m->lm_head, D, c->unpad_vocab))) {
if (!fit) what = "lm_head";
else { /* the fit's tail did not go up: every layer here, the head on the CPU */
- g_inkc_fit.tail = 0;
+ fitp->tail = 0;
inkc_refuse_from(m, L);
fprintf(stderr, "[VK] inkling chain: lm_head did not reach the device; it runs on the CPU, the layers here\n");
}
@@ -409,7 +467,7 @@ static InkChain *inkc_setup(Model *m) {
ch->nl = nl; ch->full = ch->t_lm != NULL;
/* the parameter arena of the device's layers: offsets, then one upload */
size_t n = 0;
- for (int i = 0; i < nl; i++) {
+ for (int i = lo; i < lo + nl; i++) {
int hd = L_HD(c, i), kvo = L_KV(c, i) * hd;
ch->o_in[i] = n; n += D; ch->o_post[i] = n; n += D;
ch->o_qn[i] = n; n += hd; ch->o_kn[i] = n; n += hd;
@@ -420,7 +478,7 @@ static InkChain *inkc_setup(Model *m) {
ch->o_final = n; n += D;
float *arena = calloc(n, sizeof(float));
if (!arena) { inkc_free_own(m, ch); return NULL; }
- for (int i = 0; i < nl; i++) {
+ for (int i = lo; i < lo + nl; i++) {
Layer *l = &m->L[i];
int hd = L_HD(c, i), kvo = L_KV(c, i) * hd;
memcpy(arena + ch->o_in[i], l->in_ln, D * sizeof(float));
@@ -440,7 +498,7 @@ static InkChain *inkc_setup(Model *m) {
if (!ok) { inkc_free_own(m, ch); return NULL; }
/* geometry of the scratch; the frames: a MoE layer ends one (its host step) */
inkc_geom(ch, m);
- for (int i = 0; i < nl; i++)
+ for (int i = lo; i < lo + nl; i++)
for (int b = 0; b < 4; b++) {
ch->o_csd[(size_t)b * L + i] = ch->csd_n; ch->csd_n += (size_t)inkc_cs_cells(c, b, i);
if (!(ch->ring[b][i] = vkc_buf((size_t)(inkc_cs_cells(c, b, i) > 0 ? inkc_cs_cells(c, b, i) : 1) * sizeof(float), VKC_DEV))) {
@@ -448,14 +506,17 @@ static InkChain *inkc_setup(Model *m) {
return NULL;
}
}
- for (int i = 0; i < L; i++) ch->kli[i] = i < nl && !c->local[i] ? ch->nglob++ : -1;
+ for (int i = 0; i < L; i++) ch->kli[i] = i >= lo && i < lo + nl && !c->local[i] ? ch->nglob++ : -1;
ch->cs_where = INKC_HOST; ch->host_zero = 0;
ch->ok = 1;
- int nsp = 0; for (int i = 0; i < nl; i++) nsp += c->sparse[i];
- fprintf(stderr, "[VK] inkling chain: %d layers on the device (%d MoE), %.1f MiB of parameters\n",
- nl, nsp, n * 4 / 1048576.0);
+ int nsp = 0; for (int i = lo; i < lo + nl; i++) nsp += c->sparse[i];
+ if (d) fprintf(stderr, "[VK] inkling chain: layers %d..%d on the second device (%d MoE)%s, %.1f MiB of parameters\n",
+ lo, lo + nl - 1, nsp, ch->full ? ", lm_head too" : "", n * 4 / 1048576.0);
+ else fprintf(stderr, "[VK] inkling chain: %d layers on the device (%d MoE), %.1f MiB of parameters\n",
+ nl, nsp, n * 4 / 1048576.0);
return ch;
}
+static InkChain *inkc_setup(Model *m) { return inkc_setup_dev(m, 0); }
/* The chain declined after its layers went up: the copies only it holds (its own, not the
* per-matrix path's tables) are freed, with whatever device memory it took. */
static void inkc_free_own(Model *m, InkChain *ch) {
@@ -507,7 +568,7 @@ static int inkc_chunk_rows(InkChain *ch, Model *m) {
}
row += (3 * (size_t)heads * (dim + 2) + ch->qo_max + ch->ro_max) * sizeof(float);
}
- return vkc_chunk_rows("inkling", row);
+ return vkc_chunk_rows(inkc_name(ch), row);
}
/* the scratch buffers for `rows` rows */
static int inkc_bufs(InkChain *ch, Model *m, int rows) {
@@ -537,9 +598,9 @@ static int inkc_mirror(InkChain *ch, Model *m) {
if (b > row) row = b;
held += b * (size_t)ch->drows[i];
}
- int plan = ch->nglob ? vkc_kv_plan(&ch->ks, "inkling", ch->nglob, row, m->max_t, 1, 0, held) : 1;
+ int plan = ch->nglob ? vkc_kv_plan(&ch->ks, inkc_name(ch), ch->nglob, row, m->max_t, 1, 0, held) : 1;
if (!plan) { ch->max_t = 0; ch->hostK = NULL; return 0; }
- for (int i = 0; i < ch->nl; i++) {
+ for (int i = ch->lo; i < ch->lo + ch->nl; i++) {
vkc_free(ch->kc[i]); vkc_free(ch->vc[i]); ch->kc[i] = ch->vc[i] = NULL;
int cap = kv_ring_rows(c, i, m->max_t), hd = L_HD(c, i), KV = L_KV(c, i);
int split = ch->ks.on && ch->kli[i] >= 0;
@@ -582,22 +643,27 @@ static int inkc_scratch(InkChain *ch, Model *m, int rows) {
/* ---- state between the host and the device ----------------------------------- */
/* The host wrote its convolution states (zero: state_reset's zeros). */
static void inkc_host_wrote(Model *m, int zero) {
- InkChain *ch = (InkChain *)m->vkchain;
- if (!ch || !ch->ok) return;
- ch->cs_where = INKC_HOST; ch->host_zero = zero;
+ for (int d = 0; d < 2; d++) {
+ InkChain *ch = inkc_of(m, d);
+ if (!ch || !ch->ok) continue;
+ ch->cs_where = INKC_HOST; ch->host_zero = zero;
+ }
}
-/* A CPU step of S rows from pos_base: its K/V rows and states are the host's alone. */
-static void inkc_cpu_step(Model *m, int pos_base, int S) {
- InkChain *ch = (InkChain *)m->vkchain;
+/* A CPU step of S rows from pos_base over one chain's layers: their K/V rows and states are
+ * the host's alone. */
+static void inkc_cpu_one(InkChain *ch, int pos_base, int S) {
if (!ch || !ch->ok) return;
ch->cs_where = INKC_HOST; ch->host_zero = 0;
- for (int i = 0; i < ch->nl; i++) { /* the device's layers: the CPU's keep their state on the host */
+ for (int i = ch->lo; i < ch->lo + ch->nl; i++) { /* the device's layers: the CPU's keep their state on the host */
if (ch->ks.on && ch->kli[i] >= 0) { vkc_kv_lower(&ch->ks, ch->kli[i], pos_base); continue; }
if (ch->dlo[i] >= ch->dhi[i]) { ch->dlo[i] = pos_base; ch->dhi[i] = pos_base + S; continue; }
if (pos_base < ch->dlo[i]) ch->dlo[i] = pos_base;
if (pos_base + S > ch->dhi[i]) ch->dhi[i] = pos_base + S;
}
}
+static void inkc_cpu_step(Model *m, int pos_base, int S) {
+ for (int d = 0; d < 2; d++) inkc_cpu_one(inkc_of(m, d), pos_base, S);
+}
/* Record the uploads that make the device state the host's: the convolution states if
* the host's are newer, and the rows of the last `cap` positions the host wrote alone;
* a split layer: its window placed for the n rows from pos_base, its resident rows below
@@ -605,13 +671,13 @@ static void inkc_cpu_step(Model *m, int pos_base, int S) {
static int inkc_push_state(InkChain *ch, Model *m, int pos_base, int n) {
Cfg *c = &m->c; int ok = 1;
if (ch->cs_where == INKC_HOST) {
- for (int i = 0; i < ch->nl && ok; i++) for (int b = 0; b < 4 && ok; b++) {
+ for (int i = ch->lo; i < ch->lo + ch->nl && ok; i++) for (int b = 0; b < 4 && ok; b++) {
size_t nc = (size_t)inkc_cs_cells(c, b, i);
ok = ch->host_zero ? vkc_zero(ch->ring[b][i], 0, nc) : vkc_write(ch->ring[b][i], 0, m->cs[b][i], nc * sizeof(float));
}
ch->cs_where = INKC_BOTH;
}
- for (int i = 0; i < ch->nl && ok; i++) {
+ for (int i = ch->lo; i < ch->lo + ch->nl && ok; i++) {
if (ch->ks.on && ch->kli[i] >= 0) {
int KV = L_KV(c, i), hd = L_HD(c, i);
VkcKvPart pt[2] = {{KV, hd, m->K[i], (size_t)ch->cap[i] * hd, ch->kc[i], 0},
@@ -642,10 +708,12 @@ static int inkc_push_state(InkChain *ch, Model *m, int pos_base, int n) {
* states are the step's start: rebuild both on the CPU from the `upto` positions the
* prefix record names, and leave the chain off. */
static void inkc_recover(Model *m, int upto) {
- InkChain *ch = (InkChain *)m->vkchain;
+ InkChain *ch = (InkChain *)m->vkchain, *ch2 = (InkChain *)m->vkchain2;
Cfg *c = &m->c; int D = c->hidden, nl = ch ? ch->nl : c->n_layers;
+ if (ch && ch->ok && ch2 && ch2->ok) nl += ch2->nl; /* the second device's layers ran in that forward too */
g_vk_chain = 0;
if (ch) ch->failed = 1;
+ if (ch2) ch2->failed = 1;
/* the device's layers only: a partial chain's CPU layers hold their state on the host,
* whole up to the lost step (they run after the device's, once per forward) */
for (int i = 0; i < nl; i++)
@@ -782,13 +850,13 @@ static int inkc_join(InkChain *ch, Model *m, int i, int n) {
* partial chain's share taken: its first ch->nl layers ran here for every row, xh holds
* the residual after the last of them, and the caller runs the CPU's layers (from
* inkc_layers) and lm_head on it. */
-static int inkc_forward(Model *m, float *xh, int S, int pos_base, int want_x, float *logit) {
- if (!g_vk_chain) return 0;
- if (g_vk_chain == COLI_VK_CHAIN_PREFILL && S <= 2) return 0; /* prompts only: decode on the CPU */
- InkChain *ch = inkc_setup(m);
- if (!ch || ch->failed) return 0;
- Cfg *c = &m->c; int D = c->hidden, L = ch->nl, ET = c->n_experts + c->n_shared, ns = c->n_shared;
- int full = ch->full, LT = c->n_layers; /* full: every layer and lm_head here */
+/* One chain's layers (lo..lo+nl-1, on its device) from the residual rows in xh: 1 with the
+ * head, 2 with the residual after its last layer in xh, 0 not taken (*lost = 1: a frame
+ * failed, the state rebuilt on the CPU and the chains off; else nothing changed). */
+static int inkc_forward_seg(Model *m, InkChain *ch, float *xh, int S, int pos_base, int want_x, float *logit, int *lost) {
+ *lost = 0;
+ Cfg *c = &m->c; int D = c->hidden, L = ch->nl, lo = ch->lo, hi = lo + L, ET = c->n_experts + c->n_shared, ns = c->n_shared;
+ int full = ch->full, LT = c->n_layers; /* full: the last layer and lm_head here */
if (!full) want_x = 1; /* the CPU's layers read the residual */
int mirror_ok = inkc_mirror(ch, m);
int CH = mirror_ok ? inkc_chunk_rows(ch, m) : 1, rows = S < CH ? S : CH;
@@ -811,8 +879,8 @@ static int inkc_forward(Model *m, float *xh, int S, int pos_base, int want_x, fl
if (c->log_floor > 0) { double en = (double)(pb + s + 1) / c->log_floor; if (en > 1.0) t = 1.f + c->log_alpha * (float)log(en); }
tau[s] = t; tau[ch->rows + s] = 1.f;
}
- int ok = 1, pending = 0, f0 = 0;
- for (int i = 0; i < L && ok; i++) {
+ int ok = 1, pending = 0, f0 = lo;
+ for (int i = lo; i < hi && ok; i++) {
Layer *l = &m->L[i];
VkcEw add = {VKC_EW_ADD, n * D, D, 1, 0, 1, 0, 0, 0, 0, 0, 1.f};
if (pending) ok = inkc_join(ch, m, i - 1, n);
@@ -845,9 +913,9 @@ static int inkc_forward(Model *m, float *xh, int S, int pos_base, int want_x, fl
ok = ok && vkc_begin();
pending = 1;
}
- if (ok && pending) ok = inkc_join(ch, m, L - 1, n);
+ if (ok && pending) ok = inkc_join(ch, m, hi - 1, n);
if (ok && want_x) ok = vkc_copy(ch->xd, 0, ch->x, 0, (size_t)n * D); /* a partial chain: the handoff */
- for (int i = 0; i < L && ok; i++) for (int b = 0; b < 4 && ok; b++) /* the convolution states, for the host */
+ for (int i = lo; i < hi && ok; i++) for (int b = 0; b < 4 && ok; b++) /* the convolution states, for the host */
ok = vkc_copy(ch->csd, ch->o_csd[(size_t)b * LT + i], ch->ring[b][i], 0, (size_t)inkc_cs_cells(c, b, i));
int last = c0 + n == S;
if (ok && last && full) {
@@ -859,12 +927,12 @@ static int inkc_forward(Model *m, float *xh, int S, int pos_base, int want_x, fl
ok = ok && vkc_submit(1);
m->t_attn += now_s() - t0;
if (!ok) goto lost;
- if (f0 < L) inkc_kv_down(ch, m, f0, L - 1, n, pb);
+ if (f0 < hi) inkc_kv_down(ch, m, f0, hi - 1, n, pb);
const float *csd = (const float *)vkc_ptr(ch->csd);
- for (int i = 0; i < L; i++) for (int b = 0; b < 4; b++)
+ for (int i = lo; i < hi; i++) for (int b = 0; b < 4; b++)
memcpy(m->cs[b][i], csd + ch->o_csd[(size_t)b * LT + i], (size_t)inkc_cs_cells(c, b, i) * sizeof(float));
ch->cs_where = INKC_BOTH; ch->host_zero = 0;
- for (int i = 0; i < L; i++) if (ch->kli[i] >= 0) vkc_kv_done(&ch->ks, ch->kli[i], pb + n);
+ for (int i = lo; i < hi; i++) if (ch->kli[i] >= 0) vkc_kv_done(&ch->ks, ch->kli[i], pb + n);
if (want_x) memcpy(xfin + (size_t)c0 * D, vkc_ptr(ch->xd), (size_t)n * D * sizeof(float));
if (last && full) memcpy(logit, vkc_ptr(ch->outd), (size_t)c->unpad_vocab * sizeof(float));
}
@@ -872,21 +940,65 @@ static int inkc_forward(Model *m, float *xh, int S, int pos_base, int want_x, fl
ch->forwards++;
return full ? 1 : 2;
lost: /* a frame failed: the device is gone (or would not take a command); the CPU takes over */
- if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost(); }
+ if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost_dev(ch->d); }
free(xfin);
+ vkc_device(0);
inkc_recover(m, pos_base);
+ *lost = 1;
return 0;
}
+/* Every layer the chains hold for S rows from host rows xh, the last row's logits into
+ * `logit`; xh gets the final rows back when want_x. 0 = not taken: xh is as it was, and
+ * the caller runs the step on the CPU (after inkc_recover when a device was lost). 1 =
+ * taken. 2 = a partial share taken: the primary's layers, then the second device's, ran
+ * here for every row, xh holds the residual after the last of them, and the caller runs
+ * the CPU's layers (from inkc_ran) and lm_head on it. */
+static int inkc_forward(Model *m, float *xh, int S, int pos_base, int want_x, float *logit) {
+ g_inkc_ran = 0;
+ if (!g_vk_chain) return 0;
+ if (g_vk_chain == COLI_VK_CHAIN_PREFILL && S <= 2) return 0; /* prompts only: decode on the CPU */
+ InkChain *ch = inkc_setup(m), *ch2 = NULL;
+ if (!ch || ch->failed) return 0;
+ if (g_inkc_fit2_on && !ch->full && ch->nl == g_inkc_fit.n) {
+ vkc_device(1);
+ ch2 = inkc_setup_dev(m, 1);
+ vkc_device(0);
+ if (ch2 && (ch2->failed || ch2->lo != ch->lo + ch->nl)) ch2 = NULL;
+ }
+ int lost = 0, D = m->c.hidden;
+ /* the rows as they came in: the primary's residual replaces them, and if the second
+ * device is lost the caller runs the step from these */
+ float *keep = ch2 ? malloc((size_t)S * D * sizeof(float)) : NULL;
+ if (ch2 && !keep) ch2 = NULL;
+ if (keep) memcpy(keep, xh, (size_t)S * D * sizeof(float));
+ int r = inkc_forward_seg(m, ch, xh, S, pos_base, ch2 ? 1 : want_x, logit, &lost);
+ if (!r || !ch2) { free(keep); if (r) g_inkc_ran = ch->nl; return r; }
+ g_inkc_ran = ch->nl;
+ vkc_device(1);
+ int r2 = inkc_forward_seg(m, ch2, xh, S, pos_base, want_x, logit, &lost);
+ vkc_device(0);
+ if (lost) { memcpy(xh, keep, (size_t)S * D * sizeof(float)); free(keep); g_inkc_ran = 0; return 0; }
+ free(keep);
+ if (!r2) { inkc_cpu_one(ch2, pos_base, S); return 2; } /* declined: the CPU runs its layers this step */
+ g_inkc_ran += ch2->nl;
+ return r2;
+}
+
static void inkc_report(Model *m) {
- InkChain *ch = (InkChain *)m->vkchain;
- if (!ch || !ch->ok || !ch->forwards) return;
- VkcStats st; vkc_stats(&st);
- fprintf(stderr, "[VK] inkling chain: %llu forwards, %llu frames (%llu ops, %llu matmuls, %llu tiled GEMM), "
- "%.1f ms waiting for the device, %.1f ms of routed experts on the host, %.1f MiB on the device\n",
- ch->forwards, st.frames, st.ops, st.matmuls, st.gemms, st.wait_ms, ch->host_ms, st.dev_bytes / 1048576.0);
- vkc_kv_report(&ch->ks);
- vkc_prof_print();
+ for (int d = 0; d < 2; d++) {
+ InkChain *ch = inkc_of(m, d);
+ if (!ch || !ch->ok || !ch->forwards) continue;
+ int was = vkc_device(d);
+ VkcStats st; vkc_stats(&st);
+ fprintf(stderr, "[VK] %s chain: %llu forwards, %llu frames (%llu ops, %llu matmuls, %llu tiled GEMM), "
+ "%.1f ms waiting for the device, %.1f ms of routed experts on the host, %.1f MiB on the device\n",
+ inkc_name(ch), ch->forwards, st.frames, st.ops, st.matmuls, st.gemms, st.wait_ms, ch->host_ms,
+ st.dev_bytes / 1048576.0);
+ vkc_kv_report(&ch->ks);
+ vkc_prof_print();
+ vkc_device(was);
+ }
}
/* COLI_VK_CHAIN, decided after the tier (ink_vk_tier_start): CUDA and Metal keep their
@@ -906,5 +1018,5 @@ static void inkc_start(Model *m) {
g_vk_chain = coli_vk_chain_decide("inkling", vkt_ready(), INKLING_CHAIN_IGPU);
if (g_inkc_fit.L && !g_inkc_fit.n) g_vk_chain = 0; /* the fit left no layer to the device (its line said so) */
if (g_vk_chain && !vkc_init()) g_vk_chain = 0;
- if (g_vk_chain) atexit(vkc_shutdown);
+ if (g_vk_chain || g_inkc_fit2_on) atexit(vkc_shutdown_all);
}
diff --git a/c/tests/vulkan_layers_dev2.sh b/c/tests/vulkan_layers_dev2.sh
index 496d63e2..f53e57d2 100644
--- a/c/tests/vulkan_layers_dev2.sh
+++ b/c/tests/vulkan_layers_dev2.sh
@@ -179,20 +179,40 @@ ld2_mimo() {
echo "OK ld2 mimo second device lost: tokens = CPU, $(grep -o 'the device was lost at position [0-9]*' mimo-vk.err)"
}
+ld2_inkling() {
+ [ -f tiny_inkling/ref_long.json ] || kv_inkling_fixtures
+ local R=tiny_inkling/ref_inkling.json T=SNAP=tiny_inkling b
+ ld2_gate inkling "ld2 inkling 3 + 5, lm_head on the second device" 3 5 $T -- 8 0 $R
+ ld2_gate inkling "ld2 inkling 2 + 2, the CPU the rest and lm_head" 2 2 $T -- 8 0 $R
+ ld2_gate inkling "ld2 inkling 1 + 7, prompt in chunks of 3" 1 7 $T COLI_VK_CHAIN_ROWS=3 -- 8 0 $R
+ for b in 4 8; do ld2_gate inkling "ld2 inkling runtime int$b" 3 5 $T -- 2 $b $R; done
+ ld2_gate inkling "ld2 inkling experts on both devices too" 3 5 $T COLI_VK_TIER_GB=0.00002 -- 8 0 $R
+ CHAINMODE=2 ld2_gate inkling "ld2 inkling prompts only" 3 5 $T -- 8 0 $R
+ # the global layers' KV split on both devices
+ ld2_gate inkling "ld2 inkling KV split on both" 4 4 COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 $T -- 8 0 tiny_inkling/ref_long.json
+ [ "$(kv_hostparts inkling vk.log)" -gt 0 ] && [ "$(kv_hostparts "inkling dev2" vk.log)" -gt 0 ] ||
+ { cat vk.log; fail "ld2 inkling KV split on both: a device's split ran no host part"; }
+ LD2_EXPECT=setup ld2_lost inkling "ld2 inkling second device lost at its setup" 3 5 1 $T -- 8 0 $R
+ ld2_lost inkling "ld2 inkling second device lost mid-decode" 3 5 30 $T -- 8 0 $R
+ CHAIN_SERVE_EXPECT='inkling dev2 chain: [1-9][0-9]* forwards' \
+ $PY tests/vulkan_chain_serve.py ./inkling tiny_inkling INK_PREFIX_LOG=1 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=3 COLI_VK_CHAIN_LAYERS2=5
+}
+
family_layers_dev2() {
export OMP_NUM_THREADS=2
- make qwen36 qwen38 olmoe mimo tests/test_vk_chain VK=1
+ make qwen36 qwen38 olmoe mimo inkling tests/test_vk_chain VK=1
COLI_VK_DEV2=0 ./tests/test_vk_chain shaders/qmatmul.spv | tee vk_chain.log
tail -1 vk_chain.log | grep -qx PASS || fail "the chain's ops on two devices"
ld2_qwen36
ld2_qwen38
ld2_olmoe
ld2_mimo
+ ld2_inkling
unset OMP_NUM_THREADS
}
family_layers_dev2_sanitize() {
make clean >/dev/null 2>&1 || true
- make qwen36 qwen38 olmoe VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
+ make qwen36 qwen38 olmoe mimo inkling VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
[ -d qwen36_kv_c ] || kv_qwen36_fixtures
kv_san qwen36 "asan ld2 qwen36 KV split on both" COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=4 COLI_VK_CHAIN_LAYERS2=4 \
@@ -206,5 +226,9 @@ family_layers_dev2_sanitize() {
kv_san olmoe "asan ld2 olmoe KV split on both, PILOT" COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=2 \
COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 PILOT=1 WIDE=2 SNAP=olmoe_tiny_c ./olmoe 2 8 olmoe_tiny/ref_olmoe_long.json
[ "$(ld2_forwards olmoe san.log)" -gt 0 ] || { cat san.log; fail "asan ld2 olmoe: the second device's chain never ran"; }
+ [ -f tiny_inkling/ref_long.json ] || kv_inkling_fixtures
+ kv_san inkling "asan ld2 inkling KV split on both" COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=4 COLI_VK_CHAIN_LAYERS2=4 \
+ COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 SNAP=tiny_inkling ./inkling 8 0 tiny_inkling/ref_long.json
+ [ "$(ld2_forwards inkling san.log)" -gt 0 ] || { cat san.log; fail "asan ld2 inkling: the second device's chain never ran"; }
make clean >/dev/null 2>&1 || true
}