Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(vulkan): deepseek_v41 builds each chain layer whole in its device-only placement

JustVugg2c14f7ea39f7
diff --git a/c/deepseek_v41.c b/c/deepseek_v41.c
index f1e44c5e..bffc211e 100644
--- a/c/deepseek_v41.c
+++ b/c/deepseek_v41.c
@@ -1750,6 +1750,9 @@ static void attn_project_check(const Cfg *c);
static int v41c_decide(Model *m); /* deepseek_v41_chain.h: COLI_VK_CHAIN, decided once */
static int v41c_fit_now(Model *m); /* deepseek_v41_chain.h: how many layers the chain places, once */
static void v41c_fit_recount(Model *m);
+static int v41c_prepare(Model *m);
+static int v41c_build_layer(Model *m, int i, const char **why);
+static void v41c_cut_layer(Model *m, int i, const char *why);
/* What the device would hold alone: the trunk matrices model_load reads into mappings,
* of the first `layers` layers (a partial chain's; every layer otherwise). */
static size_t v41_dho_bytes(Model *m, int layers) {
@@ -1802,6 +1805,7 @@ static void v41_dho_open(Model *m) {
if (!g_v41_dho) v41c_fit_recount(m); /* the chain's own set goes up after all */
g_v41_dho_layers = layers;
g_v41_shards = &m->S;
+ v41c_prepare(m); /* the chain's layers built whole as they are read */
}
/* One matrix up as a lookup will ask for it, marked as held by the device alone. */
static int v41_dho_up(const void *q, const uint8_t *tiles, int fmt, int O, int I) {
@@ -1851,12 +1855,14 @@ static void v41_dho_unplace(Model *m, int k) {
/* The matrices read since the last call: up, then their pages back. wo_a goes up whole
* for the chain and per output group for the per-matrix path (attention_project's
* block views), each only where that path runs. With a partial chain (the fit) a layer
- * goes up whole or not at all: its pages go back only once all of it is on the device,
- * and a matrix the device refuses ends the chain before its layer (vkc_fit_shrink). */
+ * goes up whole or not at all, its chain state with it (v41c_build_layer): its pages go
+ * back only once all of it is on the device, so a layer that does not get there reads
+ * nothing back; it ends the chain before it (vkc_fit_shrink). */
static unsigned g_v41_dho_kept;
static void v41_dho_place(Model *m, int mark) { /* mark: the layer just read (its fault count noted), -1 none */
const Cfg *c = &m->c;
int chain = v41c_decide(m), fit = g_v41_fit.L > 0, failed = -1;
+ const char *why = "a matrix the device refused";
size_t from = g_v41_home_n;
for (size_t k = 0; k < g_v41_home_n; k++) if (!g_v41_home[k].placed) { from = k; break; }
for (size_t k = from; k < g_v41_home_n; k++) {
@@ -1879,9 +1885,9 @@ static void v41_dho_place(Model *m, int mark) { /* mark: the layer just read (
else g_v41_dho_kept++;
}
if (!fit) return;
+ if (failed < 0 && mark >= 0 && mark < g_v41_fit.n && !v41c_build_layer(m, mark, &why)) failed = mark;
if (failed >= 0) { /* everything of that layer and after off the device, the chain cut before it */
- v41_dho_unplace(m, failed);
- vkc_fit_shrink("deepseek_v41", &g_v41_fit, failed, "a matrix the device refused");
+ v41c_cut_layer(m, failed, why);
return;
}
for (size_t k = from; k < g_v41_home_n; k++) { /* the whole batch is there: its pages back */
@@ -2006,6 +2012,12 @@ static void model_load(Model *m, const char *snap, int ecap, int engram_cache_ro
V41_DHO(wb_load(&m->S, &l->idx_wproj, NAME("layers.%d.attn.indexer.weights_proj.weight", i),
c->index_n_heads, dim));
}
+ /* an engram table's projection, read with its layer (the device-only placement puts
+ * a layer up whole); the tables themselves open below */
+ for (int t = 0; m->engram.active && t < m->engram.n_layers; t++)
+ if (m->engram.layer_of[t] == i)
+ V41_DHO(w8_load(&m->S, &l->eng_wkv, NAME("layers.%d.engram.wkv.weight", i),
+ dim * (hc + 1), m->engram.cols * m->engram.head_dim));
#ifdef COLI_VULKAN
if (g_v41_dho) v41_dho_place(m, i); /* one layer in RAM at a time */
#endif
@@ -2018,14 +2030,6 @@ static void model_load(Model *m, const char *snap, int ecap, int engram_cache_ro
fprintf(stderr, "[engram] table %d names layer %d\n", t, layer); exit(1); }
m->L[layer].engram_index = t;
engram_table_open(&e->table[t], &m->S, layer, e->head_dim, engram_cache_rows);
-#ifdef COLI_VULKAN
- g_v41_load_layer = layer;
-#endif
- V41_DHO(w8_load(&m->S, &m->L[layer].eng_wkv, NAME("layers.%d.engram.wkv.weight", layer),
- dim * (hc + 1), e->cols * e->head_dim));
-#ifdef COLI_VULKAN
- if (g_v41_dho) v41_dho_place(m, -1);
-#endif
wf_load(&m->S, &m->L[layer].eng_q, NAME("layers.%d.engram.q_weight", layer),
(int64_t)hc * dim);
wf_load(&m->S, &m->L[layer].eng_k, NAME("layers.%d.engram.k_weight", layer),
diff --git a/c/deepseek_v41_chain.h b/c/deepseek_v41_chain.h
index 044addbf..a57f0bfc 100644
--- a/c/deepseek_v41_chain.h
+++ b/c/deepseek_v41_chain.h
@@ -282,24 +282,32 @@ static void v41c_fit_recount(Model *m) {
for (int i = 0; i < g_v41_fit.L; i++) g_v41_fit.per[i] = v41c_layer_bytes(m, i, rows, 0, 0, &g_v41_fit.mat[i]);
}
+/* Layer i's state on the device, nothing of it read from disk: its mHC mix matrices (f32,
+ * the host keeps them), its window ring and its compressor's group. Kept when there. */
+static int v41c_layer_state(V41Chain *ch, Model *m, int i, const char **why) {
+ Cfg *c = &m->c; Layer *l = &m->L[i];
+ int H = c->hc_mult, D = c->dim, nm = (2 + H) * H, r = c->compress_ratio[i];
+ *why = "a matrix the device refused";
+ if (!coli_vk_tensor_ensure(&ch->fna[i], l->hc_attn_fn.w, NULL, 10, H * D, nm, 0) ||
+ !coli_vk_tensor_ensure(&ch->fnf[i], l->hc_ffn_fn.w, NULL, 10, H * D, nm, 0)) return 0;
+ *why = "device memory for its state refused";
+ if (!ch->win[i] && !(ch->win[i] = vkc_buf((size_t)ch->Wd * c->head_dim * sizeof(float), VKC_DEV))) return 0;
+ if (c->kv_source[i] && r > 1 && !ch->ring[i] && !(ch->ring[i] = vkc_buf((size_t)2 * r * c->head_dim * sizeof(float), VKC_DEV)))
+ return 0;
+ return 1;
+}
/* Layer i's tensors and state on the device (its matrices through the per-matrix path's
* map: already there when the trunk lives on the device only). 0 with *why set. */
static int v41c_layer_up(V41Chain *ch, Model *m, int i, const char **why) {
Cfg *c = &m->c; Layer *l = &m->L[i];
- int H = c->hc_mult, D = c->dim, nm = (2 + H) * H, r = c->compress_ratio[i];
+ int r = c->compress_ratio[i];
*why = "a matrix the device refused";
- int ok = coli_vk_tensor_ensure(&ch->fna[i], l->hc_attn_fn.w, NULL, 10, H * D, nm, 0) &&
- coli_vk_tensor_ensure(&ch->fnf[i], l->hc_ffn_fn.w, NULL, 10, H * D, nm, 0) &&
- v41c_w8(&l->wq_a) && v41c_w8(&l->wq_b) && v41c_w8(&l->wkv) && v41c_w8(&l->wo_a) && v41c_w8(&l->wo_b) &&
+ int ok = v41c_w8(&l->wq_a) && v41c_w8(&l->wq_b) && v41c_w8(&l->wkv) && v41c_w8(&l->wo_a) && v41c_w8(&l->wo_b) &&
v41c_w8(&l->sh_w1) && v41c_w8(&l->sh_w3) && v41c_w8(&l->sh_w2);
if (ok && c->kv_source[i]) ok = v41c_wb(&l->comp_wkv) && (r <= 1 || v41c_wb(&l->comp_wgate)) && v41c_wb(&l->idx_wk);
if (ok && c->index_source[i] && r > 0) ok = v41c_w8(&l->idx_wq_b) && v41c_wb(&l->idx_wproj);
- if (ok && l->engram_index >= 0) ok = v41c_w8(&l->eng_wkv) != NULL;
- if (!ok) return 0;
- *why = "device memory for its state refused";
- if (!(ch->win[i] = vkc_buf((size_t)ch->Wd * c->head_dim * sizeof(float), VKC_DEV))) return 0;
- if (c->kv_source[i] && r > 1 && !(ch->ring[i] = vkc_buf((size_t)2 * r * c->head_dim * sizeof(float), VKC_DEV))) return 0;
- return 1;
+ if (ok && l->eng_wkv.q) ok = v41c_w8(&l->eng_wkv) != NULL;
+ return ok && v41c_layer_state(ch, m, i, why);
}
/* Everything of layer i off the device: the chain's tensors and buffers, and the device
* copies of its matrices (their entries refused: the CPU multiplies them from now on). */
@@ -325,28 +333,64 @@ static void v41c_cut(V41Chain *ch, Model *m, int i, const char *why) {
vkc_fit_shrink("deepseek_v41", &g_v41_fit, i, why);
}
-static int v41c_setup(Model *m) {
+/* The chain's struct, made once: by the device-only placement, which builds each of the
+ * chain's layers whole as model_load reads it (v41c_build_layer), else by the setup. */
+static V41Chain *v41c_alloc(Model *m) {
+ if (g_v41c) return g_v41c;
Cfg *c = &m->c;
- int L = c->n_layers, D = c->dim, H = c->hc_mult, nm = (2 + H) * H;
- const char *why = v41c_unsupported(m);
- if (why) { fprintf(stderr, "[VK] deepseek_v41 chain: %s; the CPU runs the layers\n", why); return 0; }
- int N = g_v41_fit.L > 0 ? g_v41_fit.n : L;
- if (N < 1) return 0;
+ int L = c->n_layers;
V41Chain *ch = calloc(1, sizeof *ch);
- if (!ch) return 0;
+ if (!ch) return NULL;
size_t **offs[] = {&ch->o_an, &ch->o_fn, &ch->o_qn, &ch->o_kn, &ch->o_sink, &ch->o_hca, &ch->o_hcf, &ch->o_cn, &ch->o_ikn,
&ch->o_eq, &ch->o_ek};
- for (size_t k = 0; k < sizeof offs / sizeof *offs; k++) if (!(*offs[k] = calloc(L, sizeof(size_t)))) return 0;
+ for (size_t k = 0; k < sizeof offs / sizeof *offs; k++) if (!(*offs[k] = calloc(L, sizeof(size_t)))) return NULL;
ch->fna = calloc(L, sizeof(void *)); ch->fnf = calloc(L, sizeof(void *));
ch->win = calloc(L, sizeof(void *)); ch->ckv = calloc(L, sizeof(void *));
ch->ikey = calloc(L, sizeof(void *)); ch->ring = calloc(L, sizeof(void *));
ch->kv_valid = calloc(L, sizeof(int)); ch->ccap = calloc(L, sizeof(int)); ch->sli = malloc(L * sizeof(int));
- if (!ch->fna || !ch->fnf || !ch->win || !ch->ckv || !ch->ikey || !ch->ring || !ch->kv_valid || !ch->ccap || !ch->sli) return 0;
- ch->n = N;
- g_v41c = ch;
+ if (!ch->fna || !ch->fnf || !ch->win || !ch->ckv || !ch->ikey || !ch->ring || !ch->kv_valid || !ch->ccap || !ch->sli) return NULL;
+ ch->n = g_v41_fit.L > 0 ? g_v41_fit.n : L;
ch->W = c->window; ch->K = c->index_topk > 0 ? c->index_topk : 0; ch->rmax = 1;
for (int i = 0; i < L; i++) if (c->kv_source[i] && c->compress_ratio[i] > ch->rmax) ch->rmax = c->compress_ratio[i];
ch->Wd = ch->W + v41c_rows();
+ return g_v41c = ch;
+}
+/* The device-only placement of layer i put its matrices up (model_load, v41_dho_place):
+ * its state goes up too before its host pages are given back, so the layer reaches the
+ * device whole or not at all and a failure reads nothing back. 0 with *why set. */
+static int g_v41c_build; /* the chain's layers are built in the placement (v41_dho_open) */
+static int v41c_build_layer(Model *m, int i, const char **why) {
+ V41Chain *ch = g_v41c;
+ if (!g_v41c_build || !ch || i >= ch->n) return 1;
+ return v41c_layer_state(ch, m, i, why);
+}
+
+/* Layer i did not reach the device (v41_dho_place, as model_load reads it): the chain cut
+ * before it, whatever of it and of the layers after it is up freed. */
+static void v41c_cut_layer(Model *m, int i, const char *why) {
+ if (g_v41c) { v41c_cut(g_v41c, m, i, why); return; }
+ if (g_v41_dho) v41_dho_unplace(m, i);
+ vkc_fit_shrink("deepseek_v41", &g_v41_fit, i, why);
+}
+/* Before the layers are read, when the trunk lives on the device only: the chain's
+ * pipelines and its struct, so the placement builds each of its layers whole. 0: it will
+ * not (the setup says why later). */
+static int v41c_prepare(Model *m) {
+ if (!g_v41_dho || g_v41_fit.L < 1 || g_v41_fit.n < 1 || !v41c_decide(m) || v41c_unsupported(m)) return 0;
+ if (!(g_v41c_inited = vkc_init()) || !(vkc_mla_ready() && vkc_mhc_ready() && vkc_dsv4_ready())) return 0;
+ return g_v41c_build = v41c_alloc(m) != NULL;
+}
+
+static int v41c_setup(Model *m) {
+ Cfg *c = &m->c;
+ int L = c->n_layers, D = c->dim, H = c->hc_mult, nm = (2 + H) * H;
+ const char *why = v41c_unsupported(m);
+ if (why) { fprintf(stderr, "[VK] deepseek_v41 chain: %s; the CPU runs the layers\n", why); return 0; }
+ if ((g_v41_fit.L > 0 ? g_v41_fit.n : L) < 1) return 0;
+ V41Chain *ch = v41c_alloc(m);
+ if (!ch) return 0;
+ int N = ch->n;
+ if (N < 1) return 0;
/* the parameter arena, the chain's layers' */
size_t n = 0;
for (int i = 0; i < N; i++) {