Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(vulkan): deepseek_v41 builds each chain layer whole in its device-only placement
JustVugg2c14f7ea39f7
diff --git a/c/deepseek_v41.c b/c/deepseek_v41.cindex f1e44c5e..bffc211e 100644--- a/c/deepseek_v41.c+++ b/c/deepseek_v41.c@@ -1750,6 +1750,9 @@ static void attn_project_check(const Cfg *c);static int v41c_decide(Model *m); /* deepseek_v41_chain.h: COLI_VK_CHAIN, decided once */static int v41c_fit_now(Model *m); /* deepseek_v41_chain.h: how many layers the chain places, once */static void v41c_fit_recount(Model *m);+static int v41c_prepare(Model *m);+static int v41c_build_layer(Model *m, int i, const char **why);+static void v41c_cut_layer(Model *m, int i, const char *why);/* What the device would hold alone: the trunk matrices model_load reads into mappings,* of the first `layers` layers (a partial chain's; every layer otherwise). */static size_t v41_dho_bytes(Model *m, int layers) {@@ -1802,6 +1805,7 @@ static void v41_dho_open(Model *m) {if (!g_v41_dho) v41c_fit_recount(m); /* the chain's own set goes up after all */g_v41_dho_layers = layers;g_v41_shards = &m->S;+ v41c_prepare(m); /* the chain's layers built whole as they are read */}/* One matrix up as a lookup will ask for it, marked as held by the device alone. */static int v41_dho_up(const void *q, const uint8_t *tiles, int fmt, int O, int I) {@@ -1851,12 +1855,14 @@ static void v41_dho_unplace(Model *m, int k) {/* The matrices read since the last call: up, then their pages back. wo_a goes up whole* for the chain and per output group for the per-matrix path (attention_project's* block views), each only where that path runs. With a partial chain (the fit) a layer- * goes up whole or not at all: its pages go back only once all of it is on the device,- * and a matrix the device refuses ends the chain before its layer (vkc_fit_shrink). */+ * goes up whole or not at all, its chain state with it (v41c_build_layer): its pages go+ * back only once all of it is on the device, so a layer that does not get there reads+ * nothing back; it ends the chain before it (vkc_fit_shrink). */static unsigned g_v41_dho_kept;static void v41_dho_place(Model *m, int mark) { /* mark: the layer just read (its fault count noted), -1 none */const Cfg *c = &m->c;int chain = v41c_decide(m), fit = g_v41_fit.L > 0, failed = -1;+ const char *why = "a matrix the device refused";size_t from = g_v41_home_n;for (size_t k = 0; k < g_v41_home_n; k++) if (!g_v41_home[k].placed) { from = k; break; }for (size_t k = from; k < g_v41_home_n; k++) {@@ -1879,9 +1885,9 @@ static void v41_dho_place(Model *m, int mark) { /* mark: the layer just read (else g_v41_dho_kept++;}if (!fit) return;+ if (failed < 0 && mark >= 0 && mark < g_v41_fit.n && !v41c_build_layer(m, mark, &why)) failed = mark;if (failed >= 0) { /* everything of that layer and after off the device, the chain cut before it */- v41_dho_unplace(m, failed);- vkc_fit_shrink("deepseek_v41", &g_v41_fit, failed, "a matrix the device refused");+ v41c_cut_layer(m, failed, why);return;}for (size_t k = from; k < g_v41_home_n; k++) { /* the whole batch is there: its pages back */@@ -2006,6 +2012,12 @@ static void model_load(Model *m, const char *snap, int ecap, int engram_cache_roV41_DHO(wb_load(&m->S, &l->idx_wproj, NAME("layers.%d.attn.indexer.weights_proj.weight", i),c->index_n_heads, dim));}+ /* an engram table's projection, read with its layer (the device-only placement puts+ * a layer up whole); the tables themselves open below */+ for (int t = 0; m->engram.active && t < m->engram.n_layers; t++)+ if (m->engram.layer_of[t] == i)+ V41_DHO(w8_load(&m->S, &l->eng_wkv, NAME("layers.%d.engram.wkv.weight", i),+ dim * (hc + 1), m->engram.cols * m->engram.head_dim));#ifdef COLI_VULKANif (g_v41_dho) v41_dho_place(m, i); /* one layer in RAM at a time */#endif@@ -2018,14 +2030,6 @@ static void model_load(Model *m, const char *snap, int ecap, int engram_cache_rofprintf(stderr, "[engram] table %d names layer %d\n", t, layer); exit(1); }m->L[layer].engram_index = t;engram_table_open(&e->table[t], &m->S, layer, e->head_dim, engram_cache_rows);-#ifdef COLI_VULKAN- g_v41_load_layer = layer;-#endif- V41_DHO(w8_load(&m->S, &m->L[layer].eng_wkv, NAME("layers.%d.engram.wkv.weight", layer),- dim * (hc + 1), e->cols * e->head_dim));-#ifdef COLI_VULKAN- if (g_v41_dho) v41_dho_place(m, -1);-#endifwf_load(&m->S, &m->L[layer].eng_q, NAME("layers.%d.engram.q_weight", layer),(int64_t)hc * dim);wf_load(&m->S, &m->L[layer].eng_k, NAME("layers.%d.engram.k_weight", layer),diff --git a/c/deepseek_v41_chain.h b/c/deepseek_v41_chain.hindex 044addbf..a57f0bfc 100644--- a/c/deepseek_v41_chain.h+++ b/c/deepseek_v41_chain.h@@ -282,24 +282,32 @@ static void v41c_fit_recount(Model *m) {for (int i = 0; i < g_v41_fit.L; i++) g_v41_fit.per[i] = v41c_layer_bytes(m, i, rows, 0, 0, &g_v41_fit.mat[i]);}+/* Layer i's state on the device, nothing of it read from disk: its mHC mix matrices (f32,+ * the host keeps them), its window ring and its compressor's group. Kept when there. */+static int v41c_layer_state(V41Chain *ch, Model *m, int i, const char **why) {+ Cfg *c = &m->c; Layer *l = &m->L[i];+ int H = c->hc_mult, D = c->dim, nm = (2 + H) * H, r = c->compress_ratio[i];+ *why = "a matrix the device refused";+ if (!coli_vk_tensor_ensure(&ch->fna[i], l->hc_attn_fn.w, NULL, 10, H * D, nm, 0) ||+ !coli_vk_tensor_ensure(&ch->fnf[i], l->hc_ffn_fn.w, NULL, 10, H * D, nm, 0)) return 0;+ *why = "device memory for its state refused";+ if (!ch->win[i] && !(ch->win[i] = vkc_buf((size_t)ch->Wd * c->head_dim * sizeof(float), VKC_DEV))) return 0;+ if (c->kv_source[i] && r > 1 && !ch->ring[i] && !(ch->ring[i] = vkc_buf((size_t)2 * r * c->head_dim * sizeof(float), VKC_DEV)))+ return 0;+ return 1;+}/* Layer i's tensors and state on the device (its matrices through the per-matrix path's* map: already there when the trunk lives on the device only). 0 with *why set. */static int v41c_layer_up(V41Chain *ch, Model *m, int i, const char **why) {Cfg *c = &m->c; Layer *l = &m->L[i];- int H = c->hc_mult, D = c->dim, nm = (2 + H) * H, r = c->compress_ratio[i];+ int r = c->compress_ratio[i];*why = "a matrix the device refused";- int ok = coli_vk_tensor_ensure(&ch->fna[i], l->hc_attn_fn.w, NULL, 10, H * D, nm, 0) &&- coli_vk_tensor_ensure(&ch->fnf[i], l->hc_ffn_fn.w, NULL, 10, H * D, nm, 0) &&- v41c_w8(&l->wq_a) && v41c_w8(&l->wq_b) && v41c_w8(&l->wkv) && v41c_w8(&l->wo_a) && v41c_w8(&l->wo_b) &&+ int ok = v41c_w8(&l->wq_a) && v41c_w8(&l->wq_b) && v41c_w8(&l->wkv) && v41c_w8(&l->wo_a) && v41c_w8(&l->wo_b) &&v41c_w8(&l->sh_w1) && v41c_w8(&l->sh_w3) && v41c_w8(&l->sh_w2);if (ok && c->kv_source[i]) ok = v41c_wb(&l->comp_wkv) && (r <= 1 || v41c_wb(&l->comp_wgate)) && v41c_wb(&l->idx_wk);if (ok && c->index_source[i] && r > 0) ok = v41c_w8(&l->idx_wq_b) && v41c_wb(&l->idx_wproj);- if (ok && l->engram_index >= 0) ok = v41c_w8(&l->eng_wkv) != NULL;- if (!ok) return 0;- *why = "device memory for its state refused";- if (!(ch->win[i] = vkc_buf((size_t)ch->Wd * c->head_dim * sizeof(float), VKC_DEV))) return 0;- if (c->kv_source[i] && r > 1 && !(ch->ring[i] = vkc_buf((size_t)2 * r * c->head_dim * sizeof(float), VKC_DEV))) return 0;- return 1;+ if (ok && l->eng_wkv.q) ok = v41c_w8(&l->eng_wkv) != NULL;+ return ok && v41c_layer_state(ch, m, i, why);}/* Everything of layer i off the device: the chain's tensors and buffers, and the device* copies of its matrices (their entries refused: the CPU multiplies them from now on). */@@ -325,28 +333,64 @@ static void v41c_cut(V41Chain *ch, Model *m, int i, const char *why) {vkc_fit_shrink("deepseek_v41", &g_v41_fit, i, why);}-static int v41c_setup(Model *m) {+/* The chain's struct, made once: by the device-only placement, which builds each of the+ * chain's layers whole as model_load reads it (v41c_build_layer), else by the setup. */+static V41Chain *v41c_alloc(Model *m) {+ if (g_v41c) return g_v41c;Cfg *c = &m->c;- int L = c->n_layers, D = c->dim, H = c->hc_mult, nm = (2 + H) * H;- const char *why = v41c_unsupported(m);- if (why) { fprintf(stderr, "[VK] deepseek_v41 chain: %s; the CPU runs the layers\n", why); return 0; }- int N = g_v41_fit.L > 0 ? g_v41_fit.n : L;- if (N < 1) return 0;+ int L = c->n_layers;V41Chain *ch = calloc(1, sizeof *ch);- if (!ch) return 0;+ if (!ch) return NULL;size_t **offs[] = {&ch->o_an, &ch->o_fn, &ch->o_qn, &ch->o_kn, &ch->o_sink, &ch->o_hca, &ch->o_hcf, &ch->o_cn, &ch->o_ikn,&ch->o_eq, &ch->o_ek};- for (size_t k = 0; k < sizeof offs / sizeof *offs; k++) if (!(*offs[k] = calloc(L, sizeof(size_t)))) return 0;+ for (size_t k = 0; k < sizeof offs / sizeof *offs; k++) if (!(*offs[k] = calloc(L, sizeof(size_t)))) return NULL;ch->fna = calloc(L, sizeof(void *)); ch->fnf = calloc(L, sizeof(void *));ch->win = calloc(L, sizeof(void *)); ch->ckv = calloc(L, sizeof(void *));ch->ikey = calloc(L, sizeof(void *)); ch->ring = calloc(L, sizeof(void *));ch->kv_valid = calloc(L, sizeof(int)); ch->ccap = calloc(L, sizeof(int)); ch->sli = malloc(L * sizeof(int));- if (!ch->fna || !ch->fnf || !ch->win || !ch->ckv || !ch->ikey || !ch->ring || !ch->kv_valid || !ch->ccap || !ch->sli) return 0;- ch->n = N;- g_v41c = ch;+ if (!ch->fna || !ch->fnf || !ch->win || !ch->ckv || !ch->ikey || !ch->ring || !ch->kv_valid || !ch->ccap || !ch->sli) return NULL;+ ch->n = g_v41_fit.L > 0 ? g_v41_fit.n : L;ch->W = c->window; ch->K = c->index_topk > 0 ? c->index_topk : 0; ch->rmax = 1;for (int i = 0; i < L; i++) if (c->kv_source[i] && c->compress_ratio[i] > ch->rmax) ch->rmax = c->compress_ratio[i];ch->Wd = ch->W + v41c_rows();+ return g_v41c = ch;+}+/* The device-only placement of layer i put its matrices up (model_load, v41_dho_place):+ * its state goes up too before its host pages are given back, so the layer reaches the+ * device whole or not at all and a failure reads nothing back. 0 with *why set. */+static int g_v41c_build; /* the chain's layers are built in the placement (v41_dho_open) */+static int v41c_build_layer(Model *m, int i, const char **why) {+ V41Chain *ch = g_v41c;+ if (!g_v41c_build || !ch || i >= ch->n) return 1;+ return v41c_layer_state(ch, m, i, why);+}++/* Layer i did not reach the device (v41_dho_place, as model_load reads it): the chain cut+ * before it, whatever of it and of the layers after it is up freed. */+static void v41c_cut_layer(Model *m, int i, const char *why) {+ if (g_v41c) { v41c_cut(g_v41c, m, i, why); return; }+ if (g_v41_dho) v41_dho_unplace(m, i);+ vkc_fit_shrink("deepseek_v41", &g_v41_fit, i, why);+}+/* Before the layers are read, when the trunk lives on the device only: the chain's+ * pipelines and its struct, so the placement builds each of its layers whole. 0: it will+ * not (the setup says why later). */+static int v41c_prepare(Model *m) {+ if (!g_v41_dho || g_v41_fit.L < 1 || g_v41_fit.n < 1 || !v41c_decide(m) || v41c_unsupported(m)) return 0;+ if (!(g_v41c_inited = vkc_init()) || !(vkc_mla_ready() && vkc_mhc_ready() && vkc_dsv4_ready())) return 0;+ return g_v41c_build = v41c_alloc(m) != NULL;+}++static int v41c_setup(Model *m) {+ Cfg *c = &m->c;+ int L = c->n_layers, D = c->dim, H = c->hc_mult, nm = (2 + H) * H;+ const char *why = v41c_unsupported(m);+ if (why) { fprintf(stderr, "[VK] deepseek_v41 chain: %s; the CPU runs the layers\n", why); return 0; }+ if ((g_v41_fit.L > 0 ? g_v41_fit.n : L) < 1) return 0;+ V41Chain *ch = v41c_alloc(m);+ if (!ch) return 0;+ int N = ch->n;+ if (N < 1) return 0;/* the parameter arena, the chain's layers' */size_t n = 0;for (int i = 0; i < N; i++) {