Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(deepseek_v4): a partial dense chain, the first N layers on the device
JustVugga201e1fe2850
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.cindex 5725f0b4..7424cdb7 100644--- a/c/deepseek_v4.c+++ b/c/deepseek_v4.c@@ -1078,8 +1078,8 @@ int coli_v4_layer_load(ColiV4Engine *engine,if (!resident_enabled_v2(engine))return coli_v4_layer_resident_reference_load(NULL, weights, effective_config, index, layer, error, error_size);- if (engine->dense_resident.device_only && coli_v4_dense_device_lost &&- coli_v4_dense_device_lost()) {+ int device_layer = engine->dense_resident.device_only && layer < engine->dense_resident.device_layers;+ if (device_layer && coli_v4_dense_device_lost && coli_v4_dense_device_lost()) {/* the device that held the dense layers is gone: the CPU reads them per forward,* as a low-memory plan does (its two-layer reserve is in the plan) */if (coli_v4_layer_resident_reference_load(@@ -1097,7 +1097,7 @@ int coli_v4_layer_load(ColiV4Engine *engine,}engine->dense_resident.index = index;if (!engine->dense_resident.ready[layer]) {- g_v4_layer_map = engine->dense_resident.device_only; /* droppable tensors in mappings of their own */+ g_v4_layer_map = device_layer; /* droppable tensors in mappings of their own */int failed = coli_v4_layer_resident_reference_load(NULL, &engine->dense_resident.layers[layer], effective_config, index,layer, error, error_size);@@ -1113,8 +1113,9 @@ int coli_v4_layer_load(ColiV4Engine *engine,engine->dense_resident.total_bytes +=engine->dense_resident.layers[layer].stats.total_bytes;/* device only: the layer's matrices to the device and their host pages back,- * before the next layer is read (one layer in RAM at a time) */- if (engine->dense_resident.device_only && coli_v4_dense_place)+ * before the next layer is read (one layer in RAM at a time); a partial chain's+ * CPU layers keep theirs */+ if (device_layer && coli_v4_dense_place)coli_v4_dense_place(engine, layer);if (layer == effective_config->num_hidden_layers - 1)fprintf(stderr, "v4_dense_resident layers=%d bytes=%.3fGiB\n",@@ -1545,10 +1546,11 @@ static int build_runtime_plan(ColiV4Engine *engine,* binary built with VK=1, NULL in every other link. */ColiV4DenseDeviceDecide coli_v4_dense_device_decide;-/* The bytes of the dense layers a device may hold alone (coli_v4_dense_device_only_tensor). */-static uint64_t v4_dense_device_bytes(const ColiV4Engine *engine) {+/* The bytes of the first `layers` dense layers a device may hold alone+ * (coli_v4_dense_device_only_tensor). */+static uint64_t v4_dense_device_bytes(const ColiV4Engine *engine, int layers) {uint64_t total = 0;- for (int layer = 0; layer < engine->config.num_hidden_layers; layer++) {+ for (int layer = 0; layer < layers && layer < engine->config.num_hidden_layers; layer++) {ColiDeepSeekV4LayerPlan plan;if (coli_v4_layer_plan(&plan, &engine->config, layer, NULL, 0)) return 0;for (size_t i = 0; i < plan.tensor_count; i++)@@ -1584,11 +1586,14 @@ int coli_v4_expert_store_open_planned(* would have taken goes to the experts' cache. */uint64_t device_bytes = 0;if (coli_v4_dense_device_decide && dense_bytes) {- uint64_t candidate = v4_dense_device_bytes(engine);+ int layers = engine->config.num_hidden_layers;+ uint64_t candidate = v4_dense_device_bytes(engine, layers);+ engine->dense_resident.device_layers = layers;if (candidate && candidate < dense_bytes &&coli_v4_dense_device_decide(engine, candidate, dense_bytes - candidate)) {- device_bytes = candidate;- dense_bytes -= candidate;+ /* a partial chain: only its layers' tensors leave RAM */+ device_bytes = v4_dense_device_bytes(engine, engine->dense_resident.device_layers);+ dense_bytes -= device_bytes;}}ColiDeepSeekV4ResidentTierPlan tiers;@@ -12654,6 +12659,7 @@ typedef struct {} V4VkEntry;static V4VkEntry *g_v4_vk_map;static size_t g_v4_vk_cap, g_v4_vk_used;+static int v4c_partial(void); /* deepseek_v4_chain.h: a partial chain keeps the rest on the CPU */static size_t v4_vk_hash(const void *data, size_t cap) {uint64_t h = (uint64_t)(uintptr_t)data * 0x9E3779B97F4A7C15ull;@@ -12692,6 +12698,17 @@ static V4VkEntry *v4_vk_insert(const void *data, int fmt, int rows, int columns)return &g_v4_vk_map[at];}+/* A tensor of the map off the device (a layer that did not complete): freed, its entry+ * refused, so the matrix stays the CPU's. */+static void v4_vk_forget(ColiVkTensor *t) {+ if (!t) return;+ for (size_t i = 0; i < g_v4_vk_cap; i++)+ if (g_v4_vk_map[i].data && g_v4_vk_map[i].tensor == t) {+ g_v4_vk_map[i].tensor = NULL; g_v4_vk_map[i].refused = 1; g_v4_vk_map[i].dho = 0;+ }+ coli_vk_tensor_free(t);+}+/* Bytes a resident layer tensor occupies (E8M0 scales are held as f32). */static size_t v4_vk_spec_bytes(const ColiDeepSeekV4TensorSpec *spec) {size_t count = 1;@@ -12742,6 +12759,8 @@ static int v4_vk_matmul_impl(int fmt, const void *data, const float *scales,* says, for the matrices the device holds alone; the rest stay where it puts them. */if (!coli_vk_dense() && !(e && e->dho)) return -1;if (e && e->refused) return -1;+ /* a partial chain: the CPU's layers and the head stay on the CPU, nothing new goes up */+ if (v4c_partial() && !(e && e->tensor)) return -1;if (e && e->tensor)return coli_vk_matmul(&e->tensor, output, input, NULL, NULL, fmt, batch,columns, rows, fmt == 12 ? 128 : 0) ? 0 : -1;@@ -12920,7 +12939,7 @@ static void v4_vk_tier_start(const ColiV4Engine *engine) {? (size_t)(st.capacity_bytes - st.resident_bytes) : 0,/* what the device has still to take: matrices placed already* (the dense weights on the device only) are in its usage */- .dense_bytes = coli_vk_dense()+ .dense_bytes = coli_vk_dense() && !v4c_partial() /* a partial chain placed all it places */? (size_t)(engine->dense_resident.total_bytes - engine->dense_resident.dropped_bytes + engine->head_cache.bytes) : 0,.in_ram = v4_vkt_in_ram, .ram_ctx = store,.load = v4_vkt_load, .release = v4_vkt_unhold, .load_ctx = store};@@ -12991,6 +13010,21 @@ static void v4c_close(void);static int g_v4_vk_opened, g_v4_dho_asked;static pthread_mutex_t g_v4_dho_mx = PTHREAD_MUTEX_INITIALIZER;static int v4c_decide(const ColiV4Engine *engine); /* deepseek_v4_chain.h */+static int v4c_fit_now(ColiV4Engine *engine); /* the partial chain's N (vkc_fit) */+static int v4c_fit_n(void);+static int v4c_build_layer(int layer);+static void v4c_drop_layer(int layer, const char *why);+/* The bytes of the first `layers` layers' tensors a device may hold alone. */+static uint64_t v4_dho_bytes(const ColiV4Engine *engine, int layers) {+ uint64_t total = 0;+ for (int layer = 0; layer < layers && layer < engine->config.num_hidden_layers; layer++) {+ ColiDeepSeekV4LayerPlan plan;+ if (coli_v4_layer_plan(&plan, &engine->config, layer, NULL, 0)) return 0;+ for (size_t i = 0; i < plan.tensor_count; i++)+ if (coli_v4_dense_device_only_tensor(&plan.tensors[i])) total += coli_v4_dense_tensor_bytes(&plan.tensors[i]);+ }+ return total;+}static void v4_vk_device_open(const ColiV4Engine *engine) {if (g_v4_vk_opened) return;@@ -13016,7 +13050,17 @@ static int v4_dho_decide(ColiV4Engine *engine, uint64_t device_bytes, uint64_t hv4_vk_device_open(engine);if (!g_v4_vk_ready) return 0;g_v4_dho_asked = 1;- int chain = v4c_decide(engine);+ int chain = v4c_decide(engine), L = engine->config.num_hidden_layers;+ engine->dense_resident.device_layers = L;+ if (chain && v4c_fit_now(engine)) {+ /* the chain's N decided now, before any upload: only its layers drop their host+ * copies and leave the plan's RAM; the CPU's layers keep theirs */+ int n = v4c_fit_n();+ engine->dense_resident.device_layers = n;+ int r = coli_vk_dense_host_decide("deepseek_v4", n > 0, (size_t)v4_dho_bytes(engine, n));+ if (r) coli_vk_dense_host_layers(n, L);+ return r && n > 0;+ }return coli_vk_dense_host_decide("deepseek_v4", chain != 0 || coli_vk_dense(), (size_t)device_bytes);}/* The CPU needs a matrix the device holds alone: read back from disk at its address. */@@ -13059,6 +13103,11 @@ static unsigned g_v4_dho_kept;static int v4_dho_place(ColiV4Engine *engine, int layer) {ColiDeepSeekV4LayerWeights *w = &engine->dense_resident.layers[layer];const ColiDeepSeekV4Config *c = &engine->config;+ /* a chained layer goes up whole first (its matrices and state, v4c_build_layer); one+ * that does not keeps its host copies, and so does every layer after it */+ int chained = v4c_build_layer(layer);+ if (chained < 0) return 0;+ signed char up[COLI_V4_MAX_LAYER_TENSORS] = {0}; /* 1 on the device, -1 refused */for (size_t i = 0; i < w->plan.tensor_count; i++) {const ColiDeepSeekV4TensorSpec *spec = &w->plan.tensors[i];if (!w->mapped[i] || w->host_gone[i]) continue;@@ -13078,8 +13127,17 @@ static int v4_dho_place(ColiV4Engine *engine, int layer) {spec->packed_rows8, rows, I);} else ok = v4_dho_up(12, data, scales, spec->packed_rows8, O, I);} else ok = v4_dho_up(11, w->data[i], NULL, 0, O, I);- if (ok && !coli_v4_layer_host_drop(engine, layer, i))- coli_vk_dense_host_dropped((size_t)coli_v4_dense_tensor_bytes(spec));+ up[i] = ok ? 1 : -1;+ if (!ok && chained) { /* a chained layer is the device's whole or not at all */+ v4c_drop_layer(layer, "a matrix the device refused");+ return 0;+ }+ }+ /* the pages back once the layer is on the device */+ for (size_t i = 0; i < w->plan.tensor_count; i++) {+ if (!up[i]) continue;+ if (up[i] > 0 && !coli_v4_layer_host_drop(engine, layer, i))+ coli_vk_dense_host_dropped((size_t)coli_v4_dense_tensor_bytes(&w->plan.tensors[i]));else g_v4_dho_kept++;}return 0;@@ -13109,6 +13167,7 @@ static void v4_vk_open(const ColiV4Engine *engine) {break;}}+ coli_vk_dense_host_layers(engine->dense_resident.device_layers, engine->config.num_hidden_layers);coli_vk_dense_host_placed("deepseek_v4", g_v4_dho_kept? "norms, block scales, the router, weights_proj, the mHC mixes, the head; and matrices the device refused": "norms, block scales, the router, the indexer's weights_proj, the mHC mixes, the head");@@ -13717,15 +13776,18 @@ static int target_batch_impl(ColiV4Engine *engine, float **state_ptr, float **ne* does not change the caller's semantic distinction: speculative decode* always passes use_prefill_pool=0. */int pool_experts = use_prefill_pool && v4_prefill_pool_enabled();+ int first_layer = 0;#ifdef COLI_VULKAN- { /* the dense chain runs every layer on the device; 0: the CPU runs them, as below */+ { /* the dense chain runs its layers on the device (every layer, or a partial chain's+ * first ones, the CPU the rest from their streams); 0: the CPU runs them all */int chained = v4c_forward(engine, state_ptr, next_ptr, attention, config, experts, tokens,start, batch, pool_experts, error, error_size);- if (chained) return chained > 0 ? 0 : -1;- v4c_cpu_step(start);+ if (chained < 0) return -1;+ if (!chained) v4c_cpu_step(start);+ else { first_layer = chained; state = *state_ptr; next = *next_ptr; }}#endif- for (int layer_id = 0; layer_id < config->num_hidden_layers; layer_id++) {+ for (int layer_id = first_layer; layer_id < config->num_hidden_layers; layer_id++) {ColiDeepSeekV4LayerWeights layer;if (coli_v4_layer_load(engine, &layer, config, index, layer_id,error, error_size)) {@@ -13822,15 +13884,17 @@ static int target_token_impl(ColiV4Engine *engine, float **state_ptr, float **nechar *error, size_t error_size) {float *state = *state_ptr, *next = *next_ptr;if (load_embedding(state, index, config, token)) return -1;+ int first_layer = 0;#ifdef COLI_VULKAN{ /* the dense chain, as in target_batch */int chained = v4c_forward(engine, state_ptr, next_ptr, attention, config, experts, &token,position, 1, 0, error, error_size);- if (chained) return chained > 0 ? 0 : -1;- v4c_cpu_step(position);+ if (chained < 0) return -1;+ if (!chained) v4c_cpu_step(position);+ else { first_layer = chained; state = *state_ptr; next = *next_ptr; }}#endif- for (int layer_id = 0; layer_id < config->num_hidden_layers; layer_id++) {+ for (int layer_id = first_layer; layer_id < config->num_hidden_layers; layer_id++) {ColiDeepSeekV4LayerWeights layer;if (coli_v4_layer_load(engine, &layer, config, index, layer_id,error, error_size)) return -1;diff --git a/c/deepseek_v4_chain.h b/c/deepseek_v4_chain.hindex 2686e943..1b3f876e 100644--- a/c/deepseek_v4_chain.h+++ b/c/deepseek_v4_chain.h@@ -89,6 +89,7 @@ typedef struct {ColiVkTensor *cwkv, *cwg, *iwkv, *iwg, *iwq, *iwp;VkcBuf *win, *ckv, *ring, *ikey, *iring;int ccap, kv_valid, ring_ok, icap, ik_valid, iring_ok;+ int built; /* its tensors and state reached the device */size_t pw, pc, pr, pk, pir; /* this forward's places in the pull buffer */int ns, q1; /* the ring rows it pulls: all (the overlap), or the slots of q1.. */} V4cLayer;@@ -96,6 +97,7 @@ typedef struct {typedef struct {int ok, failed, told;ColiV4Engine *engine;+ int Lm; /* the model's layers; L below: the chain's, its first L */int L, D, H, HD, nm, hr, nh, hd, rd, QL, og, ol, gw, W, Wd, IH, ID, I, K, rmax, rows, nkind;int kratio[COLI_V4_MAX_LAYERS + 1]; /* the ratio of each list kind (0: the indexer's) */V4cLayer *ly;@@ -247,41 +249,26 @@ static const char *v4c_unsupported(const ColiDeepSeekV4Config *c) {return NULL;}-static int v4c_setup(ColiV4Engine *engine) {+/* ---- the partial chain (vk_chain.h, vkc_fit): the first N layers on the device ---------- */+static VkcFit g_v4c_fit;+static int g_v4c_fit_done, g_v4c_building;++/* The chain's geometry for its first L layers (nothing allocated): the per-layer kinds,+ * the split's tables, the list kinds. Pointers are left alone, so it runs again when a+ * layer that did not reach the device lowers L. */+static void v4c_geom(V4Chain *ch, ColiV4Engine *engine, int L) {const ColiDeepSeekV4Config *c = &engine->config;- const char *why = v4c_unsupported(c);- if (!why && !engine->runtime.dense_resident) why = "the dense layers are not resident (a low-memory plan reloads them per forward)";- if (!why && (engine->gpu.enabled || (engine->experts && engine->experts->gpu))) why = "the CUDA tier is on and keeps its place";- if (why) { fprintf(stderr, "[VK] deepseek_v4 chain: %s; the CPU runs the layers\n", why); return 0; }- int L = c->num_hidden_layers;- char err[256] = {0};- const ColiSafetensorsIndex *index = coli_v4_engine_target_index(engine);- for (int i = 0; i < L; i++) { /* the resident layers, loaded now */- ColiDeepSeekV4LayerWeights w;- if (coli_v4_layer_load(engine, &w, c, index, i, err, sizeof err) || !engine->dense_resident.ready[i]) {- fprintf(stderr, "[VK] deepseek_v4 chain: layer %d did not load (%s); the CPU runs the layers\n", i, err);- return 0;- }- }- V4Chain *ch = calloc(1, sizeof *ch);- if (!ch || !(ch->ly = calloc((size_t)L, sizeof *ch->ly)) || !(ch->sli = malloc((size_t)L * sizeof(int)))) {- if (ch) free(ch->ly);- free(ch); return 0;- }- g_v4c = ch;- ch->engine = engine; ch->L = L; ch->D = c->hidden_size; ch->H = c->hc_mult; ch->HD = ch->H * ch->D;+ ch->engine = engine; ch->Lm = c->num_hidden_layers; ch->L = L;+ ch->D = c->hidden_size; ch->H = c->hc_mult; ch->HD = ch->H * ch->D;ch->nm = (2 + ch->H) * ch->H; ch->hr = 2 * ch->H + ch->H * ch->H;ch->nh = c->num_attention_heads; ch->hd = c->head_dim; ch->rd = c->qk_rope_head_dim; ch->QL = c->q_lora_rank;ch->og = c->o_groups; ch->ol = c->o_lora_rank; ch->gw = (ch->nh / ch->og) * ch->hd;- ch->W = c->sliding_window; ch->Wd = ch->W + v4c_rows();- if (!(ch->slot_pos = malloc((size_t)ch->Wd * sizeof(int)))) return 0;- for (int k = 0; k < ch->Wd; k++) ch->slot_pos[k] = -1;+ ch->W = c->sliding_window;ch->IH = c->index_n_heads > 0 ? c->index_n_heads : 1; ch->ID = c->index_head_dim > 0 ? c->index_head_dim : 2;- ch->I = c->moe_intermediate_size; ch->rmax = 1; ch->E = -1; ch->nkind = 1;- int D = ch->D, nm = ch->nm;+ ch->I = c->moe_intermediate_size; ch->rmax = 1; ch->nkind = 1; ch->nsl = 0; ch->rmin = 0;for (int i = 0; i < L; i++) {V4cLayer *ly = &ch->ly[i];- ly->ratio = c->compress_ratios[i]; ly->idx = ly->ratio == 4;+ ly->ratio = c->compress_ratios[i]; ly->idx = ly->ratio == 4; ly->kind = 0;if (ly->ratio > ch->rmax) ch->rmax = ly->ratio;ly->crows = ly->ratio == 4 ? 8 : ly->ratio; ly->cproj = (ly->ratio == 4 ? 2 : 1) * ch->hd;ch->sli[i] = ly->ratio > 0 ? ch->nsl++ : -1; /* the split's tables: one per compressed layer */@@ -293,20 +280,249 @@ static int v4c_setup(ColiV4Engine *engine) {ly->kind = k;}}- /* the parameter arena: norms decoded from bf16, sinks, hc scales and bases, the rings' biases */+}+/* The parameter arena's offsets for the chain's layers: norms decoded from bf16, sinks,+ * hc scales and bases, the rings' biases. Returns its floats. */+static size_t v4c_offsets(V4Chain *ch) {size_t n = 0;- for (int i = 0; i < L; i++) {+ for (int i = 0; i < ch->L; i++) {V4cLayer *ly = &ch->ly[i];- ly->o_an = n; n += D; ly->o_fn = n; n += D; ly->o_qn = n; n += ch->QL; ly->o_kn = n; n += ch->hd;- ly->o_sink = n; n += ch->nh; ly->o_hca = n; n += 3 + nm; ly->o_hcf = n; n += 3 + nm;+ ly->o_an = n; n += ch->D; ly->o_fn = n; n += ch->D; ly->o_qn = n; n += ch->QL; ly->o_kn = n; n += ch->hd;+ ly->o_sink = n; n += ch->nh; ly->o_hca = n; n += 3 + ch->nm; ly->o_hcf = n; n += 3 + ch->nm;if (ly->ratio > 0) { ly->o_cn = n; n += ch->hd; ly->o_ape = n; n += (size_t)ly->ratio * ly->cproj; }if (ly->idx) { ly->o_icn = n; n += ch->ID; ly->o_iape = n; n += (size_t)4 * 2 * ch->ID; }}+ return n;+}+/* What layer i puts on the device, as vkc_fit counts it: its matrices (the chain's set,+ * which holds the device-only placement's), its state at its first size (the window+ * ring, the compressors' rings, 64 compressed rows and keys), its part of the parameters+ * and of a forward's pull buffer at `rows` rows; *mat its matrices' payload. */+static size_t v4c_layer_bytes(const V4Chain *ch, int i, int rows, size_t *mat) {+ const ColiDeepSeekV4Config *c = &ch->engine->config;+ ColiDeepSeekV4LayerPlan plan;+ size_t b = 0, m = 0;+ *mat = 0;+ if (coli_v4_layer_plan(&plan, c, i, NULL, 0)) return 0;+ for (size_t t = 0; t < plan.tensor_count; t++) {+ const ColiDeepSeekV4TensorSpec *sp = &plan.tensors[t];+ if (sp->rank != 2) continue;+ const char *suf = sp->name;+ if (!strncmp(suf, "layers.", 7)) { const char *d = strchr(suf + 7, '.'); suf = d ? d + 1 : suf; }+ int O = (int)sp->shape[0], I = (int)sp->shape[1], fmt = 0, parts = 1;+ if (sp->dtype == COLI_ST_F8_E4M3) {+ fmt = 12;+ if (!strcmp(suf, "attn.wo_a.weight")) { parts = ch->og; O = ch->ol; } /* one tensor per output group */+ } else if (sp->dtype == COLI_ST_BF16 &&+ (coli_v4_dense_device_only_tensor(sp) || !strcmp(suf, "attn.indexer.weights_proj.weight"))) fmt = 11;+ else if (sp->dtype == COLI_ST_F32 && (!strcmp(suf, "hc_attn_fn") || !strcmp(suf, "hc_ffn_fn"))) fmt = 10;+ if (!fmt) continue;+ for (int p = 0; p < parts; p++) {+ b += vkc_fit_tensor(fmt, I, O, fmt == 12 ? 128 : 0);+ m += coli_vk_tensor_payload(fmt, I, O, fmt == 12 ? 128 : 0);+ }+ }+ const V4cLayer *ly = &ch->ly[i];+ int W = ch->W, hd = ch->hd, ID = ch->ID, E = rows;+ b += vkc_fit_buf((size_t)(W + v4c_rows()) * hd * sizeof(float));+ if (ly->ratio > 0) b += vkc_fit_buf((size_t)2 * ly->crows * ly->cproj * sizeof(float)) + vkc_fit_buf((size_t)64 * hd * sizeof(float));+ if (ly->idx) b += vkc_fit_buf((size_t)2 * 8 * 2 * ID * sizeof(float)) + vkc_fit_buf((size_t)64 * ID * sizeof(float));+ size_t f = (size_t)2 * ch->D + ch->QL + hd + ch->nh + (size_t)2 * (3 + ch->nm);+ if (ly->ratio > 0) f += hd + (size_t)ly->ratio * ly->cproj;+ if (ly->idx) f += ID + (size_t)8 * ID;+ f += (size_t)(E < W ? E : W) * hd;+ if (ly->ratio > 0) f += (size_t)(E / ly->ratio) * hd + (size_t)2 * (ly->ratio == 4 ? ly->crows : E < ly->ratio ? E : ly->ratio) * ly->cproj;+ if (ly->idx) f += (size_t)(E / 4) * ID + (size_t)2 * 8 * 2 * ID;+ *mat = m;+ return b + f * sizeof(float);+}+static int v4c_scratch(V4Chain *ch, int rows, int E, int LR);+/* What the chain allocates whatever N: the scratch of one forward of `rows` rows (the+ * counting pass of v4c_scratch, every layer's list kinds). */+static size_t v4c_fixed_bytes(V4Chain *ch, int rows) {+ const ColiDeepSeekV4Config *c = &ch->engine->config;+ int E = ch->W + rows, K = 0;+ for (int i = 0; i < ch->L; i++) {+ V4cLayer *ly = &ch->ly[i];+ int k = ly->idx ? c->index_topk : ly->ratio > 0 ? E / ly->ratio : 0;+ if (k > K) K = k;+ }+ g_v4c_count = 0;+ v4c_scratch(ch, rows, E, ch->W + (K > 0 ? K : 1));+ size_t b = (size_t)g_v4c_count;+ g_v4c_count = -1;+ return b;+}+/* With every layer on the device, the per-matrix path (COLI_VK_DENSE) also puts the bf16+ * head and each layer's router there as they are first used: the tail. */+static size_t v4c_tail_bytes(const ColiV4Engine *engine) {+ if (!coli_vk_dense()) return 0;+ const ColiDeepSeekV4Config *c = &engine->config;+ return vkc_fit_tensor(11, c->hidden_size, c->vocab_size, 0) ++ (size_t)c->num_hidden_layers * vkc_fit_tensor(11, c->hidden_size, c->n_routed_experts, 0);+}+static const char *v4c_unsupported(const ColiDeepSeekV4Config *c);+static int v4c_decide(const ColiV4Engine *engine);+/* The decision (vkc_fit), once and before any upload: by the RAM plan when it asks about+ * the device-only weights, else at v4c_start. 0: no fit (the chain will not run). */+static int v4c_fit_now(ColiV4Engine *engine) {+ if (g_v4c_fit_done) return g_v4c_fit.L > 0;+ g_v4c_fit_done = 1;+ if (!g_v4_vk_ready || !engine || !v4c_decide(engine) || v4c_unsupported(&engine->config)) return 0;+ int L = engine->config.num_hidden_layers;+ V4Chain t;+ memset(&t, 0, sizeof t);+ t.ly = calloc((size_t)L, sizeof *t.ly); t.sli = malloc((size_t)L * sizeof(int));+ size_t *per = calloc((size_t)L, sizeof(size_t)), *mat = calloc((size_t)L, sizeof(size_t));+ int ok = t.ly && t.sli && per && mat;+ if (ok) {+ v4c_geom(&t, engine, L);+ int rows = v4c_rows();+ for (int i = 0; i < L; i++) per[i] = v4c_layer_bytes(&t, i, rows, &mat[i]);+ vkc_fit("deepseek_v4", L, per, mat, v4c_fixed_bytes(&t, rows), v4c_tail_bytes(engine), &g_v4c_fit);+ }+ free(t.ly); free(t.sli); free(per); free(mat);+ return ok;+}+static int v4c_fit_n(void) { return g_v4c_fit.n; }+/* 1: something the full chain would place stays on the CPU (the CPU layers, the head):+ * the per-matrix path uploads nothing new. */+static int v4c_partial(void) { return g_v4c_fit_done && vkc_fit_partial(&g_v4c_fit); }++/* Layer i's matrices (the per-matrix path's device copies, one per matrix) and its state+ * on the device. 0 with *why set: something did not reach it (v4c_layer_free undoes it). */+static int v4c_layer_up(V4Chain *ch, int i, const char **why) {+ V4cLayer *ly = &ch->ly[i];+ const ColiDeepSeekV4LayerWeights *w = &ch->engine->dense_resident.layers[i];+ *why = "a matrix the device refused";+ ly->wo_a = calloc((size_t)ch->og, sizeof(void *));+ int ok = ly->wo_a && (ly->fna = v4c_plain(w, "hc_attn_fn", 10)) && (ly->fnf = v4c_plain(w, "hc_ffn_fn", 10)) &&+ (ly->wq_a = v4c_fp8(w, "attn.wq_a", 0, 0)) && (ly->wq_b = v4c_fp8(w, "attn.wq_b", 0, 0)) &&+ (ly->wkv = v4c_fp8(w, "attn.wkv", 0, 0)) && (ly->wo_b = v4c_fp8(w, "attn.wo_b", 0, 0)) &&+ (ly->sh1 = v4c_fp8(w, "ffn.shared_experts.w1", 0, 0)) && (ly->sh2 = v4c_fp8(w, "ffn.shared_experts.w2", 0, 0)) &&+ (ly->sh3 = v4c_fp8(w, "ffn.shared_experts.w3", 0, 0));+ for (int g = 0; ok && g < ch->og; g++) ok = (ly->wo_a[g] = v4c_fp8(w, "attn.wo_a", g, ch->ol)) != NULL;+ if (ok && ly->ratio > 0)+ ok = (ly->cwkv = v4c_plain(w, "attn.compressor.wkv.weight", 11)) && (ly->cwg = v4c_plain(w, "attn.compressor.wgate.weight", 11));+ if (ok && ly->idx)+ ok = (ly->iwkv = v4c_plain(w, "attn.indexer.compressor.wkv.weight", 11)) &&+ (ly->iwg = v4c_plain(w, "attn.indexer.compressor.wgate.weight", 11)) &&+ (ly->iwp = v4c_plain(w, "attn.indexer.weights_proj.weight", 11)) && (ly->iwq = v4c_fp8(w, "attn.indexer.wq_b", 0, 0));+ if (!ok) return 0;+ *why = "device memory for its state refused";+ if (ly->ratio > 0 && !(ly->ring = vkc_buf((size_t)2 * ly->crows * ly->cproj * sizeof(float), VKC_DEV))) return 0;+ if (ly->idx && !(ly->iring = vkc_buf((size_t)2 * 8 * 2 * ch->ID * sizeof(float), VKC_DEV))) return 0;+ return (ly->win = vkc_buf((size_t)ch->Wd * ch->hd * sizeof(float), VKC_DEV)) != NULL;+}+static void v4_vk_forget(ColiVkTensor *t); /* the per-matrix map's entry: its tensor freed, the matrix the CPU's */+/* Everything of layer i off the device: its tensors (and their per-matrix entries, which+ * stay refused: the matrix is the CPU's) and its buffers, after the frames that read them. */+static void v4c_layer_free(V4Chain *ch, int i) {+ V4cLayer *ly = &ch->ly[i];+ if (vkc_ready()) vkc_finish();+ ColiVkTensor **ts[] = {&ly->fna, &ly->fnf, &ly->wq_a, &ly->wq_b, &ly->wkv, &ly->wo_b, &ly->sh1, &ly->sh2, &ly->sh3,+ &ly->cwkv, &ly->cwg, &ly->iwkv, &ly->iwg, &ly->iwq, &ly->iwp};+ for (size_t k = 0; k < sizeof ts / sizeof *ts; k++) { v4_vk_forget(*ts[k]); *ts[k] = NULL; }+ for (int g = 0; ly->wo_a && g < ch->og; g++) v4_vk_forget(ly->wo_a[g]);+ free(ly->wo_a); ly->wo_a = NULL;+ VkcBuf **bs[] = {&ly->win, &ly->ckv, &ly->ring, &ly->ikey, &ly->iring};+ for (size_t k = 0; k < sizeof bs / sizeof *bs; k++) { vkc_free(*bs[k]); *bs[k] = NULL; }+ ly->ccap = ly->icap = ly->kv_valid = ly->ik_valid = ly->ring_ok = ly->iring_ok = 0;+ ly->built = 0;+}+/* Layer i did not reach the device: the chain keeps the layers before it, and with the+ * dense weights on the device only, the layers from i on keep their host copies. */+static void v4c_cut(V4Chain *ch, int i, const char *why) {+ ColiV4Engine *e = ch->engine;+ if (i < ch->L) ch->L = i;+ if (e->dense_resident.device_only && e->dense_resident.device_layers > i) {+ fprintf(stderr, "[VK] deepseek_v4: layers %d to %d keep their host copies after all (the RAM plan had counted "+ "them on the device)\n", i, e->dense_resident.device_layers - 1);+ e->dense_resident.device_layers = i;+ coli_vk_dense_host_layers(i, ch->Lm);+ }+ vkc_fit_shrink("deepseek_v4", &g_v4c_fit, i, why);+}+/* The placement of the dense weights on the device only (v4_dho_place) builds the chain's+ * layer first while the chain is being set up: 1 built, 0 not the chain's, -1 it did not+ * reach the device (the chain was cut there; the layer keeps its host copies). */+static int v4c_build_layer(int layer) {+ V4Chain *ch = g_v4c;+ if (!g_v4c_building || !ch || layer >= ch->L) return 0;+ if (ch->ly[layer].built) return 1;+ const char *why = NULL;+ if (v4c_layer_up(ch, layer, &why)) { ch->ly[layer].built = 1; return 1; }+ v4c_layer_free(ch, layer);+ v4c_cut(ch, layer, why);+ return -1;+}+/* The device-only placement of a chained layer failed after the chain built it: all of+ * the layer off the device, the chain cut before it. */+static void v4c_drop_layer(int layer, const char *why) {+ if (!g_v4c) return;+ v4c_layer_free(g_v4c, layer);+ v4c_cut(g_v4c, layer, why);+}+/* Every chained layer off the device and the chain off (a layer that would not load, the+ * parameters refused): what the device held alone comes back from disk when the CPU asks. */+static void v4c_teardown(V4Chain *ch, const char *why) {+ for (int i = 0; i < ch->L; i++) if (ch->ly[i].built) v4c_layer_free(ch, i);+ vkc_free(ch->prm); ch->prm = NULL;+ v4c_cut(ch, 0, why);+}++static int v4c_setup(ColiV4Engine *engine) {+ const ColiDeepSeekV4Config *c = &engine->config;+ const char *why = v4c_unsupported(c);+ if (!why && !engine->runtime.dense_resident) why = "the dense layers are not resident (a low-memory plan reloads them per forward)";+ if (!why && (engine->gpu.enabled || (engine->experts && engine->experts->gpu))) why = "the CUDA tier is on and keeps its place";+ if (why) { fprintf(stderr, "[VK] deepseek_v4 chain: %s; the CPU runs the layers\n", why); return 0; }+ int L = c->num_hidden_layers, N = g_v4c_fit.n;+ if (N < 1) return 0; /* the fit's line said so: the chain stays off */+ V4Chain *ch = calloc(1, sizeof *ch);+ if (!ch || !(ch->ly = calloc((size_t)L, sizeof *ch->ly)) || !(ch->sli = malloc((size_t)L * sizeof(int)))) {+ if (ch) free(ch->ly);+ free(ch); return 0;+ }+ g_v4c = ch;+ v4c_geom(ch, engine, N);+ ch->Wd = ch->W + v4c_rows(); ch->E = -1;+ if (!(ch->slot_pos = malloc((size_t)ch->Wd * sizeof(int)))) return 0;+ for (int k = 0; k < ch->Wd; k++) ch->slot_pos[k] = -1;+ size_t n = v4c_offsets(ch);+ if (!(ch->prm = vkc_buf((n ? n : 1) * sizeof(float), VKC_DEV))) {+ v4c_cut(ch, 0, "device memory for the parameters refused");+ return 0;+ }+ /* the resident layers, loaded now; each chained one goes up whole (with the dense+ * weights on the device only, in its placement), or the chain stops before it */+ char err[256] = {0};+ const ColiSafetensorsIndex *index = coli_v4_engine_target_index(engine);+ g_v4c_building = 1;+ for (int i = 0; i < L; i++) {+ ColiDeepSeekV4LayerWeights w;+ if (coli_v4_layer_load(engine, &w, c, index, i, err, sizeof err) || !engine->dense_resident.ready[i]) {+ g_v4c_building = 0;+ fprintf(stderr, "[VK] deepseek_v4 chain: layer %d did not load (%s); the CPU runs the layers\n", i, err);+ v4c_teardown(ch, "a layer did not load");+ return 0;+ }+ if (i >= ch->L) continue;+ if (v4c_build_layer(i) > 0) vkc_fit_mark(&g_v4c_fit, i);+ }+ g_v4c_building = 0;+ if (ch->L < 1) { /* layer 0 did not reach the device: nothing of the chain stays there */+ vkc_free(ch->prm); ch->prm = NULL;+ vkc_fit_placed("deepseek_v4", &g_v4c_fit);+ return 0;+ }+ v4c_geom(ch, engine, ch->L); /* the split's tables and list kinds for the layers it kept */float *a = calloc(n ? n : 1, sizeof(float));int ok = a != NULL;- for (int i = 0; ok && i < L; i++) {+ for (int i = 0; ok && i < ch->L; i++) {V4cLayer *ly = &ch->ly[i];const ColiDeepSeekV4LayerWeights *w = &engine->dense_resident.layers[i];+ int D = ch->D, nm = ch->nm;const uint16_t *an = v4c_val(w, "attn_norm.weight"), *fn = v4c_val(w, "ffn_norm.weight"),*qn = v4c_val(w, "attn.q_norm.weight"), *kn = v4c_val(w, "attn.kv_norm.weight");const float *sink = v4c_val(w, "attn.attn_sink"), *hsa = v4c_val(w, "hc_attn_scale"), *hba = v4c_val(w, "hc_attn_base"),@@ -335,39 +551,23 @@ static int v4c_setup(ColiV4Engine *engine) {if (ok) memcpy(a + ly->o_iape, iape, (size_t)8 * ch->ID * sizeof(float));}}- ok = ok && (ch->prm = vkc_buf((n ? n : 1) * sizeof(float), VKC_DEV)) && vkc_begin() &&- vkc_write(ch->prm, 0, a, n * sizeof(float)) && vkc_submit(1);+ ok = ok && vkc_begin() && vkc_write(ch->prm, 0, a, n * sizeof(float)) && vkc_submit(1);free(a);- /* the tensors and the per-layer state on the device */- for (int i = 0; ok && i < L; i++) {- V4cLayer *ly = &ch->ly[i];- const ColiDeepSeekV4LayerWeights *w = &engine->dense_resident.layers[i];- ly->wo_a = calloc((size_t)ch->og, sizeof(void *));- ok = ly->wo_a && (ly->fna = v4c_plain(w, "hc_attn_fn", 10)) && (ly->fnf = v4c_plain(w, "hc_ffn_fn", 10)) &&- (ly->wq_a = v4c_fp8(w, "attn.wq_a", 0, 0)) && (ly->wq_b = v4c_fp8(w, "attn.wq_b", 0, 0)) &&- (ly->wkv = v4c_fp8(w, "attn.wkv", 0, 0)) && (ly->wo_b = v4c_fp8(w, "attn.wo_b", 0, 0)) &&- (ly->sh1 = v4c_fp8(w, "ffn.shared_experts.w1", 0, 0)) && (ly->sh2 = v4c_fp8(w, "ffn.shared_experts.w2", 0, 0)) &&- (ly->sh3 = v4c_fp8(w, "ffn.shared_experts.w3", 0, 0));- for (int g = 0; ok && g < ch->og; g++) ok = (ly->wo_a[g] = v4c_fp8(w, "attn.wo_a", g, ch->ol)) != NULL;- if (ok && ly->ratio > 0)- ok = (ly->cwkv = v4c_plain(w, "attn.compressor.wkv.weight", 11)) && (ly->cwg = v4c_plain(w, "attn.compressor.wgate.weight", 11)) &&- (ly->ring = vkc_buf((size_t)2 * ly->crows * ly->cproj * sizeof(float), VKC_DEV));- if (ok && ly->idx)- ok = (ly->iwkv = v4c_plain(w, "attn.indexer.compressor.wkv.weight", 11)) &&- (ly->iwg = v4c_plain(w, "attn.indexer.compressor.wgate.weight", 11)) &&- (ly->iwp = v4c_plain(w, "attn.indexer.weights_proj.weight", 11)) && (ly->iwq = v4c_fp8(w, "attn.indexer.wq_b", 0, 0)) &&- (ly->iring = vkc_buf((size_t)2 * 8 * 2 * ch->ID * sizeof(float), VKC_DEV));- if (ok) ok = (ly->win = vkc_buf((size_t)ch->Wd * ch->hd * sizeof(float), VKC_DEV)) != NULL;+ if (!ok) {+ fprintf(stderr, "[VK] deepseek_v4 chain: the parameters did not reach the device; the CPU runs the layers\n");+ v4c_teardown(ch, "the parameters did not reach it");+ vkc_fit_placed("deepseek_v4", &g_v4c_fit);+ return 0;}- if (!ok) { fprintf(stderr, "[VK] deepseek_v4 chain: a matrix or the state did not reach the device; the CPU runs the layers\n"); return 0; }ch->ok = 1;int ncomp = 0, nidx = 0;- for (int i = 0; i < L; i++) { ncomp += ch->ly[i].ratio > 0; nidx += ch->ly[i].idx; }+ for (int i = 0; i < ch->L; i++) { ncomp += ch->ly[i].ratio > 0; nidx += ch->ly[i].idx; }size_t bytes = 0, tensors = 0;coli_vk_mem_info(&bytes, &tensors);fprintf(stderr, "[VK] deepseek_v4 chain: %d layers on the device (%d compressing, %d indexing), %d streams, ""%.1f MiB of parameters, %zu matrices (%.1f MiB) on the device\n",- L, ncomp, nidx, ch->H, n * 4 / 1048576.0, tensors, bytes / 1048576.0);+ ch->L, ncomp, nidx, ch->H, n * 4 / 1048576.0, tensors, bytes / 1048576.0);+ vkc_fit_placed("deepseek_v4", &g_v4c_fit);return 1;}@@ -663,10 +863,12 @@ static int v4c_read_count(ColiDeepSeekV4WindowAttentionState *s, int *count, intreturn 1;}-/* Every layer for the n rows of streams in *state_ptr (positions start..), the final- * streams into *state_ptr (swapped with *next_ptr, as the CPU's layer loop leaves them).- * 1: done; 0: not taken (the CPU runs the layers, nothing changed); -1: the MoE failed- * (error set, the host state as before the forward). */+/* The chain's layers (the first ch->L of the model's) for the n rows of streams in+ * *state_ptr (positions start..), the streams after its last layer into *state_ptr+ * (swapped with *next_ptr, as the CPU's layer loop leaves them): the handoff of a partial+ * chain, where the caller's CPU loop runs the remaining layers from there. Returns the+ * layers it ran (ch->L); 0: not taken (the CPU runs every layer, nothing changed); -1: the+ * MoE failed (error set, the host state as before the forward). */static int v4c_forward(ColiV4Engine *engine, float **state_ptr, float **next_ptr,ColiDeepSeekV4WindowAttentionState **attention, const ColiDeepSeekV4Config *config,ColiExpertStore *experts, const int *tokens, int start, int n, int pool,@@ -761,7 +963,8 @@ static int v4c_forward(ColiV4Engine *engine, float **state_ptr, float **next_ptrstatic int ident = -1;if (ident < 0) { const char *e = getenv("V4_IDX_IDENTITY"); ident = e && *e == '1'; }float *state = *state_ptr, *next = *next_ptr;- int T0 = L - 3, taps = coli_v4_full_dspark_wanted; /* DSpark's targets: the last three layers */+ int T0 = ch->Lm - 3, taps = coli_v4_full_dspark_wanted; /* DSpark's targets: the model's last three layers+ * (those the chain runs; the CPU taps the rest) */for (int c0 = 0; c0 < n; c0 += rows) {int nr = n - c0 < rows ? n - c0 : rows, pb = start + c0, last = c0 + nr == n;V4cFwd f = {start, n, pb, nr, 0, nr * rd, LR, 0};@@ -973,7 +1176,7 @@ static int v4c_forward(ColiV4Engine *engine, float **state_ptr, float **next_ptr*state_ptr = next; *next_ptr = state;free(cs); free(wl); free(cos_); free(sin_);ch->forwards++;- return 1;+ return L; /* a partial chain: the CPU runs layers L.. from these streams */host_oom: /* the host state half written: as a failed CPU forward leaves it, an error */free(cs); free(wl); free(cos_); free(sin_);v4c_lower(0);@@ -1022,6 +1225,7 @@ static void v4c_start(const ColiV4Engine *cengine) {no = "the MLA, mHC or DeepSeek shaders are missing (chain_hgemv, chain_mhc, chain_dsv4)";if (no) fprintf(stderr, "[VK] deepseek_v4: %s: the dense chain stays off\n", no);if (!on || no) return;+ if (!v4c_fit_now(engine) || g_v4c_fit.n < 1) return; /* how many layers fit (vkc_fit), once */if (!v4c_setup(engine)) return;g_v4c_mode = on;}diff --git a/c/deepseek_v4_internal.h b/c/deepseek_v4_internal.hindex 945dccd9..3321e2d7 100644--- a/c/deepseek_v4_internal.h+++ b/c/deepseek_v4_internal.h@@ -289,6 +289,8 @@ const void *coli_v4_layer_data(const ColiDeepSeekV4LayerWeights *weights,* coli_v4_dense_device_decide: asked by the RAM plan (EXPERT_STORE_AUTO) once it has* measured the memory, with the bytes the device would hold and the RAM it would give* back; 1 = the dense layers live on the device, their droppable tensors out of RAM.+ * It sets dense_resident.device_layers: only those first layers go (a partial chain+ * takes a prefix of the layers), and the plan counts only their bytes out of RAM.* coli_v4_dense_place: after a resident layer is read, its tensors to the device and* the droppable ones' pages given back (coli_v4_layer_host_drop).* coli_v4_dense_device_lost: 1 once the device is gone; resident layers are then read@@ -1035,6 +1037,11 @@ struct ColiV4Engine {* (the plan left them out of RAM); what the device holds and what RAM gave back */int device_only;uint64_t device_bytes, dropped_bytes;+ /* the layers whose droppable tensors the device holds alone: the first+ * device_layers (a partial chain, docs/vulkan.md "A partial chain"); the others+ * keep their host copies. Set by coli_v4_dense_device_decide, lowered when a+ * layer does not reach the device. */+ int device_layers;} dense_resident;/* Optional CUDA tier (compiled in only when the engine build defines* COLI_V4_GPU_TIER on Windows). enabled is 1 only after the loader resolved