Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(deepseek_v4): a partial dense chain, the first N layers on the device

JustVugga201e1fe2850
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.c
index 5725f0b4..7424cdb7 100644
--- a/c/deepseek_v4.c
+++ b/c/deepseek_v4.c
@@ -1078,8 +1078,8 @@ int coli_v4_layer_load(ColiV4Engine *engine,
if (!resident_enabled_v2(engine))
return coli_v4_layer_resident_reference_load(
NULL, weights, effective_config, index, layer, error, error_size);
- if (engine->dense_resident.device_only && coli_v4_dense_device_lost &&
- coli_v4_dense_device_lost()) {
+ int device_layer = engine->dense_resident.device_only && layer < engine->dense_resident.device_layers;
+ if (device_layer && coli_v4_dense_device_lost && coli_v4_dense_device_lost()) {
/* the device that held the dense layers is gone: the CPU reads them per forward,
* as a low-memory plan does (its two-layer reserve is in the plan) */
if (coli_v4_layer_resident_reference_load(
@@ -1097,7 +1097,7 @@ int coli_v4_layer_load(ColiV4Engine *engine,
}
engine->dense_resident.index = index;
if (!engine->dense_resident.ready[layer]) {
- g_v4_layer_map = engine->dense_resident.device_only; /* droppable tensors in mappings of their own */
+ g_v4_layer_map = device_layer; /* droppable tensors in mappings of their own */
int failed = coli_v4_layer_resident_reference_load(
NULL, &engine->dense_resident.layers[layer], effective_config, index,
layer, error, error_size);
@@ -1113,8 +1113,9 @@ int coli_v4_layer_load(ColiV4Engine *engine,
engine->dense_resident.total_bytes +=
engine->dense_resident.layers[layer].stats.total_bytes;
/* device only: the layer's matrices to the device and their host pages back,
- * before the next layer is read (one layer in RAM at a time) */
- if (engine->dense_resident.device_only && coli_v4_dense_place)
+ * before the next layer is read (one layer in RAM at a time); a partial chain's
+ * CPU layers keep theirs */
+ if (device_layer && coli_v4_dense_place)
coli_v4_dense_place(engine, layer);
if (layer == effective_config->num_hidden_layers - 1)
fprintf(stderr, "v4_dense_resident layers=%d bytes=%.3fGiB\n",
@@ -1545,10 +1546,11 @@ static int build_runtime_plan(ColiV4Engine *engine,
* binary built with VK=1, NULL in every other link. */
ColiV4DenseDeviceDecide coli_v4_dense_device_decide;
-/* The bytes of the dense layers a device may hold alone (coli_v4_dense_device_only_tensor). */
-static uint64_t v4_dense_device_bytes(const ColiV4Engine *engine) {
+/* The bytes of the first `layers` dense layers a device may hold alone
+ * (coli_v4_dense_device_only_tensor). */
+static uint64_t v4_dense_device_bytes(const ColiV4Engine *engine, int layers) {
uint64_t total = 0;
- for (int layer = 0; layer < engine->config.num_hidden_layers; layer++) {
+ for (int layer = 0; layer < layers && layer < engine->config.num_hidden_layers; layer++) {
ColiDeepSeekV4LayerPlan plan;
if (coli_v4_layer_plan(&plan, &engine->config, layer, NULL, 0)) return 0;
for (size_t i = 0; i < plan.tensor_count; i++)
@@ -1584,11 +1586,14 @@ int coli_v4_expert_store_open_planned(
* would have taken goes to the experts' cache. */
uint64_t device_bytes = 0;
if (coli_v4_dense_device_decide && dense_bytes) {
- uint64_t candidate = v4_dense_device_bytes(engine);
+ int layers = engine->config.num_hidden_layers;
+ uint64_t candidate = v4_dense_device_bytes(engine, layers);
+ engine->dense_resident.device_layers = layers;
if (candidate && candidate < dense_bytes &&
coli_v4_dense_device_decide(engine, candidate, dense_bytes - candidate)) {
- device_bytes = candidate;
- dense_bytes -= candidate;
+ /* a partial chain: only its layers' tensors leave RAM */
+ device_bytes = v4_dense_device_bytes(engine, engine->dense_resident.device_layers);
+ dense_bytes -= device_bytes;
}
}
ColiDeepSeekV4ResidentTierPlan tiers;
@@ -12654,6 +12659,7 @@ typedef struct {
} V4VkEntry;
static V4VkEntry *g_v4_vk_map;
static size_t g_v4_vk_cap, g_v4_vk_used;
+static int v4c_partial(void); /* deepseek_v4_chain.h: a partial chain keeps the rest on the CPU */
static size_t v4_vk_hash(const void *data, size_t cap) {
uint64_t h = (uint64_t)(uintptr_t)data * 0x9E3779B97F4A7C15ull;
@@ -12692,6 +12698,17 @@ static V4VkEntry *v4_vk_insert(const void *data, int fmt, int rows, int columns)
return &g_v4_vk_map[at];
}
+/* A tensor of the map off the device (a layer that did not complete): freed, its entry
+ * refused, so the matrix stays the CPU's. */
+static void v4_vk_forget(ColiVkTensor *t) {
+ if (!t) return;
+ for (size_t i = 0; i < g_v4_vk_cap; i++)
+ if (g_v4_vk_map[i].data && g_v4_vk_map[i].tensor == t) {
+ g_v4_vk_map[i].tensor = NULL; g_v4_vk_map[i].refused = 1; g_v4_vk_map[i].dho = 0;
+ }
+ coli_vk_tensor_free(t);
+}
+
/* Bytes a resident layer tensor occupies (E8M0 scales are held as f32). */
static size_t v4_vk_spec_bytes(const ColiDeepSeekV4TensorSpec *spec) {
size_t count = 1;
@@ -12742,6 +12759,8 @@ static int v4_vk_matmul_impl(int fmt, const void *data, const float *scales,
* says, for the matrices the device holds alone; the rest stay where it puts them. */
if (!coli_vk_dense() && !(e && e->dho)) return -1;
if (e && e->refused) return -1;
+ /* a partial chain: the CPU's layers and the head stay on the CPU, nothing new goes up */
+ if (v4c_partial() && !(e && e->tensor)) return -1;
if (e && e->tensor)
return coli_vk_matmul(&e->tensor, output, input, NULL, NULL, fmt, batch,
columns, rows, fmt == 12 ? 128 : 0) ? 0 : -1;
@@ -12920,7 +12939,7 @@ static void v4_vk_tier_start(const ColiV4Engine *engine) {
? (size_t)(st.capacity_bytes - st.resident_bytes) : 0,
/* what the device has still to take: matrices placed already
* (the dense weights on the device only) are in its usage */
- .dense_bytes = coli_vk_dense()
+ .dense_bytes = coli_vk_dense() && !v4c_partial() /* a partial chain placed all it places */
? (size_t)(engine->dense_resident.total_bytes - engine->dense_resident.dropped_bytes + engine->head_cache.bytes) : 0,
.in_ram = v4_vkt_in_ram, .ram_ctx = store,
.load = v4_vkt_load, .release = v4_vkt_unhold, .load_ctx = store};
@@ -12991,6 +13010,21 @@ static void v4c_close(void);
static int g_v4_vk_opened, g_v4_dho_asked;
static pthread_mutex_t g_v4_dho_mx = PTHREAD_MUTEX_INITIALIZER;
static int v4c_decide(const ColiV4Engine *engine); /* deepseek_v4_chain.h */
+static int v4c_fit_now(ColiV4Engine *engine); /* the partial chain's N (vkc_fit) */
+static int v4c_fit_n(void);
+static int v4c_build_layer(int layer);
+static void v4c_drop_layer(int layer, const char *why);
+/* The bytes of the first `layers` layers' tensors a device may hold alone. */
+static uint64_t v4_dho_bytes(const ColiV4Engine *engine, int layers) {
+ uint64_t total = 0;
+ for (int layer = 0; layer < layers && layer < engine->config.num_hidden_layers; layer++) {
+ ColiDeepSeekV4LayerPlan plan;
+ if (coli_v4_layer_plan(&plan, &engine->config, layer, NULL, 0)) return 0;
+ for (size_t i = 0; i < plan.tensor_count; i++)
+ if (coli_v4_dense_device_only_tensor(&plan.tensors[i])) total += coli_v4_dense_tensor_bytes(&plan.tensors[i]);
+ }
+ return total;
+}
static void v4_vk_device_open(const ColiV4Engine *engine) {
if (g_v4_vk_opened) return;
@@ -13016,7 +13050,17 @@ static int v4_dho_decide(ColiV4Engine *engine, uint64_t device_bytes, uint64_t h
v4_vk_device_open(engine);
if (!g_v4_vk_ready) return 0;
g_v4_dho_asked = 1;
- int chain = v4c_decide(engine);
+ int chain = v4c_decide(engine), L = engine->config.num_hidden_layers;
+ engine->dense_resident.device_layers = L;
+ if (chain && v4c_fit_now(engine)) {
+ /* the chain's N decided now, before any upload: only its layers drop their host
+ * copies and leave the plan's RAM; the CPU's layers keep theirs */
+ int n = v4c_fit_n();
+ engine->dense_resident.device_layers = n;
+ int r = coli_vk_dense_host_decide("deepseek_v4", n > 0, (size_t)v4_dho_bytes(engine, n));
+ if (r) coli_vk_dense_host_layers(n, L);
+ return r && n > 0;
+ }
return coli_vk_dense_host_decide("deepseek_v4", chain != 0 || coli_vk_dense(), (size_t)device_bytes);
}
/* The CPU needs a matrix the device holds alone: read back from disk at its address. */
@@ -13059,6 +13103,11 @@ static unsigned g_v4_dho_kept;
static int v4_dho_place(ColiV4Engine *engine, int layer) {
ColiDeepSeekV4LayerWeights *w = &engine->dense_resident.layers[layer];
const ColiDeepSeekV4Config *c = &engine->config;
+ /* a chained layer goes up whole first (its matrices and state, v4c_build_layer); one
+ * that does not keeps its host copies, and so does every layer after it */
+ int chained = v4c_build_layer(layer);
+ if (chained < 0) return 0;
+ signed char up[COLI_V4_MAX_LAYER_TENSORS] = {0}; /* 1 on the device, -1 refused */
for (size_t i = 0; i < w->plan.tensor_count; i++) {
const ColiDeepSeekV4TensorSpec *spec = &w->plan.tensors[i];
if (!w->mapped[i] || w->host_gone[i]) continue;
@@ -13078,8 +13127,17 @@ static int v4_dho_place(ColiV4Engine *engine, int layer) {
spec->packed_rows8, rows, I);
} else ok = v4_dho_up(12, data, scales, spec->packed_rows8, O, I);
} else ok = v4_dho_up(11, w->data[i], NULL, 0, O, I);
- if (ok && !coli_v4_layer_host_drop(engine, layer, i))
- coli_vk_dense_host_dropped((size_t)coli_v4_dense_tensor_bytes(spec));
+ up[i] = ok ? 1 : -1;
+ if (!ok && chained) { /* a chained layer is the device's whole or not at all */
+ v4c_drop_layer(layer, "a matrix the device refused");
+ return 0;
+ }
+ }
+ /* the pages back once the layer is on the device */
+ for (size_t i = 0; i < w->plan.tensor_count; i++) {
+ if (!up[i]) continue;
+ if (up[i] > 0 && !coli_v4_layer_host_drop(engine, layer, i))
+ coli_vk_dense_host_dropped((size_t)coli_v4_dense_tensor_bytes(&w->plan.tensors[i]));
else g_v4_dho_kept++;
}
return 0;
@@ -13109,6 +13167,7 @@ static void v4_vk_open(const ColiV4Engine *engine) {
break;
}
}
+ coli_vk_dense_host_layers(engine->dense_resident.device_layers, engine->config.num_hidden_layers);
coli_vk_dense_host_placed("deepseek_v4", g_v4_dho_kept
? "norms, block scales, the router, weights_proj, the mHC mixes, the head; and matrices the device refused"
: "norms, block scales, the router, the indexer's weights_proj, the mHC mixes, the head");
@@ -13717,15 +13776,18 @@ static int target_batch_impl(ColiV4Engine *engine, float **state_ptr, float **ne
* does not change the caller's semantic distinction: speculative decode
* always passes use_prefill_pool=0. */
int pool_experts = use_prefill_pool && v4_prefill_pool_enabled();
+ int first_layer = 0;
#ifdef COLI_VULKAN
- { /* the dense chain runs every layer on the device; 0: the CPU runs them, as below */
+ { /* the dense chain runs its layers on the device (every layer, or a partial chain's
+ * first ones, the CPU the rest from their streams); 0: the CPU runs them all */
int chained = v4c_forward(engine, state_ptr, next_ptr, attention, config, experts, tokens,
start, batch, pool_experts, error, error_size);
- if (chained) return chained > 0 ? 0 : -1;
- v4c_cpu_step(start);
+ if (chained < 0) return -1;
+ if (!chained) v4c_cpu_step(start);
+ else { first_layer = chained; state = *state_ptr; next = *next_ptr; }
}
#endif
- for (int layer_id = 0; layer_id < config->num_hidden_layers; layer_id++) {
+ for (int layer_id = first_layer; layer_id < config->num_hidden_layers; layer_id++) {
ColiDeepSeekV4LayerWeights layer;
if (coli_v4_layer_load(engine, &layer, config, index, layer_id,
error, error_size)) {
@@ -13822,15 +13884,17 @@ static int target_token_impl(ColiV4Engine *engine, float **state_ptr, float **ne
char *error, size_t error_size) {
float *state = *state_ptr, *next = *next_ptr;
if (load_embedding(state, index, config, token)) return -1;
+ int first_layer = 0;
#ifdef COLI_VULKAN
{ /* the dense chain, as in target_batch */
int chained = v4c_forward(engine, state_ptr, next_ptr, attention, config, experts, &token,
position, 1, 0, error, error_size);
- if (chained) return chained > 0 ? 0 : -1;
- v4c_cpu_step(position);
+ if (chained < 0) return -1;
+ if (!chained) v4c_cpu_step(position);
+ else { first_layer = chained; state = *state_ptr; next = *next_ptr; }
}
#endif
- for (int layer_id = 0; layer_id < config->num_hidden_layers; layer_id++) {
+ for (int layer_id = first_layer; layer_id < config->num_hidden_layers; layer_id++) {
ColiDeepSeekV4LayerWeights layer;
if (coli_v4_layer_load(engine, &layer, config, index, layer_id,
error, error_size)) return -1;
diff --git a/c/deepseek_v4_chain.h b/c/deepseek_v4_chain.h
index 2686e943..1b3f876e 100644
--- a/c/deepseek_v4_chain.h
+++ b/c/deepseek_v4_chain.h
@@ -89,6 +89,7 @@ typedef struct {
ColiVkTensor *cwkv, *cwg, *iwkv, *iwg, *iwq, *iwp;
VkcBuf *win, *ckv, *ring, *ikey, *iring;
int ccap, kv_valid, ring_ok, icap, ik_valid, iring_ok;
+ int built; /* its tensors and state reached the device */
size_t pw, pc, pr, pk, pir; /* this forward's places in the pull buffer */
int ns, q1; /* the ring rows it pulls: all (the overlap), or the slots of q1.. */
} V4cLayer;
@@ -96,6 +97,7 @@ typedef struct {
typedef struct {
int ok, failed, told;
ColiV4Engine *engine;
+ int Lm; /* the model's layers; L below: the chain's, its first L */
int L, D, H, HD, nm, hr, nh, hd, rd, QL, og, ol, gw, W, Wd, IH, ID, I, K, rmax, rows, nkind;
int kratio[COLI_V4_MAX_LAYERS + 1]; /* the ratio of each list kind (0: the indexer's) */
V4cLayer *ly;
@@ -247,41 +249,26 @@ static const char *v4c_unsupported(const ColiDeepSeekV4Config *c) {
return NULL;
}
-static int v4c_setup(ColiV4Engine *engine) {
+/* ---- the partial chain (vk_chain.h, vkc_fit): the first N layers on the device ---------- */
+static VkcFit g_v4c_fit;
+static int g_v4c_fit_done, g_v4c_building;
+
+/* The chain's geometry for its first L layers (nothing allocated): the per-layer kinds,
+ * the split's tables, the list kinds. Pointers are left alone, so it runs again when a
+ * layer that did not reach the device lowers L. */
+static void v4c_geom(V4Chain *ch, ColiV4Engine *engine, int L) {
const ColiDeepSeekV4Config *c = &engine->config;
- const char *why = v4c_unsupported(c);
- if (!why && !engine->runtime.dense_resident) why = "the dense layers are not resident (a low-memory plan reloads them per forward)";
- if (!why && (engine->gpu.enabled || (engine->experts && engine->experts->gpu))) why = "the CUDA tier is on and keeps its place";
- if (why) { fprintf(stderr, "[VK] deepseek_v4 chain: %s; the CPU runs the layers\n", why); return 0; }
- int L = c->num_hidden_layers;
- char err[256] = {0};
- const ColiSafetensorsIndex *index = coli_v4_engine_target_index(engine);
- for (int i = 0; i < L; i++) { /* the resident layers, loaded now */
- ColiDeepSeekV4LayerWeights w;
- if (coli_v4_layer_load(engine, &w, c, index, i, err, sizeof err) || !engine->dense_resident.ready[i]) {
- fprintf(stderr, "[VK] deepseek_v4 chain: layer %d did not load (%s); the CPU runs the layers\n", i, err);
- return 0;
- }
- }
- V4Chain *ch = calloc(1, sizeof *ch);
- if (!ch || !(ch->ly = calloc((size_t)L, sizeof *ch->ly)) || !(ch->sli = malloc((size_t)L * sizeof(int)))) {
- if (ch) free(ch->ly);
- free(ch); return 0;
- }
- g_v4c = ch;
- ch->engine = engine; ch->L = L; ch->D = c->hidden_size; ch->H = c->hc_mult; ch->HD = ch->H * ch->D;
+ ch->engine = engine; ch->Lm = c->num_hidden_layers; ch->L = L;
+ ch->D = c->hidden_size; ch->H = c->hc_mult; ch->HD = ch->H * ch->D;
ch->nm = (2 + ch->H) * ch->H; ch->hr = 2 * ch->H + ch->H * ch->H;
ch->nh = c->num_attention_heads; ch->hd = c->head_dim; ch->rd = c->qk_rope_head_dim; ch->QL = c->q_lora_rank;
ch->og = c->o_groups; ch->ol = c->o_lora_rank; ch->gw = (ch->nh / ch->og) * ch->hd;
- ch->W = c->sliding_window; ch->Wd = ch->W + v4c_rows();
- if (!(ch->slot_pos = malloc((size_t)ch->Wd * sizeof(int)))) return 0;
- for (int k = 0; k < ch->Wd; k++) ch->slot_pos[k] = -1;
+ ch->W = c->sliding_window;
ch->IH = c->index_n_heads > 0 ? c->index_n_heads : 1; ch->ID = c->index_head_dim > 0 ? c->index_head_dim : 2;
- ch->I = c->moe_intermediate_size; ch->rmax = 1; ch->E = -1; ch->nkind = 1;
- int D = ch->D, nm = ch->nm;
+ ch->I = c->moe_intermediate_size; ch->rmax = 1; ch->nkind = 1; ch->nsl = 0; ch->rmin = 0;
for (int i = 0; i < L; i++) {
V4cLayer *ly = &ch->ly[i];
- ly->ratio = c->compress_ratios[i]; ly->idx = ly->ratio == 4;
+ ly->ratio = c->compress_ratios[i]; ly->idx = ly->ratio == 4; ly->kind = 0;
if (ly->ratio > ch->rmax) ch->rmax = ly->ratio;
ly->crows = ly->ratio == 4 ? 8 : ly->ratio; ly->cproj = (ly->ratio == 4 ? 2 : 1) * ch->hd;
ch->sli[i] = ly->ratio > 0 ? ch->nsl++ : -1; /* the split's tables: one per compressed layer */
@@ -293,20 +280,249 @@ static int v4c_setup(ColiV4Engine *engine) {
ly->kind = k;
}
}
- /* the parameter arena: norms decoded from bf16, sinks, hc scales and bases, the rings' biases */
+}
+/* The parameter arena's offsets for the chain's layers: norms decoded from bf16, sinks,
+ * hc scales and bases, the rings' biases. Returns its floats. */
+static size_t v4c_offsets(V4Chain *ch) {
size_t n = 0;
- for (int i = 0; i < L; i++) {
+ for (int i = 0; i < ch->L; i++) {
V4cLayer *ly = &ch->ly[i];
- ly->o_an = n; n += D; ly->o_fn = n; n += D; ly->o_qn = n; n += ch->QL; ly->o_kn = n; n += ch->hd;
- ly->o_sink = n; n += ch->nh; ly->o_hca = n; n += 3 + nm; ly->o_hcf = n; n += 3 + nm;
+ ly->o_an = n; n += ch->D; ly->o_fn = n; n += ch->D; ly->o_qn = n; n += ch->QL; ly->o_kn = n; n += ch->hd;
+ ly->o_sink = n; n += ch->nh; ly->o_hca = n; n += 3 + ch->nm; ly->o_hcf = n; n += 3 + ch->nm;
if (ly->ratio > 0) { ly->o_cn = n; n += ch->hd; ly->o_ape = n; n += (size_t)ly->ratio * ly->cproj; }
if (ly->idx) { ly->o_icn = n; n += ch->ID; ly->o_iape = n; n += (size_t)4 * 2 * ch->ID; }
}
+ return n;
+}
+/* What layer i puts on the device, as vkc_fit counts it: its matrices (the chain's set,
+ * which holds the device-only placement's), its state at its first size (the window
+ * ring, the compressors' rings, 64 compressed rows and keys), its part of the parameters
+ * and of a forward's pull buffer at `rows` rows; *mat its matrices' payload. */
+static size_t v4c_layer_bytes(const V4Chain *ch, int i, int rows, size_t *mat) {
+ const ColiDeepSeekV4Config *c = &ch->engine->config;
+ ColiDeepSeekV4LayerPlan plan;
+ size_t b = 0, m = 0;
+ *mat = 0;
+ if (coli_v4_layer_plan(&plan, c, i, NULL, 0)) return 0;
+ for (size_t t = 0; t < plan.tensor_count; t++) {
+ const ColiDeepSeekV4TensorSpec *sp = &plan.tensors[t];
+ if (sp->rank != 2) continue;
+ const char *suf = sp->name;
+ if (!strncmp(suf, "layers.", 7)) { const char *d = strchr(suf + 7, '.'); suf = d ? d + 1 : suf; }
+ int O = (int)sp->shape[0], I = (int)sp->shape[1], fmt = 0, parts = 1;
+ if (sp->dtype == COLI_ST_F8_E4M3) {
+ fmt = 12;
+ if (!strcmp(suf, "attn.wo_a.weight")) { parts = ch->og; O = ch->ol; } /* one tensor per output group */
+ } else if (sp->dtype == COLI_ST_BF16 &&
+ (coli_v4_dense_device_only_tensor(sp) || !strcmp(suf, "attn.indexer.weights_proj.weight"))) fmt = 11;
+ else if (sp->dtype == COLI_ST_F32 && (!strcmp(suf, "hc_attn_fn") || !strcmp(suf, "hc_ffn_fn"))) fmt = 10;
+ if (!fmt) continue;
+ for (int p = 0; p < parts; p++) {
+ b += vkc_fit_tensor(fmt, I, O, fmt == 12 ? 128 : 0);
+ m += coli_vk_tensor_payload(fmt, I, O, fmt == 12 ? 128 : 0);
+ }
+ }
+ const V4cLayer *ly = &ch->ly[i];
+ int W = ch->W, hd = ch->hd, ID = ch->ID, E = rows;
+ b += vkc_fit_buf((size_t)(W + v4c_rows()) * hd * sizeof(float));
+ if (ly->ratio > 0) b += vkc_fit_buf((size_t)2 * ly->crows * ly->cproj * sizeof(float)) + vkc_fit_buf((size_t)64 * hd * sizeof(float));
+ if (ly->idx) b += vkc_fit_buf((size_t)2 * 8 * 2 * ID * sizeof(float)) + vkc_fit_buf((size_t)64 * ID * sizeof(float));
+ size_t f = (size_t)2 * ch->D + ch->QL + hd + ch->nh + (size_t)2 * (3 + ch->nm);
+ if (ly->ratio > 0) f += hd + (size_t)ly->ratio * ly->cproj;
+ if (ly->idx) f += ID + (size_t)8 * ID;
+ f += (size_t)(E < W ? E : W) * hd;
+ if (ly->ratio > 0) f += (size_t)(E / ly->ratio) * hd + (size_t)2 * (ly->ratio == 4 ? ly->crows : E < ly->ratio ? E : ly->ratio) * ly->cproj;
+ if (ly->idx) f += (size_t)(E / 4) * ID + (size_t)2 * 8 * 2 * ID;
+ *mat = m;
+ return b + f * sizeof(float);
+}
+static int v4c_scratch(V4Chain *ch, int rows, int E, int LR);
+/* What the chain allocates whatever N: the scratch of one forward of `rows` rows (the
+ * counting pass of v4c_scratch, every layer's list kinds). */
+static size_t v4c_fixed_bytes(V4Chain *ch, int rows) {
+ const ColiDeepSeekV4Config *c = &ch->engine->config;
+ int E = ch->W + rows, K = 0;
+ for (int i = 0; i < ch->L; i++) {
+ V4cLayer *ly = &ch->ly[i];
+ int k = ly->idx ? c->index_topk : ly->ratio > 0 ? E / ly->ratio : 0;
+ if (k > K) K = k;
+ }
+ g_v4c_count = 0;
+ v4c_scratch(ch, rows, E, ch->W + (K > 0 ? K : 1));
+ size_t b = (size_t)g_v4c_count;
+ g_v4c_count = -1;
+ return b;
+}
+/* With every layer on the device, the per-matrix path (COLI_VK_DENSE) also puts the bf16
+ * head and each layer's router there as they are first used: the tail. */
+static size_t v4c_tail_bytes(const ColiV4Engine *engine) {
+ if (!coli_vk_dense()) return 0;
+ const ColiDeepSeekV4Config *c = &engine->config;
+ return vkc_fit_tensor(11, c->hidden_size, c->vocab_size, 0) +
+ (size_t)c->num_hidden_layers * vkc_fit_tensor(11, c->hidden_size, c->n_routed_experts, 0);
+}
+static const char *v4c_unsupported(const ColiDeepSeekV4Config *c);
+static int v4c_decide(const ColiV4Engine *engine);
+/* The decision (vkc_fit), once and before any upload: by the RAM plan when it asks about
+ * the device-only weights, else at v4c_start. 0: no fit (the chain will not run). */
+static int v4c_fit_now(ColiV4Engine *engine) {
+ if (g_v4c_fit_done) return g_v4c_fit.L > 0;
+ g_v4c_fit_done = 1;
+ if (!g_v4_vk_ready || !engine || !v4c_decide(engine) || v4c_unsupported(&engine->config)) return 0;
+ int L = engine->config.num_hidden_layers;
+ V4Chain t;
+ memset(&t, 0, sizeof t);
+ t.ly = calloc((size_t)L, sizeof *t.ly); t.sli = malloc((size_t)L * sizeof(int));
+ size_t *per = calloc((size_t)L, sizeof(size_t)), *mat = calloc((size_t)L, sizeof(size_t));
+ int ok = t.ly && t.sli && per && mat;
+ if (ok) {
+ v4c_geom(&t, engine, L);
+ int rows = v4c_rows();
+ for (int i = 0; i < L; i++) per[i] = v4c_layer_bytes(&t, i, rows, &mat[i]);
+ vkc_fit("deepseek_v4", L, per, mat, v4c_fixed_bytes(&t, rows), v4c_tail_bytes(engine), &g_v4c_fit);
+ }
+ free(t.ly); free(t.sli); free(per); free(mat);
+ return ok;
+}
+static int v4c_fit_n(void) { return g_v4c_fit.n; }
+/* 1: something the full chain would place stays on the CPU (the CPU layers, the head):
+ * the per-matrix path uploads nothing new. */
+static int v4c_partial(void) { return g_v4c_fit_done && vkc_fit_partial(&g_v4c_fit); }
+
+/* Layer i's matrices (the per-matrix path's device copies, one per matrix) and its state
+ * on the device. 0 with *why set: something did not reach it (v4c_layer_free undoes it). */
+static int v4c_layer_up(V4Chain *ch, int i, const char **why) {
+ V4cLayer *ly = &ch->ly[i];
+ const ColiDeepSeekV4LayerWeights *w = &ch->engine->dense_resident.layers[i];
+ *why = "a matrix the device refused";
+ ly->wo_a = calloc((size_t)ch->og, sizeof(void *));
+ int ok = ly->wo_a && (ly->fna = v4c_plain(w, "hc_attn_fn", 10)) && (ly->fnf = v4c_plain(w, "hc_ffn_fn", 10)) &&
+ (ly->wq_a = v4c_fp8(w, "attn.wq_a", 0, 0)) && (ly->wq_b = v4c_fp8(w, "attn.wq_b", 0, 0)) &&
+ (ly->wkv = v4c_fp8(w, "attn.wkv", 0, 0)) && (ly->wo_b = v4c_fp8(w, "attn.wo_b", 0, 0)) &&
+ (ly->sh1 = v4c_fp8(w, "ffn.shared_experts.w1", 0, 0)) && (ly->sh2 = v4c_fp8(w, "ffn.shared_experts.w2", 0, 0)) &&
+ (ly->sh3 = v4c_fp8(w, "ffn.shared_experts.w3", 0, 0));
+ for (int g = 0; ok && g < ch->og; g++) ok = (ly->wo_a[g] = v4c_fp8(w, "attn.wo_a", g, ch->ol)) != NULL;
+ if (ok && ly->ratio > 0)
+ ok = (ly->cwkv = v4c_plain(w, "attn.compressor.wkv.weight", 11)) && (ly->cwg = v4c_plain(w, "attn.compressor.wgate.weight", 11));
+ if (ok && ly->idx)
+ ok = (ly->iwkv = v4c_plain(w, "attn.indexer.compressor.wkv.weight", 11)) &&
+ (ly->iwg = v4c_plain(w, "attn.indexer.compressor.wgate.weight", 11)) &&
+ (ly->iwp = v4c_plain(w, "attn.indexer.weights_proj.weight", 11)) && (ly->iwq = v4c_fp8(w, "attn.indexer.wq_b", 0, 0));
+ if (!ok) return 0;
+ *why = "device memory for its state refused";
+ if (ly->ratio > 0 && !(ly->ring = vkc_buf((size_t)2 * ly->crows * ly->cproj * sizeof(float), VKC_DEV))) return 0;
+ if (ly->idx && !(ly->iring = vkc_buf((size_t)2 * 8 * 2 * ch->ID * sizeof(float), VKC_DEV))) return 0;
+ return (ly->win = vkc_buf((size_t)ch->Wd * ch->hd * sizeof(float), VKC_DEV)) != NULL;
+}
+static void v4_vk_forget(ColiVkTensor *t); /* the per-matrix map's entry: its tensor freed, the matrix the CPU's */
+/* Everything of layer i off the device: its tensors (and their per-matrix entries, which
+ * stay refused: the matrix is the CPU's) and its buffers, after the frames that read them. */
+static void v4c_layer_free(V4Chain *ch, int i) {
+ V4cLayer *ly = &ch->ly[i];
+ if (vkc_ready()) vkc_finish();
+ ColiVkTensor **ts[] = {&ly->fna, &ly->fnf, &ly->wq_a, &ly->wq_b, &ly->wkv, &ly->wo_b, &ly->sh1, &ly->sh2, &ly->sh3,
+ &ly->cwkv, &ly->cwg, &ly->iwkv, &ly->iwg, &ly->iwq, &ly->iwp};
+ for (size_t k = 0; k < sizeof ts / sizeof *ts; k++) { v4_vk_forget(*ts[k]); *ts[k] = NULL; }
+ for (int g = 0; ly->wo_a && g < ch->og; g++) v4_vk_forget(ly->wo_a[g]);
+ free(ly->wo_a); ly->wo_a = NULL;
+ VkcBuf **bs[] = {&ly->win, &ly->ckv, &ly->ring, &ly->ikey, &ly->iring};
+ for (size_t k = 0; k < sizeof bs / sizeof *bs; k++) { vkc_free(*bs[k]); *bs[k] = NULL; }
+ ly->ccap = ly->icap = ly->kv_valid = ly->ik_valid = ly->ring_ok = ly->iring_ok = 0;
+ ly->built = 0;
+}
+/* Layer i did not reach the device: the chain keeps the layers before it, and with the
+ * dense weights on the device only, the layers from i on keep their host copies. */
+static void v4c_cut(V4Chain *ch, int i, const char *why) {
+ ColiV4Engine *e = ch->engine;
+ if (i < ch->L) ch->L = i;
+ if (e->dense_resident.device_only && e->dense_resident.device_layers > i) {
+ fprintf(stderr, "[VK] deepseek_v4: layers %d to %d keep their host copies after all (the RAM plan had counted "
+ "them on the device)\n", i, e->dense_resident.device_layers - 1);
+ e->dense_resident.device_layers = i;
+ coli_vk_dense_host_layers(i, ch->Lm);
+ }
+ vkc_fit_shrink("deepseek_v4", &g_v4c_fit, i, why);
+}
+/* The placement of the dense weights on the device only (v4_dho_place) builds the chain's
+ * layer first while the chain is being set up: 1 built, 0 not the chain's, -1 it did not
+ * reach the device (the chain was cut there; the layer keeps its host copies). */
+static int v4c_build_layer(int layer) {
+ V4Chain *ch = g_v4c;
+ if (!g_v4c_building || !ch || layer >= ch->L) return 0;
+ if (ch->ly[layer].built) return 1;
+ const char *why = NULL;
+ if (v4c_layer_up(ch, layer, &why)) { ch->ly[layer].built = 1; return 1; }
+ v4c_layer_free(ch, layer);
+ v4c_cut(ch, layer, why);
+ return -1;
+}
+/* The device-only placement of a chained layer failed after the chain built it: all of
+ * the layer off the device, the chain cut before it. */
+static void v4c_drop_layer(int layer, const char *why) {
+ if (!g_v4c) return;
+ v4c_layer_free(g_v4c, layer);
+ v4c_cut(g_v4c, layer, why);
+}
+/* Every chained layer off the device and the chain off (a layer that would not load, the
+ * parameters refused): what the device held alone comes back from disk when the CPU asks. */
+static void v4c_teardown(V4Chain *ch, const char *why) {
+ for (int i = 0; i < ch->L; i++) if (ch->ly[i].built) v4c_layer_free(ch, i);
+ vkc_free(ch->prm); ch->prm = NULL;
+ v4c_cut(ch, 0, why);
+}
+
+static int v4c_setup(ColiV4Engine *engine) {
+ const ColiDeepSeekV4Config *c = &engine->config;
+ const char *why = v4c_unsupported(c);
+ if (!why && !engine->runtime.dense_resident) why = "the dense layers are not resident (a low-memory plan reloads them per forward)";
+ if (!why && (engine->gpu.enabled || (engine->experts && engine->experts->gpu))) why = "the CUDA tier is on and keeps its place";
+ if (why) { fprintf(stderr, "[VK] deepseek_v4 chain: %s; the CPU runs the layers\n", why); return 0; }
+ int L = c->num_hidden_layers, N = g_v4c_fit.n;
+ if (N < 1) return 0; /* the fit's line said so: the chain stays off */
+ V4Chain *ch = calloc(1, sizeof *ch);
+ if (!ch || !(ch->ly = calloc((size_t)L, sizeof *ch->ly)) || !(ch->sli = malloc((size_t)L * sizeof(int)))) {
+ if (ch) free(ch->ly);
+ free(ch); return 0;
+ }
+ g_v4c = ch;
+ v4c_geom(ch, engine, N);
+ ch->Wd = ch->W + v4c_rows(); ch->E = -1;
+ if (!(ch->slot_pos = malloc((size_t)ch->Wd * sizeof(int)))) return 0;
+ for (int k = 0; k < ch->Wd; k++) ch->slot_pos[k] = -1;
+ size_t n = v4c_offsets(ch);
+ if (!(ch->prm = vkc_buf((n ? n : 1) * sizeof(float), VKC_DEV))) {
+ v4c_cut(ch, 0, "device memory for the parameters refused");
+ return 0;
+ }
+ /* the resident layers, loaded now; each chained one goes up whole (with the dense
+ * weights on the device only, in its placement), or the chain stops before it */
+ char err[256] = {0};
+ const ColiSafetensorsIndex *index = coli_v4_engine_target_index(engine);
+ g_v4c_building = 1;
+ for (int i = 0; i < L; i++) {
+ ColiDeepSeekV4LayerWeights w;
+ if (coli_v4_layer_load(engine, &w, c, index, i, err, sizeof err) || !engine->dense_resident.ready[i]) {
+ g_v4c_building = 0;
+ fprintf(stderr, "[VK] deepseek_v4 chain: layer %d did not load (%s); the CPU runs the layers\n", i, err);
+ v4c_teardown(ch, "a layer did not load");
+ return 0;
+ }
+ if (i >= ch->L) continue;
+ if (v4c_build_layer(i) > 0) vkc_fit_mark(&g_v4c_fit, i);
+ }
+ g_v4c_building = 0;
+ if (ch->L < 1) { /* layer 0 did not reach the device: nothing of the chain stays there */
+ vkc_free(ch->prm); ch->prm = NULL;
+ vkc_fit_placed("deepseek_v4", &g_v4c_fit);
+ return 0;
+ }
+ v4c_geom(ch, engine, ch->L); /* the split's tables and list kinds for the layers it kept */
float *a = calloc(n ? n : 1, sizeof(float));
int ok = a != NULL;
- for (int i = 0; ok && i < L; i++) {
+ for (int i = 0; ok && i < ch->L; i++) {
V4cLayer *ly = &ch->ly[i];
const ColiDeepSeekV4LayerWeights *w = &engine->dense_resident.layers[i];
+ int D = ch->D, nm = ch->nm;
const uint16_t *an = v4c_val(w, "attn_norm.weight"), *fn = v4c_val(w, "ffn_norm.weight"),
*qn = v4c_val(w, "attn.q_norm.weight"), *kn = v4c_val(w, "attn.kv_norm.weight");
const float *sink = v4c_val(w, "attn.attn_sink"), *hsa = v4c_val(w, "hc_attn_scale"), *hba = v4c_val(w, "hc_attn_base"),
@@ -335,39 +551,23 @@ static int v4c_setup(ColiV4Engine *engine) {
if (ok) memcpy(a + ly->o_iape, iape, (size_t)8 * ch->ID * sizeof(float));
}
}
- ok = ok && (ch->prm = vkc_buf((n ? n : 1) * sizeof(float), VKC_DEV)) && vkc_begin() &&
- vkc_write(ch->prm, 0, a, n * sizeof(float)) && vkc_submit(1);
+ ok = ok && vkc_begin() && vkc_write(ch->prm, 0, a, n * sizeof(float)) && vkc_submit(1);
free(a);
- /* the tensors and the per-layer state on the device */
- for (int i = 0; ok && i < L; i++) {
- V4cLayer *ly = &ch->ly[i];
- const ColiDeepSeekV4LayerWeights *w = &engine->dense_resident.layers[i];
- ly->wo_a = calloc((size_t)ch->og, sizeof(void *));
- ok = ly->wo_a && (ly->fna = v4c_plain(w, "hc_attn_fn", 10)) && (ly->fnf = v4c_plain(w, "hc_ffn_fn", 10)) &&
- (ly->wq_a = v4c_fp8(w, "attn.wq_a", 0, 0)) && (ly->wq_b = v4c_fp8(w, "attn.wq_b", 0, 0)) &&
- (ly->wkv = v4c_fp8(w, "attn.wkv", 0, 0)) && (ly->wo_b = v4c_fp8(w, "attn.wo_b", 0, 0)) &&
- (ly->sh1 = v4c_fp8(w, "ffn.shared_experts.w1", 0, 0)) && (ly->sh2 = v4c_fp8(w, "ffn.shared_experts.w2", 0, 0)) &&
- (ly->sh3 = v4c_fp8(w, "ffn.shared_experts.w3", 0, 0));
- for (int g = 0; ok && g < ch->og; g++) ok = (ly->wo_a[g] = v4c_fp8(w, "attn.wo_a", g, ch->ol)) != NULL;
- if (ok && ly->ratio > 0)
- ok = (ly->cwkv = v4c_plain(w, "attn.compressor.wkv.weight", 11)) && (ly->cwg = v4c_plain(w, "attn.compressor.wgate.weight", 11)) &&
- (ly->ring = vkc_buf((size_t)2 * ly->crows * ly->cproj * sizeof(float), VKC_DEV));
- if (ok && ly->idx)
- ok = (ly->iwkv = v4c_plain(w, "attn.indexer.compressor.wkv.weight", 11)) &&
- (ly->iwg = v4c_plain(w, "attn.indexer.compressor.wgate.weight", 11)) &&
- (ly->iwp = v4c_plain(w, "attn.indexer.weights_proj.weight", 11)) && (ly->iwq = v4c_fp8(w, "attn.indexer.wq_b", 0, 0)) &&
- (ly->iring = vkc_buf((size_t)2 * 8 * 2 * ch->ID * sizeof(float), VKC_DEV));
- if (ok) ok = (ly->win = vkc_buf((size_t)ch->Wd * ch->hd * sizeof(float), VKC_DEV)) != NULL;
+ if (!ok) {
+ fprintf(stderr, "[VK] deepseek_v4 chain: the parameters did not reach the device; the CPU runs the layers\n");
+ v4c_teardown(ch, "the parameters did not reach it");
+ vkc_fit_placed("deepseek_v4", &g_v4c_fit);
+ return 0;
}
- if (!ok) { fprintf(stderr, "[VK] deepseek_v4 chain: a matrix or the state did not reach the device; the CPU runs the layers\n"); return 0; }
ch->ok = 1;
int ncomp = 0, nidx = 0;
- for (int i = 0; i < L; i++) { ncomp += ch->ly[i].ratio > 0; nidx += ch->ly[i].idx; }
+ for (int i = 0; i < ch->L; i++) { ncomp += ch->ly[i].ratio > 0; nidx += ch->ly[i].idx; }
size_t bytes = 0, tensors = 0;
coli_vk_mem_info(&bytes, &tensors);
fprintf(stderr, "[VK] deepseek_v4 chain: %d layers on the device (%d compressing, %d indexing), %d streams, "
"%.1f MiB of parameters, %zu matrices (%.1f MiB) on the device\n",
- L, ncomp, nidx, ch->H, n * 4 / 1048576.0, tensors, bytes / 1048576.0);
+ ch->L, ncomp, nidx, ch->H, n * 4 / 1048576.0, tensors, bytes / 1048576.0);
+ vkc_fit_placed("deepseek_v4", &g_v4c_fit);
return 1;
}
@@ -663,10 +863,12 @@ static int v4c_read_count(ColiDeepSeekV4WindowAttentionState *s, int *count, int
return 1;
}
-/* Every layer for the n rows of streams in *state_ptr (positions start..), the final
- * streams into *state_ptr (swapped with *next_ptr, as the CPU's layer loop leaves them).
- * 1: done; 0: not taken (the CPU runs the layers, nothing changed); -1: the MoE failed
- * (error set, the host state as before the forward). */
+/* The chain's layers (the first ch->L of the model's) for the n rows of streams in
+ * *state_ptr (positions start..), the streams after its last layer into *state_ptr
+ * (swapped with *next_ptr, as the CPU's layer loop leaves them): the handoff of a partial
+ * chain, where the caller's CPU loop runs the remaining layers from there. Returns the
+ * layers it ran (ch->L); 0: not taken (the CPU runs every layer, nothing changed); -1: the
+ * MoE failed (error set, the host state as before the forward). */
static int v4c_forward(ColiV4Engine *engine, float **state_ptr, float **next_ptr,
ColiDeepSeekV4WindowAttentionState **attention, const ColiDeepSeekV4Config *config,
ColiExpertStore *experts, const int *tokens, int start, int n, int pool,
@@ -761,7 +963,8 @@ static int v4c_forward(ColiV4Engine *engine, float **state_ptr, float **next_ptr
static int ident = -1;
if (ident < 0) { const char *e = getenv("V4_IDX_IDENTITY"); ident = e && *e == '1'; }
float *state = *state_ptr, *next = *next_ptr;
- int T0 = L - 3, taps = coli_v4_full_dspark_wanted; /* DSpark's targets: the last three layers */
+ int T0 = ch->Lm - 3, taps = coli_v4_full_dspark_wanted; /* DSpark's targets: the model's last three layers
+ * (those the chain runs; the CPU taps the rest) */
for (int c0 = 0; c0 < n; c0 += rows) {
int nr = n - c0 < rows ? n - c0 : rows, pb = start + c0, last = c0 + nr == n;
V4cFwd f = {start, n, pb, nr, 0, nr * rd, LR, 0};
@@ -973,7 +1176,7 @@ static int v4c_forward(ColiV4Engine *engine, float **state_ptr, float **next_ptr
*state_ptr = next; *next_ptr = state;
free(cs); free(wl); free(cos_); free(sin_);
ch->forwards++;
- return 1;
+ return L; /* a partial chain: the CPU runs layers L.. from these streams */
host_oom: /* the host state half written: as a failed CPU forward leaves it, an error */
free(cs); free(wl); free(cos_); free(sin_);
v4c_lower(0);
@@ -1022,6 +1225,7 @@ static void v4c_start(const ColiV4Engine *cengine) {
no = "the MLA, mHC or DeepSeek shaders are missing (chain_hgemv, chain_mhc, chain_dsv4)";
if (no) fprintf(stderr, "[VK] deepseek_v4: %s: the dense chain stays off\n", no);
if (!on || no) return;
+ if (!v4c_fit_now(engine) || g_v4c_fit.n < 1) return; /* how many layers fit (vkc_fit), once */
if (!v4c_setup(engine)) return;
g_v4c_mode = on;
}
diff --git a/c/deepseek_v4_internal.h b/c/deepseek_v4_internal.h
index 945dccd9..3321e2d7 100644
--- a/c/deepseek_v4_internal.h
+++ b/c/deepseek_v4_internal.h
@@ -289,6 +289,8 @@ const void *coli_v4_layer_data(const ColiDeepSeekV4LayerWeights *weights,
* coli_v4_dense_device_decide: asked by the RAM plan (EXPERT_STORE_AUTO) once it has
* measured the memory, with the bytes the device would hold and the RAM it would give
* back; 1 = the dense layers live on the device, their droppable tensors out of RAM.
+ * It sets dense_resident.device_layers: only those first layers go (a partial chain
+ * takes a prefix of the layers), and the plan counts only their bytes out of RAM.
* coli_v4_dense_place: after a resident layer is read, its tensors to the device and
* the droppable ones' pages given back (coli_v4_layer_host_drop).
* coli_v4_dense_device_lost: 1 once the device is gone; resident layers are then read
@@ -1035,6 +1037,11 @@ struct ColiV4Engine {
* (the plan left them out of RAM); what the device holds and what RAM gave back */
int device_only;
uint64_t device_bytes, dropped_bytes;
+ /* the layers whose droppable tensors the device holds alone: the first
+ * device_layers (a partial chain, docs/vulkan.md "A partial chain"); the others
+ * keep their host copies. Set by coli_v4_dense_device_decide, lowered when a
+ * layer does not reach the device. */
+ int device_layers;
} dense_resident;
/* Optional CUDA tier (compiled in only when the engine build defines
* COLI_V4_GPU_TIER on Windows). enabled is 1 only after the loader resolved