Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

deepseek_v4: the dense chain on the device (COLI_VK_CHAIN=1)

JustVuggea64d5ca4937
diff --git a/c/Makefile.deepseek-v4 b/c/Makefile.deepseek-v4
index f8b2823e..54529a14 100644
--- a/c/Makefile.deepseek-v4
+++ b/c/Makefile.deepseek-v4
@@ -240,6 +240,13 @@ V4_VK_SPV = shaders/qmatmul.spv shaders/qmatmul_gate_up.spv \
shaders/attention_absorb.spv shaders/rmsnorm.spv \
shaders/qmatmul_gemm.spv shaders/qmatmul_coop.spv \
shaders/expert_act.spv shaders/expert_act_v4.spv
+# the dense chain (vk_chain.c, deepseek_v4_chain.h: COLI_VK_CHAIN) and its shaders
+V4_VK_OBJ += vk_chain_v4.o
+V4_VK_SPV += shaders/chain_norm.spv shaders/chain_rope.spv shaders/chain_attn.spv \
+ shaders/chain_dnconv.spv shaders/chain_dnrec.spv shaders/chain_ew.spv \
+ shaders/chain_qsa.spv shaders/chain_ple.spv shaders/chain_gemv.spv \
+ shaders/chain_mla.spv shaders/chain_hgemv.spv shaders/chain_dsa.spv \
+ shaders/chain_kda.spv shaders/chain_mhc.spv shaders/chain_dsv4.spv
endif
include Makefile.deepseek-v4.units
@@ -315,6 +322,8 @@ backend_vulkan_v4.o: backend_vulkan.c backend_vulkan.h $(V4_FLAGS_STAMP)
$(CC) $(CFLAGS) -c backend_vulkan.c -o $@
vk_tier_v4.o: vk_tier.c vk_tier.h backend_vulkan.h tier.h $(V4_FLAGS_STAMP)
$(CC) $(CFLAGS) -c vk_tier.c -o $@
+vk_chain_v4.o: vk_chain.c vk_chain.h vk_alloc.h backend_vulkan.h $(V4_FLAGS_STAMP)
+ $(CC) $(CFLAGS) -c vk_chain.c -o $@
shaders/%.spv: shaders/%.comp
@command -v $(GLSLC) >/dev/null 2>&1 || { echo "glslc not found: install shaderc (for VK=1)" >&2; exit 1; }
$(GLSLC) --target-env=vulkan1.2 $< -o $@
@@ -332,7 +341,7 @@ $(REGISTRY_OBJ): expert_store_registry.c expert_store_registry.h expert_store.h
$(TARGET_OBJS) $(TEST_UNIT_OBJS): %.o: deepseek_v4.c deepseek_v4.h \
deepseek_v4_internal.h deepseek_v4_dspark.inc st.h json.h compat.h tensor.h quant.h fp8_format.h sse41_kernels.h \
- route_trace.h vk_tier.h \
+ route_trace.h vk_tier.h deepseek_v4_chain.h vk_chain.h \
native_quant.h native_quant_batch.h native_quant_dual.h \
native_quant_fp4_rows16.h expert_store_registry.h $(V4_FLAGS_STAMP)
$(CC) $(CFLAGS) -D$* -c deepseek_v4.c -o $@
@@ -384,4 +393,4 @@ deepseek-v4-clean:
rm -f $(V4_BINARY) $(V4_OBJS) $(TEST_UNIT_OBJS) $(V4_HOT_TEST_OBJ) \
$(V4_BATCH_TEST_OBJ) \
$(V4_SERVE_TEST) test_expert_store_registry $(V4_FLAGS_STAMP) \
- $(V4_CUDA_FLAGS_STAMP) backend_vulkan_v4.o vk_tier_v4.o
+ $(V4_CUDA_FLAGS_STAMP) backend_vulkan_v4.o vk_tier_v4.o vk_chain_v4.o
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.c
index 308611e9..7f3b38df 100644
--- a/c/deepseek_v4.c
+++ b/c/deepseek_v4.c
@@ -2210,6 +2210,36 @@ int coli_v4_attention_window_token_ref(
return attention_token_impl(output, state, weights, config, input, position,
error, error_size);
}
+
+#ifdef COLI_VULKAN
+/* The dense chain's view of this state (deepseek_v4_internal.h). */
+int coli_v4_attention_view(ColiDeepSeekV4WindowAttentionState *state,
+ ColiV4AttentionView *view) {
+ if (!state || !view) return -1;
+ *view = (ColiV4AttentionView){state->kv, state->compressed, state->window_size,
+ state->head_dim, state->ratio, state->compressed_count,
+ state->compressed_capacity, state->compressor,
+ state->indexer};
+ return 0;
+}
+int coli_v4_attention_reserve(ColiDeepSeekV4WindowAttentionState *state, int rows) {
+ if (!state || rows < 0 || (rows && !state->compressed)) return -1;
+ if (rows <= state->compressed_capacity) return 0;
+ int capacity = state->compressed_capacity > 0 ? state->compressed_capacity : 16;
+ while (capacity < rows) capacity *= 2;
+ float *grown = realloc(state->compressed,
+ (size_t)capacity * state->head_dim * sizeof(*grown));
+ if (!grown) return -1;
+ state->compressed = grown;
+ state->compressed_capacity = capacity;
+ return 0;
+}
+int coli_v4_attention_set_count(ColiDeepSeekV4WindowAttentionState *state, int count) {
+ if (!state || count < 0 || count > state->compressed_capacity) return -1;
+ state->compressed_count = count;
+ return 0;
+}
+#endif
#endif /* COLI_V4_UNIT_ATTENTION */
#ifdef COLI_V4_UNIT_ATTENTION_BATCH
@@ -3482,6 +3512,18 @@ static int compressor_pool_and_emit(ColiDeepSeekV4CompressorState *state,
*produced = 1;
return 0;
}
+
+#ifdef COLI_VULKAN
+/* The dense chain's view of the ring (deepseek_v4_internal.h). */
+int coli_v4_compressor_view(ColiDeepSeekV4CompressorState *state,
+ ColiV4CompressorView *view) {
+ if (!state || !view) return -1;
+ *view = (ColiV4CompressorView){state->kv_state, state->score_state, state->state_rows,
+ state->projection_dim, state->ratio, state->head_dim,
+ state->rotate_fp4};
+ return 0;
+}
+#endif
#endif /* COLI_V4_UNIT_COMPRESSOR */
#ifdef COLI_V4_UNIT_INDEXER
@@ -4186,6 +4228,34 @@ const float *coli_v4_indexer_compressed_values(
int coli_v4_indexer_compressed_count(const ColiDeepSeekV4Indexer *state) {
return state ? state->count : 0;
}
+
+#ifdef COLI_VULKAN
+/* The dense chain's view of the keys (deepseek_v4_internal.h). */
+int coli_v4_indexer_view(ColiDeepSeekV4Indexer *state, ColiV4IndexerView *view) {
+ if (!state || !view) return -1;
+ *view = (ColiV4IndexerView){state->compressed, state->count, state->capacity,
+ state->config->index_head_dim, state->compressor};
+ return 0;
+}
+int coli_v4_indexer_reserve(ColiDeepSeekV4Indexer *state, int rows) {
+ if (!state || rows < 0) return -1;
+ if (rows <= state->capacity) return 0;
+ int dimension = state->config->index_head_dim, capacity = state->capacity > 0 ? state->capacity : 16;
+ while (capacity < rows) capacity *= 2;
+ float *grown = realloc(state->compressed, (size_t)capacity * dimension * sizeof(*grown));
+ if (!grown) return -1;
+ memset(grown + (size_t)state->capacity * dimension, 0,
+ (size_t)(capacity - state->capacity) * dimension * sizeof(*grown));
+ state->compressed = grown;
+ state->capacity = capacity;
+ return 0;
+}
+int coli_v4_indexer_set_count(ColiDeepSeekV4Indexer *state, int count) {
+ if (!state || count < 0 || count > state->capacity) return -1;
+ state->count = count;
+ return 0;
+}
+#endif
#endif /* COLI_V4_UNIT_INDEXER */
#ifdef COLI_V4_UNIT_SPARSE_ATTENTION
@@ -4280,6 +4350,15 @@ static int moe_fail(const char *format, ...) {
va_end(arguments);
return -1;
}
+#ifdef COLI_VULKAN
+/* The dense chain (deepseek_v4_chain.h) runs the shared expert on the device and adds
+ * it there: while coli_v4_moe_routed runs, the MoE below leaves the shared expert out
+ * and returns the routed sum as it stands before that add (no rounding). */
+static __thread int v4_moe_routed_only;
+#define V4_SHARED_HERE (!v4_moe_routed_only)
+#else
+#define V4_SHARED_HERE 1
+#endif
/* ######## deepseek_v4_block_hybrid.c ######## */
/* Accepted decode pipeline plus batched causal attention for prompt prefill. */
/* ---- begin include deepseek_v4_block_pipeline.c ---- */
@@ -5065,7 +5144,7 @@ static int v4_vk_token_finish(V4VkToken *vk, float *output, ColiExpertStore *sto
for (int i = 0; !result && i < d; i++) output[i] += redo[i];
}
}
- if (!result)
+ if (!result && V4_SHARED_HERE)
for (int i = 0; i < d; i++) output[i] = coli_bf16_round(output[i] + shared_output[i]);
free(redo); free(dev); free(rows);
return result;
@@ -5236,7 +5315,7 @@ static int moe_token_pipeline(float *output,
fp8_view(&w3, weights, "ffn.shared_experts.w3")))
result = moe_fail("layer %d: the shared expert's fp8 tensors are missing",
weights->plan.layer);
- if (!result) result = coli_v4_shared_expert_forward_ref(
+ if (!result && V4_SHARED_HERE) result = coli_v4_shared_expert_forward_ref(
shared_output, &w1, &w2, &w3, input, config->swiglu_limit);
if (!result) memset(output, 0, (size_t)d * sizeof(*output));
@@ -5535,7 +5614,7 @@ static int moe_token_pipeline(float *output,
if (!result)
for (int i = 0; i < d; i++) output[i] += expert_output[i];
}
- if (!result)
+ if (!result && V4_SHARED_HERE)
for (int i = 0; i < d; i++)
output[i] = coli_bf16_round(output[i] + shared_output[i]);
#if defined(COLI_VULKAN) && defined(COLI_V4_EXPERIMENTAL_DUAL_EXPERT_LOADER)
@@ -6095,7 +6174,9 @@ static int v4_moe_batch_union(
fp8_view(&w3, weights, "ffn.shared_experts.w3")))
result = moe_fail("layer %d: the shared expert's fp8 tensors are missing",
weights->plan.layer);
- if (!result && batch > 1 && v4_shared_batch_enabled())
+ if (!V4_SHARED_HERE) {
+ /* the dense chain adds the shared expert itself */
+ } else if (!result && batch > 1 && v4_shared_batch_enabled())
result = v4_shared_expert_forward_batch_ref(
shared, &w1, &w2, &w3, inputs, batch, config->swiglu_limit);
else
@@ -6196,7 +6277,7 @@ static int v4_moe_batch_union(
coli_expert_release(store, &view);
}
#endif
- for (int item = 0; !result && item < batch; item++)
+ for (int item = 0; !result && V4_SHARED_HERE && item < batch; item++)
for (int column = 0; column < d; column++)
outputs[(size_t)item * d + column] = coli_bf16_round(
outputs[(size_t)item * d + column] +
@@ -6234,6 +6315,31 @@ static double v4_block_prof_now(void) {
return ts.tv_sec + ts.tv_nsec / 1e9;
}
+#ifdef COLI_VULKAN
+/* The dense chain's MoE step (deepseek_v4_internal.h): the routes of coli_v4_block_window_
+ * batch_ref and of the decode block (one row), without the shared expert. */
+int coli_v4_moe_routed(float *routed, const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, ColiExpertStore *store,
+ const float *inputs, const int *tokens, int rows,
+ char *error, size_t error_size) {
+ if (!routed || !weights || !config || !store || !inputs || !tokens || rows < 1 || rows > 128)
+ return set_error(error, error_size, "invalid chained MoE arguments");
+ int d = config->hidden_size, result = 0;
+ v4_moe_routed_only = 1;
+ if (rows > 1 && v4_expert_union_enabled())
+ result = v4_moe_batch_union(routed, weights, config, store, inputs, tokens, rows);
+ else
+ for (int item = 0; !result && item < rows; item++)
+ result = moe_token_pipeline(routed + (size_t)item * d, weights, config, store,
+ inputs + (size_t)item * d, tokens[item]);
+ v4_moe_routed_only = 0;
+ if (!result) return 0;
+ if (moe_reason()[0])
+ return set_error(error, error_size, "chained block failed in MoE: %s", moe_reason());
+ return set_error(error, error_size, "chained block failed in MoE");
+}
+#endif
+
int coli_v4_block_window_batch_ref(
float *outputs_hc, ColiDeepSeekV4WindowAttentionState *attention,
const ColiDeepSeekV4LayerWeights *weights,
@@ -12628,6 +12734,13 @@ static void v4_vk_tier_report(const char *scope, unsigned long long hits,
}
#endif
+#ifdef COLI_VULKAN
+/* the dense chain (deepseek_v4_chain.h, included below) */
+static void v4c_start(const ColiV4Engine *engine);
+static void v4c_atexit(void);
+static void v4c_close(void);
+#endif
+
static void v4_vk_open(const ColiV4Engine *engine) {
#ifdef COLI_VULKAN
g_v4_vk_thread = pthread_self();
@@ -12637,7 +12750,9 @@ static void v4_vk_open(const ColiV4Engine *engine) {
int tier = vkt_wanted() && engine->experts && !engine->experts->gpu &&
engine->config.n_routed_experts > 0;
g_v4_vk_ready = coli_vk_init_env_tier("deepseek_v4", tier);
+ v4c_start(engine); /* COLI_VK_CHAIN: the trunk on the device, before the tier sizes itself */
v4_vk_tier_start(engine);
+ v4c_atexit(); /* after the tier's: the chain goes before the device */
if (g_v4_vk_ready && !vkt_ready() && !coli_vk_dense())
coli_vk_dense_decide("deepseek_v4", 0, 1); /* no tier after all: the trunk to the device */
if (g_v4_vk_ready && coli_vk_dense()) coli_v4_vk_matmul = v4_vk_matmul_impl; /* COLI_VK_DENSE=0: no hook */
@@ -12651,6 +12766,7 @@ static void v4_vk_open(const ColiV4Engine *engine) {
static void v4_vk_close(void) {
#ifdef COLI_VULKAN
if (!g_v4_vk_ready) return;
+ v4c_close(); /* the chain's line, its frames through before the matrices go */
if (coli_v4_vk_tier) { /* the tier's run line, then nothing reaches it any more */
ColiExpertStoreStats st = {0};
ColiExpertStore *store = coli_v4_vk_tier->store;
@@ -12733,6 +12849,18 @@ static int head_argmax(ColiV4Engine *engine, const float *hidden,
* logprobs=k, docs/brio.md) needs the whole row, so the row is computed here
* once and the argmax is a scan over it. Same head_bf16_dot per row, same scan
* order: the token picked and its logit do not change. */
+#ifdef COLI_VULKAN
+/* DUMP=<path> (VK=1 builds): every logits row the head computes, for the Vulkan gates */
+static void v4_dump_logits(const float *scores, size_t count) {
+ static FILE *dump;
+ static int dump_init;
+ if (!dump_init) { const char *d = getenv("DUMP"); dump_init = 1; if (d && *d) dump = fopen(d, "wb"); }
+ if (dump) { fwrite(scores, sizeof(float), count, dump); fflush(dump); }
+}
+#define V4_DUMP_LOGITS(scores, count) v4_dump_logits((scores), (count))
+#else
+#define V4_DUMP_LOGITS(scores, count) ((void)0)
+#endif
static int head_scores_impl(ColiV4Engine *engine, const float *hidden,
const ColiSafetensorsIndex *index,
const ColiDeepSeekV4Config *config, float *scores) {
@@ -12752,14 +12880,17 @@ static int head_scores_impl(ColiV4Engine *engine, const float *hidden,
if (resident) {
#ifdef COLI_VULKAN
if (coli_v4_vk_matmul &&
- coli_v4_vk_matmul(11, resident, NULL, 0, vocab, d, scores, hidden, 1) == 0)
+ coli_v4_vk_matmul(11, resident, NULL, 0, vocab, d, scores, hidden, 1) == 0) {
+ V4_DUMP_LOGITS(scores, (size_t)vocab);
return 0;
+ }
#endif
#pragma omp parallel for schedule(static)
for (int row = 0; row < vocab; row++) {
const uint16_t *weight = resident + (size_t)row * d;
scores[row] = head_bf16_dot(weight, hidden, d);
}
+ V4_DUMP_LOGITS(scores, (size_t)vocab);
return 0;
}
/* Low-memory fallback: stream small row tiles exactly as before. */
@@ -12783,6 +12914,7 @@ static int head_scores_impl(ColiV4Engine *engine, const float *hidden,
}
}
free(raw);
+ V4_DUMP_LOGITS(scores, (size_t)vocab);
return 0;
}
/* First maximum in vocabulary order: the tie-break head_argmax always had. */
@@ -12859,6 +12991,7 @@ static int head_argmax_batch(ColiV4Engine *engine, const float *hidden,
scores[(size_t)item * vocab + row] = head_bf16_dot(
weight, hidden + (size_t)item * d, d);
}
+ V4_DUMP_LOGITS(scores, (size_t)batch * vocab);
for (int item = 0; item < batch; item++) {
int winner = -1;
float maximum = -FLT_MAX;
@@ -13153,6 +13286,10 @@ static const float *v4_mainh_get(int64_t position) {
#include "deepseek_v4_dspark.inc"
+#ifdef COLI_VULKAN
+#include "deepseek_v4_chain.h" /* the layers as a dense chain on the device (COLI_VK_CHAIN) */
+#endif
+
static int v4_prefill_pool_enabled(void) {
static int enabled = -1;
if (enabled < 0) {
@@ -13206,6 +13343,14 @@ static int target_batch_impl(ColiV4Engine *engine, float **state_ptr, float **ne
* does not change the caller's semantic distinction: speculative decode
* always passes use_prefill_pool=0. */
int pool_experts = use_prefill_pool && v4_prefill_pool_enabled();
+#ifdef COLI_VULKAN
+ { /* the dense chain runs every layer on the device; 0: the CPU runs them, as below */
+ int chained = v4c_forward(engine, state_ptr, next_ptr, attention, config, experts, tokens,
+ start, batch, pool_experts, error, error_size);
+ if (chained) return chained > 0 ? 0 : -1;
+ v4c_cpu_step(start);
+ }
+#endif
for (int layer_id = 0; layer_id < config->num_hidden_layers; layer_id++) {
ColiDeepSeekV4LayerWeights layer;
if (coli_v4_layer_load(engine, &layer, config, index, layer_id,
@@ -13303,6 +13448,14 @@ static int target_token_impl(ColiV4Engine *engine, float **state_ptr, float **ne
char *error, size_t error_size) {
float *state = *state_ptr, *next = *next_ptr;
if (load_embedding(state, index, config, token)) return -1;
+#ifdef COLI_VULKAN
+ { /* the dense chain, as in target_batch */
+ int chained = v4c_forward(engine, state_ptr, next_ptr, attention, config, experts, &token,
+ position, 1, 0, error, error_size);
+ if (chained) return chained > 0 ? 0 : -1;
+ v4c_cpu_step(position);
+ }
+#endif
for (int layer_id = 0; layer_id < config->num_hidden_layers; layer_id++) {
ColiDeepSeekV4LayerWeights layer;
if (coli_v4_layer_load(engine, &layer, config, index, layer_id,
@@ -14028,6 +14181,9 @@ static int v4_ckpt_restore(ColiV4Session *session, int prompt_count) {
fprintf(stderr, "v4_ckpt restore failed layer=%d\n", layer);
return 0; /* partial restore is harmless: caller full-resets */
}
+#ifdef COLI_VULKAN
+ v4c_reset(); /* a snapshot from another history: the chain's copies go */
+#endif
kv_prefix_clear(&session->fed);
kv_prefix_record(&session->fed, v4_ckpt_slots[best].ids, 0, best_len);
v4_ckpt_slots[best].used = ++v4_ckpt_clock;
@@ -14199,6 +14355,9 @@ int coli_v4_session_generate(ColiV4Session *session,
#endif
for (int layer = 0; layer < config->num_hidden_layers; layer++)
coli_v4_window_attention_reset(session->attention[layer]);
+#ifdef COLI_VULKAN
+ v4c_reset();
+#endif
if (coli_v4_full_dspark_wanted) v4_ds_reset_history();
/* Plan a capture: the longest common prefix of two successive fresh
* prompts IS the stable system prefix. Snapshot there mid-prefill. */
@@ -14521,6 +14680,9 @@ int coli_v4_session_generate(ColiV4Session *session,
batch, 0, NULL, NULL, error, error_size)) {
(void)spec_attention_restore(
attention, snapshots, config->num_hidden_layers);
+#ifdef COLI_VULKAN
+ v4c_restored(old_last + 1);
+#endif
spec_attention_free(snapshots,
config->num_hidden_layers);
kv_prefix_taint(&session->fed);
@@ -14542,6 +14704,9 @@ int coli_v4_session_generate(ColiV4Session *session,
if (!heads_ok) {
(void)spec_attention_restore(
attention, snapshots, config->num_hidden_layers);
+#ifdef COLI_VULKAN
+ v4c_restored(old_last + 1);
+#endif
spec_attention_free(snapshots,
config->num_hidden_layers);
kv_prefix_taint(&session->fed);
@@ -14595,6 +14760,9 @@ int coli_v4_session_generate(ColiV4Session *session,
* Restore the exact snapshot and replay only inputs that
* really correspond to emitted outputs. */
if (retained < batch) {
+#ifdef COLI_VULKAN
+ v4c_restored(old_last + 1); /* the rows past it rejected */
+#endif
if (spec_attention_restore(
attention, snapshots,
config->num_hidden_layers)) {
@@ -15378,6 +15546,7 @@ static int v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,
#endif
#ifdef COLI_VULKAN
v4_vk_tier_report("turn", (unsigned long long)hits, (unsigned long long)misses);
+ v4c_report(); /* the dense chain's line, when it ran */
#endif
coli_v4_expert_store_emit_hits(engine->experts);
coli_v4_expert_store_emit_emap(engine->experts);
@@ -15778,6 +15947,9 @@ int main(int argc, char **argv) {
(size_t)generated_count * sizeof(int));
for (int layer = 0; layer < config.num_hidden_layers; layer++)
coli_v4_window_attention_reset(attention[layer]);
+#ifdef COLI_VULKAN
+ v4c_reset();
+#endif
/* Rebuild tf_pred for the fixture (argmax at each position). */
size_t hd_tf = (size_t)config.hc_mult * config.hidden_size;
tf_state = malloc((size_t)full_count * hd_tf * sizeof(float));
diff --git a/c/deepseek_v4_chain.h b/c/deepseek_v4_chain.h
new file mode 100644
index 00000000..e59e0ced
--- /dev/null
+++ b/c/deepseek_v4_chain.h
@@ -0,0 +1,872 @@
+/* deepseek_v4_chain.h -- DeepSeek V4's layers as a dense chain on the Vulkan device
+ * (vk_chain.h). Included once by deepseek_v4.c's GENERATE_STATS unit in a COLI_VULKAN
+ * build, after the per-matrix path (v4_vk_*): the chain's matrices are that path's
+ * device copies, looked up in the same map, one copy per matrix. target_batch and
+ * target_token hand it their forwards (v4c_forward); COLI_VK_CHAIN decides
+ * (coli_vk_chain_decide, this engine's integrated GPU default COLI_VK_CHAIN_UNMEASURED:
+ * off, not measured on a V4 checkpoint).
+ *
+ * The residual is hc_mult streams per position (mHC); each site collapses them with the
+ * mix it computes itself (hc_pre). DeepSeek V4 rounds to bf16 after nearly every step
+ * and rounds the input of every fp8 matrix to E4M3 per 128 (the CPU kernel's activation):
+ * the chain makes each of those roundings where the CPU makes it (vkc_dsv4_round), so
+ * the device and the CPU differ only by the order of the sums and the device's exp and
+ * sqrt, which the roundings mostly absorb (a value that lands on a bf16 boundary can go
+ * the other way, and an E4M3 block then rounds on another step: docs/vulkan.md has what
+ * that does to the logits). Every matrix takes the per-row GEMV, so a row's bits do not
+ * depend on how a forward is cut into chunks, nor on prompt against decode against a
+ * verify, as on the CPU. What runs where, per layer, for one block of rows (decode: one
+ * row; a prompt in chunks of the CPU's prefill chunk):
+ * device, frame A1: the previous layer's FFN branch (the routed sum from the host plus
+ * the shared expert, to bf16) written back into the streams (mHC post, bf16); the
+ * attention site: hc_attn_fn, the split with Sinkhorn, the collapse, bf16, the
+ * RMSNorm, bf16; the attention: the input to E4M3, wq_a, bf16, q_norm, bf16; on a
+ * compressed layer the compressor (wkv and wgate in bf16 on the f32 input, the ring
+ * with its position bias, the pooled rows to bf16, their norm, bf16, RoPE at the
+ * group's first position, the no-position part to E4M3 per 64); on a ratio-4 layer
+ * the indexer (its own compressor into the keys: norm, RoPE, the Hadamard transform,
+ * E2M1 per 32; the queries: indexer wq_b on the q latent, bf16, RoPE, Hadamard,
+ * E2M1; weights_proj; the relu-weighted scores and the top-k in score order); wq_b,
+ * bf16, the per-head RMS without weight, bf16; wkv, bf16, kv_norm, bf16; RoPE (YaRN's
+ * table on compressed layers) with bf16 on the rotated part; the key's no-position
+ * part to E4M3 per 64, bf16; the new rows into the window ring; the sparse attention
+ * with the sink over the window and the compressed rows (V4's bf16 weights and
+ * output), the inverse RoPE, bf16; wo_a per group (each group's input to E4M3),
+ * bf16, wo_b, bf16; mHC post, bf16; the FFN site's split, collapse, norm. Then the
+ * frame is waited for.
+ * host: the routed experts (coli_v4_moe_routed: the bf16 or the hash router, the
+ * expert tier's batch and the CPU's share, summed in the CPU block's order).
+ * device, frame A2 (not waited for): the shared expert (the input to E4M3, w1 and w3,
+ * V4's SwiGLU between bf16 roundings, the input to E4M3, w2, bf16).
+ * The final streams come back to the host, which runs final_hidden and the head as
+ * before; DSpark's taps of the last three layers' streams too (v4_mainh_tap).
+ *
+ * State, and who owns it: the host's stays canonical. What a forward changes there (the
+ * window ring, the compressed rows and their count, each compressor's ring, the
+ * indexer's keys and count and its compressor's ring) is written back once the
+ * forward's last frame is through, as the CPU would have left it, so a lost device, a
+ * rejected draft (spec_attention_restore), a prefix checkpoint, a reset or a CPU forward
+ * never has to rebuild anything. The device mirrors it: the window ring holds window +
+ * chunk rows (no row of a chunk overwrites one an earlier row still reads) behind a
+ * watermark win_valid, a row below it valid unless a later position took its slot (a
+ * rejected draft longer than a chunk does: slot_pos says); the compressed rows and the
+ * indexer's keys of each layer behind kv_valid / ik_valid; each ring behind a flag.
+ * Every host write lowers them: a CPU
+ * forward (v4c_cpu_step), a restored snapshot (v4c_restored), a reset or a checkpoint
+ * restore (v4c_reset), a forward that does not continue where the chain left off, an
+ * attention state the mirrors do not describe. A lost device (COLI_VK_CHAIN_FAULT=n
+ * fakes it): the forward runs again on the CPU from its input, which the chain leaves
+ * untouched until every chunk is through, and the CPU runs from there.
+ *
+ * The chain declines (the CPU runs, the watermarks follow) when the dense layers are not
+ * resident (a low-memory plan reloads them per forward; the --oracle path's own copies
+ * too), under the CUDA tier, for prompts only when COLI_VK_CHAIN=2 and the forward has
+ * two rows or fewer, for a forward whose compressed rows (window plus every row of a
+ * layer without an indexer) pass the attention's 3072 entries, and for a model its
+ * shaders do not take: head_dim above 1024, an indexer head that is not a power of two
+ * (the Hadamard transform) or above 4096 floats, a top-k above 4096, more than 8
+ * streams. */
+#include "vk_chain.h"
+
+typedef struct {
+ int ratio, idx, kind; /* compress ratio (0 none), an indexer, its list */
+ int crows, cproj; /* the compressor's ring: rows and floats a row */
+ size_t o_an, o_fn, o_qn, o_kn, o_sink, o_hca, o_hcf, o_cn, o_ape, o_icn, o_iape;
+ ColiVkTensor *fna, *fnf, *wq_a, *wq_b, *wkv, *wo_b, *sh1, *sh2, *sh3, **wo_a;
+ ColiVkTensor *cwkv, *cwg, *iwkv, *iwg, *iwq, *iwp;
+ VkcBuf *win, *ckv, *ring, *ikey, *iring;
+ int ccap, kv_valid, ring_ok, icap, ik_valid, iring_ok;
+ size_t pw, pc, pr, pk, pir; /* this forward's places in the pull buffer */
+ int ns, q1; /* the ring rows it pulls: all (the overlap), or the slots of q1.. */
+} V4cLayer;
+
+typedef struct {
+ int ok, failed, told;
+ ColiV4Engine *engine;
+ int L, D, H, HD, nm, hr, nh, hd, rd, QL, og, ol, gw, W, Wd, IH, ID, I, K, rmax, rows, nkind;
+ int kratio[COLI_V4_MAX_LAYERS + 1]; /* the ratio of each list kind (0: the indexer's) */
+ V4cLayer *ly;
+ VkcBuf *prm;
+ ColiDeepSeekV4WindowAttentionState **attn; /* the states the mirrors describe */
+ int E, win_valid; /* the position after the last chained forward */
+ int *slot_pos; /* the position each device ring row holds (-1 none) */
+ size_t wcap; /* score columns a row */
+ VkcBuf *xs, *xn, *mix, *hpa, *hpf, *col, *nrm, *nq, *qa, *qr, *qrq, *q, *kv, *heads, *grp, *br;
+ VkcBuf *craw, *cscr, *icraw, *icscr, *iq, *ihw, *isc, *list, *cs, *sq, *gs, *us, *hs, *ds;
+ VkcBuf *h2d, *routed, *xd, *taps, *pull;
+ float *host_routed;
+ unsigned long long forwards;
+ double host_ms;
+} V4Chain;
+
+static V4Chain *g_v4c;
+static int g_v4c_mode = 0; /* COLI_VK_CHAIN as decided */
+static int g_v4c_inited = 0;
+
+/* Rows per chunk: the CPU's prefill chunk (V4_PREFILL_CHUNK, 128 at most), so the host's
+ * MoE sees the batches the CPU block would; COLI_VK_CHAIN_ROWS lowers it. */
+static int v4c_rows(void) {
+ static int rows;
+ if (!rows) {
+ const char *c = getenv("V4_PREFILL_CHUNK"), *e = getenv("COLI_VK_CHAIN_ROWS");
+ int cw = c ? atoi(c) : 128;
+ if (cw < 1 || cw > 128) cw = 128;
+ int v = e && *e ? atoi(e) : cw;
+ rows = v < 1 ? 1 : v > cw ? cw : v;
+ }
+ return rows;
+}
+static int v4c_res(VkcBuf **b, size_t floats, int kind) { return vkc_reserve(b, (floats ? floats : 1) * sizeof(float), kind); }
+static const void *v4c_val(const ColiDeepSeekV4LayerWeights *w, const char *suffix) {
+ char name[COLI_V4_MAX_TENSOR_NAME];
+ snprintf(name, sizeof name, "layers.%d.%s", w->plan.layer, suffix);
+ return coli_v4_layer_data(w, name, NULL);
+}
+
+/* A resident matrix's device copy, the per-matrix path's own (v4_vk_matmul_impl's map
+ * and layout): fp8 as fmt 12 (rows unpacked from the rows8 tiles, each 128x128 block
+ * scale written out for its rows), bf16 as fmt 11, f32 as fmt 10 (the mHC mixes, which
+ * only the chain multiplies). */
+static ColiVkTensor *v4c_tensor(int fmt, const void *data, const float *scales, int rows8, int rows, int columns) {
+ if (!g_v4_vk_ready || !data || rows < 1 || columns < 1) return NULL;
+ if (fmt == 12 && (!scales || columns % 128 || (rows8 && rows % 8))) return NULL;
+ V4VkEntry *e = v4_vk_find(data, fmt, rows, columns);
+ if (e && e->tensor) return e->tensor;
+ if (e && e->refused) return NULL;
+ size_t bytes = (size_t)rows * columns * (fmt == 10 ? 4 : fmt == 11 ? 2 : 1);
+ if (!v4_vk_resident(data, bytes)) return NULL;
+ if (!e && !(e = v4_vk_insert(data, fmt, rows, columns))) return NULL;
+ const unsigned char *weights = data;
+ unsigned char *unpacked = NULL;
+ float *expanded = NULL;
+ if (fmt == 12) {
+ int groups = columns / 128;
+ expanded = malloc((size_t)rows * groups * sizeof(float));
+ if (rows8) unpacked = malloc((size_t)rows * columns);
+ if (!expanded || (rows8 && !unpacked)) { free(expanded); free(unpacked); return NULL; }
+ for (int row = 0; row < rows; row++)
+ memcpy(expanded + (size_t)row * groups, scales + (size_t)(row / 128) * groups, (size_t)groups * sizeof(float));
+ if (rows8) {
+ const unsigned char *packed = data;
+ for (int tile = 0; tile < rows / 8; tile++)
+ for (int column = 0; column < columns; column++)
+ for (int lane = 0; lane < 8; lane++)
+ unpacked[((size_t)tile * 8 + lane) * columns + column] = packed[((size_t)tile * columns + column) * 8 + lane];
+ weights = unpacked;
+ }
+ }
+ int ok = coli_vk_tensor_ensure(&e->tensor, weights, expanded, fmt, columns, rows, fmt == 12 ? 128 : 0);
+ free(unpacked); free(expanded);
+ if (!ok && !e->tensor) e->refused = 1;
+ return ok ? e->tensor : NULL;
+}
+/* an fp8 matrix "<prefix>.weight" with its ".scale", a group of rows of it when rows > 0 */
+static ColiVkTensor *v4c_fp8(const ColiDeepSeekV4LayerWeights *w, const char *prefix, int group, int rows) {
+ char name[COLI_V4_MAX_TENSOR_NAME], sname[COLI_V4_MAX_TENSOR_NAME];
+ const ColiDeepSeekV4TensorSpec *ws = NULL;
+ snprintf(name, sizeof name, "layers.%d.%s.weight", w->plan.layer, prefix);
+ snprintf(sname, sizeof sname, "layers.%d.%s.scale", w->plan.layer, prefix);
+ const unsigned char *data = coli_v4_layer_data(w, name, &ws);
+ const float *scales = coli_v4_layer_data(w, sname, NULL);
+ if (!data || !scales || !ws || ws->rank != 2 || ws->dtype != COLI_ST_F8_E4M3) return NULL;
+ int O = (int)ws->shape[0], I = (int)ws->shape[1];
+ if (rows <= 0) return v4c_tensor(12, data, scales, ws->packed_rows8, O, I);
+ /* the CPU's group view (attention_token_impl): rows group*rows.., its own scale rows */
+ int scale_columns = (I + 127) / 128, scale_rows = (rows + 127) / 128;
+ return v4c_tensor(12, data + (size_t)group * rows * I, scales + (size_t)group * scale_rows * scale_columns,
+ ws->packed_rows8, rows, I);
+}
+static ColiVkTensor *v4c_plain(const ColiDeepSeekV4LayerWeights *w, const char *suffix, int fmt) {
+ char name[COLI_V4_MAX_TENSOR_NAME];
+ const ColiDeepSeekV4TensorSpec *ws = NULL;
+ snprintf(name, sizeof name, "layers.%d.%s", w->plan.layer, suffix);
+ const void *data = coli_v4_layer_data(w, name, &ws);
+ if (!data || !ws || ws->rank != 2) return NULL;
+ return v4c_tensor(fmt, data, NULL, 0, (int)ws->shape[0], (int)ws->shape[1]);
+}
+
+/* ---- the host's writes: the device's copies follow ------------------------------------- */
+static void v4c_lower(int pos) {
+ V4Chain *ch = g_v4c;
+ if (!ch || !ch->ok) return;
+ if (pos < 0) pos = 0;
+ if (ch->win_valid > pos) ch->win_valid = pos;
+ for (int i = 0; i < ch->L; i++) {
+ V4cLayer *ly = &ch->ly[i];
+ if (ly->ratio > 0 && ly->kv_valid > pos / ly->ratio) ly->kv_valid = pos / ly->ratio;
+ if (ly->idx && ly->ik_valid > pos / 4) ly->ik_valid = pos / 4;
+ ly->ring_ok = ly->iring_ok = 0;
+ }
+ ch->E = -1;
+}
+static void v4c_cpu_step(int start) { v4c_lower(start); } /* a CPU forward from start */
+static void v4c_restored(int pos) { v4c_lower(pos); } /* the state restored to pos, the rows before it kept */
+static void v4c_reset(void) { v4c_lower(0); } /* a reset, or a state from elsewhere */
+
+/* ---- setup ----------------------------------------------------------------------------- */
+static const char *v4c_unsupported(const ColiDeepSeekV4Config *c) {
+ int any_idx = 0;
+ for (int i = 0; i < c->num_hidden_layers; i++) any_idx |= c->compress_ratios[i] == 4;
+ if (c->head_dim > 1024 || c->head_dim < 2 || c->qk_rope_head_dim < 2 || (c->qk_rope_head_dim & 1) ||
+ c->qk_rope_head_dim > c->head_dim) return "an attention geometry its shaders do not take";
+ if (c->hc_mult < 1 || c->hc_mult > 8 || c->hc_sinkhorn_iters < 1) return "hyper-connections its shaders do not take";
+ if (c->o_groups < 1 || c->num_attention_heads % c->o_groups) return "an output grouping it does not take";
+ if (c->sliding_window < 1 || c->sliding_window > 3072) return "a window its attention does not take";
+ if (any_idx && (c->index_n_heads < 1 || c->index_n_heads > 4096 || c->index_head_dim > 4096 ||
+ c->index_head_dim < c->qk_rope_head_dim || (c->index_head_dim & (c->index_head_dim - 1)) ||
+ c->index_topk < 1 || c->index_topk > 4096 || c->sliding_window + c->index_topk > 3072))
+ return "an indexer its shaders do not take";
+ for (int i = 0; i < c->num_hidden_layers; i++)
+ if (c->compress_ratios[i] < 0) return "a compression ratio it does not take";
+ return NULL;
+}
+
+static int v4c_setup(ColiV4Engine *engine) {
+ const ColiDeepSeekV4Config *c = &engine->config;
+ const char *why = v4c_unsupported(c);
+ if (!why && !engine->runtime.dense_resident) why = "the dense layers are not resident (a low-memory plan reloads them per forward)";
+ if (!why && (engine->gpu.enabled || (engine->experts && engine->experts->gpu))) why = "the CUDA tier is on and keeps its place";
+ if (why) { fprintf(stderr, "[VK] deepseek_v4 chain: %s; the CPU runs the layers\n", why); return 0; }
+ int L = c->num_hidden_layers;
+ char err[256] = {0};
+ const ColiSafetensorsIndex *index = coli_v4_engine_target_index(engine);
+ for (int i = 0; i < L; i++) { /* the resident layers, loaded now */
+ ColiDeepSeekV4LayerWeights w;
+ if (coli_v4_layer_load(engine, &w, c, index, i, err, sizeof err) || !engine->dense_resident.ready[i]) {
+ fprintf(stderr, "[VK] deepseek_v4 chain: layer %d did not load (%s); the CPU runs the layers\n", i, err);
+ return 0;
+ }
+ }
+ V4Chain *ch = calloc(1, sizeof *ch);
+ if (!ch || !(ch->ly = calloc((size_t)L, sizeof *ch->ly))) { free(ch); return 0; }
+ g_v4c = ch;
+ ch->engine = engine; ch->L = L; ch->D = c->hidden_size; ch->H = c->hc_mult; ch->HD = ch->H * ch->D;
+ ch->nm = (2 + ch->H) * ch->H; ch->hr = 2 * ch->H + ch->H * ch->H;
+ ch->nh = c->num_attention_heads; ch->hd = c->head_dim; ch->rd = c->qk_rope_head_dim; ch->QL = c->q_lora_rank;
+ ch->og = c->o_groups; ch->ol = c->o_lora_rank; ch->gw = (ch->nh / ch->og) * ch->hd;
+ ch->W = c->sliding_window; ch->Wd = ch->W + v4c_rows();
+ if (!(ch->slot_pos = malloc((size_t)ch->Wd * sizeof(int)))) return 0;
+ for (int k = 0; k < ch->Wd; k++) ch->slot_pos[k] = -1;
+ ch->IH = c->index_n_heads > 0 ? c->index_n_heads : 1; ch->ID = c->index_head_dim > 0 ? c->index_head_dim : 2;
+ ch->I = c->moe_intermediate_size; ch->rmax = 1; ch->E = -1; ch->nkind = 1;
+ int D = ch->D, nm = ch->nm;
+ for (int i = 0; i < L; i++) {
+ V4cLayer *ly = &ch->ly[i];
+ ly->ratio = c->compress_ratios[i]; ly->idx = ly->ratio == 4;
+ if (ly->ratio > ch->rmax) ch->rmax = ly->ratio;
+ ly->crows = ly->ratio == 4 ? 8 : ly->ratio; ly->cproj = (ly->ratio == 4 ? 2 : 1) * ch->hd;
+ if (ly->ratio > 0 && !ly->idx) {
+ int k = 1;
+ while (k < ch->nkind && ch->kratio[k] != ly->ratio) k++;
+ if (k == ch->nkind) ch->kratio[ch->nkind++] = ly->ratio;
+ ly->kind = k;
+ }
+ }
+ /* the parameter arena: norms decoded from bf16, sinks, hc scales and bases, the rings' biases */
+ size_t n = 0;
+ for (int i = 0; i < L; i++) {
+ V4cLayer *ly = &ch->ly[i];
+ ly->o_an = n; n += D; ly->o_fn = n; n += D; ly->o_qn = n; n += ch->QL; ly->o_kn = n; n += ch->hd;
+ ly->o_sink = n; n += ch->nh; ly->o_hca = n; n += 3 + nm; ly->o_hcf = n; n += 3 + nm;
+ if (ly->ratio > 0) { ly->o_cn = n; n += ch->hd; ly->o_ape = n; n += (size_t)ly->ratio * ly->cproj; }
+ if (ly->idx) { ly->o_icn = n; n += ch->ID; ly->o_iape = n; n += (size_t)4 * 2 * ch->ID; }
+ }
+ float *a = calloc(n ? n : 1, sizeof(float));
+ int ok = a != NULL;
+ for (int i = 0; ok && i < L; i++) {
+ V4cLayer *ly = &ch->ly[i];
+ const ColiDeepSeekV4LayerWeights *w = &engine->dense_resident.layers[i];
+ const uint16_t *an = v4c_val(w, "attn_norm.weight"), *fn = v4c_val(w, "ffn_norm.weight"),
+ *qn = v4c_val(w, "attn.q_norm.weight"), *kn = v4c_val(w, "attn.kv_norm.weight");
+ const float *sink = v4c_val(w, "attn.attn_sink"), *hsa = v4c_val(w, "hc_attn_scale"), *hba = v4c_val(w, "hc_attn_base"),
+ *hsf = v4c_val(w, "hc_ffn_scale"), *hbf = v4c_val(w, "hc_ffn_base");
+ ok = an && fn && qn && kn && sink && hsa && hba && hsf && hbf;
+ for (int k = 0; ok && k < D; k++) { a[ly->o_an + k] = coli_bf16_decode(an[k]); a[ly->o_fn + k] = coli_bf16_decode(fn[k]); }
+ for (int k = 0; ok && k < ch->QL; k++) a[ly->o_qn + k] = coli_bf16_decode(qn[k]);
+ for (int k = 0; ok && k < ch->hd; k++) a[ly->o_kn + k] = coli_bf16_decode(kn[k]);
+ if (ok) {
+ memcpy(a + ly->o_sink, sink, (size_t)ch->nh * sizeof(float));
+ memcpy(a + ly->o_hca, hsa, 3 * sizeof(float)); memcpy(a + ly->o_hca + 3, hba, (size_t)nm * sizeof(float));
+ memcpy(a + ly->o_hcf, hsf, 3 * sizeof(float)); memcpy(a + ly->o_hcf + 3, hbf, (size_t)nm * sizeof(float));
+ }
+ if (ok && ly->ratio > 0) {
+ const uint16_t *cn = v4c_val(w, "attn.compressor.norm.weight");
+ const float *ape = v4c_val(w, "attn.compressor.ape");
+ ok = cn && ape;
+ for (int k = 0; ok && k < ch->hd; k++) a[ly->o_cn + k] = coli_bf16_decode(cn[k]);
+ if (ok) memcpy(a + ly->o_ape, ape, (size_t)ly->ratio * ly->cproj * sizeof(float));
+ }
+ if (ok && ly->idx) {
+ const uint16_t *icn = v4c_val(w, "attn.indexer.compressor.norm.weight");
+ const float *iape = v4c_val(w, "attn.indexer.compressor.ape");
+ ok = icn && iape;
+ for (int k = 0; ok && k < ch->ID; k++) a[ly->o_icn + k] = coli_bf16_decode(icn[k]);
+ if (ok) memcpy(a + ly->o_iape, iape, (size_t)8 * ch->ID * sizeof(float));
+ }
+ }
+ ok = ok && (ch->prm = vkc_buf((n ? n : 1) * sizeof(float), VKC_DEV)) && vkc_begin() &&
+ vkc_write(ch->prm, 0, a, n * sizeof(float)) && vkc_submit(1);
+ free(a);
+ /* the tensors and the per-layer state on the device */
+ for (int i = 0; ok && i < L; i++) {
+ V4cLayer *ly = &ch->ly[i];
+ const ColiDeepSeekV4LayerWeights *w = &engine->dense_resident.layers[i];
+ ly->wo_a = calloc((size_t)ch->og, sizeof(void *));
+ ok = ly->wo_a && (ly->fna = v4c_plain(w, "hc_attn_fn", 10)) && (ly->fnf = v4c_plain(w, "hc_ffn_fn", 10)) &&
+ (ly->wq_a = v4c_fp8(w, "attn.wq_a", 0, 0)) && (ly->wq_b = v4c_fp8(w, "attn.wq_b", 0, 0)) &&
+ (ly->wkv = v4c_fp8(w, "attn.wkv", 0, 0)) && (ly->wo_b = v4c_fp8(w, "attn.wo_b", 0, 0)) &&
+ (ly->sh1 = v4c_fp8(w, "ffn.shared_experts.w1", 0, 0)) && (ly->sh2 = v4c_fp8(w, "ffn.shared_experts.w2", 0, 0)) &&
+ (ly->sh3 = v4c_fp8(w, "ffn.shared_experts.w3", 0, 0));
+ for (int g = 0; ok && g < ch->og; g++) ok = (ly->wo_a[g] = v4c_fp8(w, "attn.wo_a", g, ch->ol)) != NULL;
+ if (ok && ly->ratio > 0)
+ ok = (ly->cwkv = v4c_plain(w, "attn.compressor.wkv.weight", 11)) && (ly->cwg = v4c_plain(w, "attn.compressor.wgate.weight", 11)) &&
+ (ly->ring = vkc_buf((size_t)2 * ly->crows * ly->cproj * sizeof(float), VKC_DEV));
+ if (ok && ly->idx)
+ ok = (ly->iwkv = v4c_plain(w, "attn.indexer.compressor.wkv.weight", 11)) &&
+ (ly->iwg = v4c_plain(w, "attn.indexer.compressor.wgate.weight", 11)) &&
+ (ly->iwp = v4c_plain(w, "attn.indexer.weights_proj.weight", 11)) && (ly->iwq = v4c_fp8(w, "attn.indexer.wq_b", 0, 0)) &&
+ (ly->iring = vkc_buf((size_t)2 * 8 * 2 * ch->ID * sizeof(float), VKC_DEV));
+ if (ok) ok = (ly->win = vkc_buf((size_t)ch->Wd * ch->hd * sizeof(float), VKC_DEV)) != NULL;
+ }
+ if (!ok) { fprintf(stderr, "[VK] deepseek_v4 chain: a matrix or the state did not reach the device; the CPU runs the layers\n"); return 0; }
+ ch->ok = 1;
+ int ncomp = 0, nidx = 0;
+ for (int i = 0; i < L; i++) { ncomp += ch->ly[i].ratio > 0; nidx += ch->ly[i].idx; }
+ size_t bytes = 0, tensors = 0;
+ coli_vk_mem_info(&bytes, &tensors);
+ fprintf(stderr, "[VK] deepseek_v4 chain: %d layers on the device (%d compressing, %d indexing), %d streams, "
+ "%.1f MiB of parameters, %zu matrices (%.1f MiB) on the device\n",
+ L, ncomp, nidx, ch->H, n * 4 / 1048576.0, tensors, bytes / 1048576.0);
+ return 1;
+}
+
+/* room on the device for `rows` compressed rows (and keys) of layer i; growing drops them */
+static int v4c_cache(V4Chain *ch, V4cLayer *ly, int rows) {
+ if (ly->ratio > 0 && ly->ccap < rows) {
+ int cap = 64; while (cap < rows) cap *= 2;
+ vkc_free(ly->ckv);
+ ly->ckv = vkc_buf((size_t)cap * ch->hd * sizeof(float), VKC_DEV);
+ ly->kv_valid = 0; ly->ccap = ly->ckv ? cap : 0;
+ if (!ly->ckv) return 0;
+ }
+ if (ly->idx && ly->icap < rows) {
+ int cap = 64; while (cap < rows) cap *= 2;
+ vkc_free(ly->ikey);
+ ly->ikey = vkc_buf((size_t)cap * ch->ID * sizeof(float), VKC_DEV);
+ ly->ik_valid = 0; ly->icap = ly->ikey ? cap : 0;
+ if (!ly->ikey) return 0;
+ }
+ return 1;
+}
+
+static int v4c_scratch(V4Chain *ch, int rows, int E, int LR) {
+ size_t r = (size_t)rows;
+ int cp = 2 * ch->hd, taps = coli_v4_full_dspark_wanted ? 3 : 1;
+ ch->wcap = (size_t)(E / 4 > 1 ? E / 4 : 1);
+ int ok = v4c_res(&ch->xs, r * ch->HD, VKC_DEV) && v4c_res(&ch->xn, r * ch->HD, VKC_DEV) &&
+ v4c_res(&ch->mix, r * ch->nm, VKC_DEV) && v4c_res(&ch->hpa, r * ch->hr, VKC_DEV) &&
+ v4c_res(&ch->hpf, r * ch->hr, VKC_DEV) && v4c_res(&ch->col, r * ch->D, VKC_DEV) &&
+ v4c_res(&ch->nrm, r * ch->D, VKC_DEV) && v4c_res(&ch->nq, r * ch->D, VKC_DEV) &&
+ v4c_res(&ch->qa, r * ch->QL, VKC_DEV) && v4c_res(&ch->qr, r * ch->QL, VKC_DEV) &&
+ v4c_res(&ch->qrq, r * ch->QL, VKC_DEV) && v4c_res(&ch->q, r * ch->nh * ch->hd, VKC_DEV) &&
+ v4c_res(&ch->kv, r * ch->hd, VKC_DEV) && v4c_res(&ch->heads, r * ch->nh * ch->hd, VKC_DEV) &&
+ v4c_res(&ch->grp, r * ch->og * ch->ol, VKC_DEV) && v4c_res(&ch->br, r * ch->D, VKC_DEV) &&
+ v4c_res(&ch->craw, r * cp, VKC_DEV) && v4c_res(&ch->cscr, r * cp, VKC_DEV) &&
+ v4c_res(&ch->icraw, r * 2 * ch->ID, VKC_DEV) && v4c_res(&ch->icscr, r * 2 * ch->ID, VKC_DEV) &&
+ v4c_res(&ch->iq, r * ch->IH * ch->ID, VKC_DEV) && v4c_res(&ch->ihw, r * ch->IH, VKC_DEV) &&
+ v4c_res(&ch->isc, r * ch->wcap, VKC_DEV) && v4c_res(&ch->list, (size_t)ch->nkind * r * LR, VKC_DEV) &&
+ v4c_res(&ch->cs, (2 * r + ch->rmax) * ch->rd, VKC_DEV) && v4c_res(&ch->sq, r * ch->D, VKC_DEV) &&
+ v4c_res(&ch->gs, r * ch->I, VKC_DEV) && v4c_res(&ch->us, r * ch->I, VKC_DEV) &&
+ v4c_res(&ch->hs, r * ch->I, VKC_DEV) && v4c_res(&ch->ds, r * ch->D, VKC_DEV) &&
+ v4c_res(&ch->h2d, r * ch->D, VKC_DOWN) && v4c_res(&ch->routed, r * ch->D, VKC_UP) &&
+ v4c_res(&ch->xd, r * ch->HD, VKC_DOWN) && v4c_res(&ch->taps, (size_t)taps * r * ch->HD, VKC_DOWN);
+ if (!ok) return 0;
+ if (ch->rows < rows) {
+ float *hr = realloc(ch->host_routed, r * ch->D * sizeof(float));
+ if (!hr) return 0;
+ ch->host_routed = hr; ch->rows = rows;
+ }
+ return 1;
+}
+
+/* ---- one layer's pieces ----------------------------------------------------------------- */
+static int v4c_round(VkcBuf *x, VkcBuf *y, int kind, int nseg, int per_row, int len, int block, int flags,
+ int x_off, int x_row, int x_seg, int y_off, int y_row, int y_seg) {
+ VkcDsRound p = {kind, nseg, per_row, len, block, flags, x_off, x_row, x_seg, y_off, y_row, y_seg,
+ kind == VKC_DS_HADAMARD ? 1.0f / sqrtf((float)len) : 0.0f};
+ return vkc_dsv4_round(x, y, &p);
+}
+/* n rows of len floats at off, row apart, to bf16 in place */
+static int v4c_bf16(VkcBuf *x, int n, int len, int off, int row) {
+ return v4c_round(x, x, VKC_DS_BF16, n, 1, len, 0, 0, off, row, 0, off, row, 0);
+}
+/* n contiguous rows of len floats to E4M3 per 128 (an fp8 matrix's input) */
+static int v4c_e4m3(VkcBuf *x, VkcBuf *y, int n, int len) {
+ return v4c_round(x, y, VKC_DS_E4M3, n, 1, len, 128, 0, 0, len, 0, 0, len, 0);
+}
+static int v4c_norm(V4Chain *ch, VkcBuf *x, int xo, int xrow, size_t wo, VkcBuf *y, int yo, int yrow, int rows, int D, int flags) {
+ VkcNorm p = {rows, D, 1, xo, xrow, D, yo, yrow, D, (int)wo, 0, flags, ch->engine->config.rms_norm_eps, 1.f};
+ return vkc_norm(x, ch->prm, y, &p);
+}
+/* RoPE on interleaved pairs, the last rd floats of each segment, then bf16 */
+static int v4c_rope(V4Chain *ch, VkcBuf *x, int nseg, int per_row, int x_off, int x_row, int x_seg,
+ int cs_off, int cs_row, int inverse) {
+ VkcDsRope p = {nseg, per_row, ch->rd, x_off, x_row, x_seg, cs_off, cs_row, inverse, 1};
+ return vkc_dsv4_rope(x, ch->cs, &p);
+}
+/* mHC's entry for a site (hc_pre with its own mix): the mix, the split into hp, the
+ * collapse, bf16, the norm, bf16, into nrm */
+static int v4c_pre(V4Chain *ch, ColiVkTensor *fn, size_t hco, VkcBuf *hp, size_t lno, int n) {
+ const ColiDeepSeekV4Config *c = &ch->engine->config;
+ int H = ch->H, D = ch->D, HD = ch->HD;
+ VkcMhc sp = {n, H, D, c->hc_sinkhorn_iters, 0, HD, 0, ch->nm, 0, ch->hr, 0, D, (int)hco, 0, c->rms_norm_eps, c->hc_eps, 0.f};
+ return vkc_matmul(fn, ch->xs, 0, ch->mix, 0, n) && vkc_mhc(VKC_MHC_SPLIT, ch->xs, ch->mix, hp, ch->prm, NULL, &sp) &&
+ vkc_mhc(VKC_MHC_COLLAPSE, ch->xs, NULL, hp, NULL, ch->col, &sp) && v4c_bf16(ch->col, n, D, 0, D) &&
+ v4c_norm(ch, ch->col, 0, D, lno, ch->nrm, 0, D, n, D, 0) && v4c_bf16(ch->nrm, n, D, 0, D);
+}
+/* mHC's exit: xn = comb(xs) + post * br, bf16, then the two swap */
+static int v4c_post(V4Chain *ch, VkcBuf *hp, int n) {
+ const ColiDeepSeekV4Config *c = &ch->engine->config;
+ VkcMhc po = {n, ch->H, ch->D, c->hc_sinkhorn_iters, 0, ch->HD, 0, ch->D, 0, ch->hr, 0, ch->HD, 0, 0, c->rms_norm_eps, c->hc_eps, 0.f};
+ if (!vkc_mhc(VKC_MHC_POST, ch->xs, ch->br, hp, NULL, ch->xn, &po) || !v4c_bf16(ch->xn, n, ch->HD, 0, ch->HD)) return 0;
+ VkcBuf *t = ch->xs; ch->xs = ch->xn; ch->xn = t;
+ return 1;
+}
+
+/* The forward being recorded: positions, rows, the RoPE tables' places in ch->cs (the
+ * window table's row 0 is position pb; the compress table's row 0 is pb - rmax + 1). */
+typedef struct { int start, n, pb, nr, csW, csC, LR, s_ident; } V4cFwd;
+
+/* the compressor's ring for nr rows into out (rows of D floats), then the produced rows'
+ * pooled-latent pipeline: bf16, norm, bf16, RoPE at the group's first position, and the
+ * attention's E4M3 per 64 on the no-position part (or the indexer's Hadamard and E2M1) */
+static int v4c_compress(V4Chain *ch, const V4cFwd *f, ColiVkTensor *wkv, ColiVkTensor *wg, VkcBuf *kvb, VkcBuf *scb,
+ VkcBuf *ring, size_t ape, VkcBuf *out, int r, int D, size_t norm, int fp4) {
+ int nr = f->nr, pb = f->pb, P = r == 4 ? 2 * D : D, g0 = pb / r, np = (pb + nr) / r - g0, rd = ch->rd;
+ VkcDsComp cp = {nr, pb, r, P, D, r == 4, 0, P, 0, P, (int)ape, 0, D, 0};
+ if (!vkc_matmul(wkv, ch->nrm, 0, kvb, 0, nr) || !vkc_matmul(wg, ch->nrm, 0, scb, 0, nr) ||
+ !vkc_dsv4_compress(kvb, scb, ring, ch->prm, out, &cp)) return 0;
+ if (np <= 0) return 1;
+ int o = g0 * D, cs = f->csC + (g0 * r - (pb - ch->rmax + 1)) * rd;
+ if (!v4c_bf16(out, np, D, o, D) || !v4c_norm(ch, out, o, D, norm, out, o, D, np, D, 0) || !v4c_bf16(out, np, D, o, D) ||
+ !v4c_rope(ch, out, np, 1, o + D - rd, D, 0, cs, r * rd, 0)) return 0;
+ if (!fp4) return v4c_round(out, out, VKC_DS_E4M3, np, 1, D - rd, 64, 1, o, D, 0, o, D, 0);
+ return v4c_round(out, out, VKC_DS_HADAMARD, np, 1, D, 0, 0, o, D, 0, o, D, 0) &&
+ v4c_round(out, out, VKC_DS_E2M1, np, 1, D, 32, 1, o, D, 0, o, D, 0);
+}
+
+/* attention_token_impl for nr rows on the device, into ch->br */
+static int v4c_attention(V4Chain *ch, int i, const V4cFwd *f) {
+ const ColiDeepSeekV4Config *c = &ch->engine->config;
+ V4cLayer *ly = &ch->ly[i];
+ int hd = ch->hd, nh = ch->nh, rd = ch->rd, QL = ch->QL, D = ch->D, nr = f->nr, pb = f->pb, qrow = nh * hd;
+ int tcs = ly->ratio > 0 ? f->csC + (ch->rmax - 1) * rd : f->csW; /* the table row of position pb */
+ int ok = v4c_e4m3(ch->nrm, ch->nq, nr, D) &&
+ vkc_matmul(ly->wq_a, ch->nq, 0, ch->qa, 0, nr) && v4c_bf16(ch->qa, nr, QL, 0, QL) &&
+ v4c_norm(ch, ch->qa, 0, QL, ly->o_qn, ch->qr, 0, QL, nr, QL, 0) && v4c_bf16(ch->qr, nr, QL, 0, QL) &&
+ v4c_e4m3(ch->qr, ch->qrq, nr, QL);
+ if (ok && ly->ratio > 0)
+ ok = v4c_compress(ch, f, ly->cwkv, ly->cwg, ch->craw, ch->cscr, ly->ring, ly->o_ape, ly->ckv, ly->ratio, hd, ly->o_cn, 0);
+ if (ok && ly->idx) { /* the indexer: keys, queries, scores, top-k */
+ int IH = ch->IH, ID = ch->ID, width = (pb + nr) / 4, K = c->index_topk;
+ ok = v4c_compress(ch, f, ly->iwkv, ly->iwg, ch->icraw, ch->icscr, ly->iring, ly->o_iape, ly->ikey, 4, ID, ly->o_icn, 1) &&
+ vkc_matmul(ly->iwq, ch->qrq, 0, ch->iq, 0, nr) && v4c_bf16(ch->iq, nr, IH * ID, 0, IH * ID) &&
+ v4c_rope(ch, ch->iq, nr * IH, IH, ID - rd, IH * ID, ID, tcs, rd, 0) &&
+ v4c_round(ch->iq, ch->iq, VKC_DS_HADAMARD, nr * IH, IH, ID, 0, 0, 0, IH * ID, ID, 0, IH * ID, ID) &&
+ v4c_round(ch->iq, ch->iq, VKC_DS_E2M1, nr * IH, IH, ID, 32, 1, 0, IH * ID, ID, 0, IH * ID, ID) &&
+ vkc_matmul(ly->iwp, ch->nrm, 0, ch->ihw, 0, nr);
+ if (ok && width > 0) {
+ VkcDsScore sp = {nr, pb, 4, IH, ID, width, 0, IH * ID, 0, IH, 0, ID, 0, (int)ch->wcap,
+ 1.0f / sqrtf((float)(ID * IH)), 0, 0};
+ VkcDsTopk tp = {nr, width, K, (int)ch->wcap, ch->W, f->LR, ch->Wd, 1};
+ ok = vkc_dsv4_score(ch->iq, ch->ihw, ly->ikey, NULL, ch->isc, &sp) && vkc_dsv4_topk(ch->isc, ch->list, &tp);
+ if (ok && f->s_ident > 0) { /* V4_IDX_IDENTITY: index order while all fit */
+ VkcDsTopk ti = tp; ti.S = f->s_ident; ti.order = 0;
+ ok = vkc_dsv4_topk(ch->isc, ch->list, &ti);
+ }
+ }
+ }
+ ok = ok && vkc_matmul(ly->wq_b, ch->qrq, 0, ch->q, 0, nr) && v4c_bf16(ch->q, nr, qrow, 0, qrow);
+ if (ok) { /* the per-head RMS without weight, bf16 */
+ VkcNorm p = {nr * nh, hd, nh, 0, qrow, hd, 0, qrow, hd, 0, 0, VKC_NORM_NOW, c->rms_norm_eps, 1.f};
+ ok = vkc_norm(ch->q, ch->prm, ch->q, &p) && v4c_bf16(ch->q, nr, qrow, 0, qrow);
+ }
+ ok = ok && vkc_matmul(ly->wkv, ch->nq, 0, ch->kv, 0, nr) && v4c_bf16(ch->kv, nr, hd, 0, hd) &&
+ v4c_norm(ch, ch->kv, 0, hd, ly->o_kn, ch->kv, 0, hd, nr, hd, 0) && v4c_bf16(ch->kv, nr, hd, 0, hd) &&
+ v4c_rope(ch, ch->q, nr * nh, nh, hd - rd, qrow, hd, tcs, rd, 0) &&
+ v4c_rope(ch, ch->kv, nr, 1, hd - rd, hd, 0, tcs, rd, 0) &&
+ v4c_round(ch->kv, ch->kv, VKC_DS_E4M3, nr, 1, hd - rd, 64, 1, 0, hd, 0, 0, hd, 0);
+ if (!ok) return 0;
+ VkcRegion *rg = malloc((size_t)nr * sizeof *rg); /* the new rows into the window ring */
+ if (!rg) return 0;
+ for (int s = 0; s < nr; s++) rg[s] = (VkcRegion){(size_t)((pb + s) % ch->Wd) * hd, (size_t)s * hd, (size_t)hd};
+ ok = vkc_copy_regions(ly->win, ch->kv, rg, nr);
+ free(rg);
+ int cnt = ch->W;
+ if (ly->idx) cnt += c->index_topk;
+ else if (ly->ratio > 0) cnt += (pb + nr) / ly->ratio;
+ int kind = ly->ratio > 0 && !ly->idx ? ly->kind : 0;
+ VkcDsAttn a = {nr, nh, hd, cnt, kind * nr * f->LR, f->LR, ch->Wd, 0, 0, 0, qrow, 0, qrow, (int)ly->o_sink, 1,
+ 1.0f / sqrtf((float)hd)};
+ ok = ok && vkc_dsv4_attn(ch->q, ly->win, ly->ratio > 0 ? ly->ckv : NULL, ch->list, ch->prm, ch->heads, &a) &&
+ v4c_rope(ch, ch->heads, nr * nh, nh, hd - rd, qrow, hd, tcs, rd, 1) &&
+ v4c_round(ch->heads, ch->heads, VKC_DS_E4M3, nr, 1, qrow, 128, 0, 0, qrow, 0, 0, qrow, 0);
+ int og = ch->og, ol = ch->ol, gw = ch->gw;
+ if (ok && og == 1) ok = vkc_matmul(ly->wo_a[0], ch->heads, 0, ch->grp, 0, nr);
+ for (int g = 0; ok && og > 1 && g < og; g++) {
+ VkcHgemv h = {0, nr, 1, ol, ol, 0, g * gw, qrow, 0, g * ol, og * ol, 0, 0, 0, 0};
+ ok = vkc_mla_hgemv(ly->wo_a[g], ch->heads, ch->grp, NULL, &h);
+ }
+ return ok && v4c_bf16(ch->grp, nr, og * ol, 0, og * ol) && v4c_e4m3(ch->grp, ch->grp, nr, og * ol) &&
+ vkc_matmul(ly->wo_b, ch->grp, 0, ch->br, 0, nr) && v4c_bf16(ch->br, nr, D, 0, D);
+}
+
+/* the shared expert: E4M3 input, w1 and w3, V4's SwiGLU, E4M3, w2, bf16 into ds */
+static int v4c_shared(V4Chain *ch, V4cLayer *ly, int nr) {
+ float lim = ch->engine->config.swiglu_limit;
+ VkcDsSwiglu sw = {nr * ch->I, 0, 0, 0, lim > 0.0f ? lim : 0.0f};
+ return v4c_e4m3(ch->nrm, ch->sq, nr, ch->D) && vkc_matmul(ly->sh1, ch->sq, 0, ch->gs, 0, nr) &&
+ vkc_matmul(ly->sh3, ch->sq, 0, ch->us, 0, nr) && vkc_dsv4_swiglu(ch->gs, ch->us, ch->hs, &sw) &&
+ v4c_e4m3(ch->hs, ch->hs, nr, ch->I) && vkc_matmul(ly->sh2, ch->hs, 0, ch->ds, 0, nr) &&
+ v4c_bf16(ch->ds, nr, ch->D, 0, ch->D);
+}
+/* the FFN branch joins: br = bf16(routed + shared), then mHC's exit */
+static int v4c_join(V4Chain *ch, int nr) {
+ VkcEw add = {VKC_EW_ADD, nr * ch->D, ch->D, 1, 0, 1, 0, 0, 0, 0, 0, 1.f};
+ return vkc_ew(ch->br, ch->routed, ch->ds, NULL, NULL, &add) && v4c_bf16(ch->br, nr, ch->D, 0, ch->D) &&
+ v4c_post(ch, ch->hpf, nr);
+}
+
+static int v4c_read_count(ColiDeepSeekV4WindowAttentionState *s, int *count, int *keys) {
+ ColiV4AttentionView v;
+ if (coli_v4_attention_view(s, &v)) return 0;
+ *count = v.compressed_count;
+ *keys = 0;
+ if (v.indexer) { ColiV4IndexerView iv; if (coli_v4_indexer_view(v.indexer, &iv)) return 0; *keys = iv.count; }
+ return 1;
+}
+
+/* Every layer for the n rows of streams in *state_ptr (positions start..), the final
+ * streams into *state_ptr (swapped with *next_ptr, as the CPU's layer loop leaves them).
+ * 1: done; 0: not taken (the CPU runs the layers, nothing changed); -1: the MoE failed
+ * (error set, the host state as before the forward). */
+static int v4c_forward(ColiV4Engine *engine, float **state_ptr, float **next_ptr,
+ ColiDeepSeekV4WindowAttentionState **attention, const ColiDeepSeekV4Config *config,
+ ColiExpertStore *experts, const int *tokens, int start, int n, int pool,
+ char *error, size_t error_size) {
+ V4Chain *ch = g_v4c;
+ if (!g_v4c_mode || !ch || !ch->ok || ch->failed || !engine || engine != ch->engine || !attention || n < 1) return 0;
+ if (g_v4c_mode == COLI_VK_CHAIN_PREFILL && n <= 2) return 0;
+ if (vkc_lost()) { ch->failed = 1; g_v4c_mode = 0; return 0; }
+ const ColiDeepSeekV4Config *c = config;
+ int L = ch->L, D = ch->D, HD = ch->HD, hd = ch->hd, rd = ch->rd, W = ch->W, E = start + n;
+ int K = 0; /* the compressed slots of a list row */
+ for (int i = 0; i < L; i++) {
+ V4cLayer *ly = &ch->ly[i];
+ int k = ly->idx ? c->index_topk : ly->ratio > 0 ? E / ly->ratio : 0;
+ if (k > K) K = k;
+ }
+ if (W + K > 3072) {
+ if (!ch->told) fprintf(stderr, "[VK] deepseek_v4 chain: %d compressed rows at position %d pass the attention's "
+ "3072 entries; the CPU runs the layers from here\n", K, E);
+ ch->told = 1;
+ return 0;
+ }
+ char err[256] = {0};
+ for (int i = 0; i < L; i++) /* the compressors and the indexer exist on the host */
+ if (ch->ly[i].ratio > 0 &&
+ coli_v4_window_attention_prepare(attention[i], &engine->dense_resident.layers[i], config, err, sizeof err))
+ return 0;
+ if (attention != ch->attn) { v4c_lower(0); ch->attn = attention; }
+ if (start != ch->E) v4c_lower(start);
+ for (int i = 0; i < L; i++) { /* the host's state is where the forward starts */
+ V4cLayer *ly = &ch->ly[i];
+ int cnt, keys;
+ if (ly->ratio <= 0) continue;
+ if (!v4c_read_count(attention[i], &cnt, &keys) || cnt != start / ly->ratio || (ly->idx && keys != start / 4)) {
+ if (!ch->told) fprintf(stderr, "[VK] deepseek_v4 chain: layer %d holds %d compressed rows at position %d; "
+ "the CPU runs this forward\n", i, cnt, start);
+ ch->told = 1;
+ return 0;
+ }
+ }
+ int rows = n < v4c_rows() ? n : v4c_rows(), LR = W + (K > 0 ? K : 1);
+ int ok = v4c_scratch(ch, rows, E, LR);
+ for (int i = 0; ok && i < L; i++) ok = v4c_cache(ch, &ch->ly[i], E / (ch->ly[i].ratio > 0 ? ch->ly[i].ratio : 1) + 1);
+ /* what comes back at the end: the window rows, each layer's new compressed rows and
+ * keys, the rings (a ring of ratio r without the overlap: the slots this forward wrote) */
+ size_t pn = 0;
+ int q0 = E - W > start ? E - W : start, nq = E - q0;
+ for (int i = 0; ok && i < L; i++) {
+ V4cLayer *ly = &ch->ly[i];
+ ly->pw = pn; pn += (size_t)nq * hd;
+ if (ly->ratio <= 0) continue;
+ int r = ly->ratio;
+ ly->pc = pn; pn += (size_t)(E / r - start / r) * hd;
+ ly->q1 = E - r > start ? E - r : start; /* without the overlap only the slots written change */
+ ly->ns = r == 4 ? ly->crows : E - ly->q1;
+ ly->pr = pn; pn += (size_t)2 * ly->ns * ly->cproj;
+ if (!ly->idx) continue;
+ ly->pk = pn; pn += (size_t)(E / 4 - start / 4) * ch->ID;
+ ly->pir = pn; pn += (size_t)2 * 8 * 2 * ch->ID;
+ }
+ ok = ok && v4c_res(&ch->pull, pn, VKC_DOWN);
+ float *cs = malloc((size_t)(2 * rows + ch->rmax) * rd * sizeof(float));
+ int *wl = malloc((size_t)ch->nkind * rows * LR * sizeof(int));
+ float *cos_ = malloc((size_t)rd / 2 * sizeof(float)), *sin_ = malloc((size_t)rd / 2 * sizeof(float));
+ if (!ok || !cs || !wl || !cos_ || !sin_) {
+ free(cs); free(wl); free(cos_); free(sin_);
+ fprintf(stderr, "[VK] deepseek_v4 chain: device memory for %d rows at %d positions refused; the CPU runs the layers\n", rows, E);
+ ch->failed = 1; g_v4c_mode = 0;
+ return 0;
+ }
+ /* every matrix on the per-row GEMV: a row's bits do not depend on the chunk it sits
+ * in (a prompt, a verify, a decode step, a prefix reused or computed cold), as on the
+ * CPU, whose batched and per-token kernels give the same bits */
+ vkc_gemm_rows(0);
+ static int ident = -1;
+ if (ident < 0) { const char *e = getenv("V4_IDX_IDENTITY"); ident = e && *e == '1'; }
+ float *state = *state_ptr, *next = *next_ptr;
+ int T0 = L - 3, taps = coli_v4_full_dspark_wanted; /* DSpark's targets: the last three layers */
+ for (int c0 = 0; c0 < n; c0 += rows) {
+ int nr = n - c0 < rows ? n - c0 : rows, pb = start + c0, last = c0 + nr == n;
+ V4cFwd f = {start, n, pb, nr, 0, nr * rd, LR, 0};
+ if (ident) for (int s = 0; s < nr && (pb + s + 1) / 4 <= c->index_topk; s++) f.s_ident = s + 1;
+ /* this chunk's RoPE rows, the CPU's own tables: the window table (the plain base)
+ * from position pb, the compress table (YaRN, compress_rope_theta) from pb - rmax + 1 */
+ for (int s = 0; s < nr + ch->rmax - 1; s++) {
+ for (int t = 0; t < 2; t++) {
+ if (t == 0 && s >= nr) continue;
+ int p = t == 0 ? pb + s : pb - ch->rmax + 1 + s;
+ float *dst = cs + (t == 0 ? f.csW : f.csC) + (size_t)s * rd;
+ if (p < 0 || coli_v4_rope_position(cos_, sin_, rd, p, t ? c->original_max_position_embeddings : 0,
+ t ? c->compress_rope_theta : c->rope_theta, c->rope_factor,
+ c->rope_beta_fast, c->rope_beta_slow)) {
+ memset(dst, 0, (size_t)rd * sizeof(float));
+ continue;
+ }
+ for (int k = 0; k < rd / 2; k++) { dst[2 * k] = cos_[k]; dst[2 * k + 1] = sin_[k]; }
+ }
+ }
+ /* each list kind's rows: the window (the last W positions a row reaches, oldest
+ * first, as device ring rows), then the compressed rows: the indexer's top-k slots
+ * (written on the device), or every row a layer of that ratio has made, in order */
+ for (int k = 0; k < ch->nkind; k++)
+ for (int s = 0; s < nr; s++) {
+ int p = pb + s, *row = wl + ((size_t)k * nr + s) * LR;
+ for (int j = 0; j < W; j++) { int q = p - W + 1 + j; row[j] = q >= 0 ? q % ch->Wd : -1; }
+ for (int j = W; j < LR; j++) {
+ int g = j - W;
+ row[j] = k > 0 && g < (p + 1) / ch->kratio[k] ? ch->Wd + g : -1;
+ }
+ }
+ if (!vkc_begin() || !vkc_write(ch->xs, 0, state + (size_t)c0 * HD, (size_t)nr * HD * sizeof(float)) ||
+ !vkc_write(ch->list, 0, wl, (size_t)ch->nkind * nr * LR * sizeof(int)) ||
+ !vkc_write(ch->cs, 0, cs, (size_t)(f.csC + (nr + ch->rmax - 1) * rd) * sizeof(float))) goto lost;
+ if (c0 == 0) { /* the device's copies made the host's, as the forward needs them */
+ int q_lo = start - W + 1 < 0 ? 0 : start - W + 1;
+ for (int i = 0; ok && i < L; i++) {
+ V4cLayer *ly = &ch->ly[i];
+ ColiV4AttentionView v;
+ if (coli_v4_attention_view(attention[i], &v)) goto lost;
+ for (int q = q_lo; ok && q < start; q++)
+ if (q >= ch->win_valid || ch->slot_pos[q % ch->Wd] != q) /* stale, or its row taken since */
+ ok = vkc_write(ly->win, (size_t)(q % ch->Wd) * hd, v.kv + (size_t)(q % W) * hd, (size_t)hd * sizeof(float));
+ if (ly->ratio <= 0) continue;
+ int t0 = ly->kv_valid, t1 = start / ly->ratio;
+ if (ok && t1 > t0) ok = vkc_write(ly->ckv, (size_t)t0 * hd, v.compressed + (size_t)t0 * hd, (size_t)(t1 - t0) * hd * sizeof(float));
+ ColiV4CompressorView cv;
+ size_t rf = (size_t)ly->crows * ly->cproj;
+ if (ok && !ly->ring_ok)
+ ok = !coli_v4_compressor_view(v.compressor, &cv) && cv.rows == ly->crows && cv.projection == ly->cproj &&
+ vkc_write(ly->ring, 0, cv.kv, rf * sizeof(float)) && vkc_write(ly->ring, rf, cv.score, rf * sizeof(float));
+ if (!ly->idx) continue;
+ ColiV4IndexerView iv;
+ ok = ok && !coli_v4_indexer_view(v.indexer, &iv);
+ int k0 = ly->ik_valid, k1 = start / 4;
+ if (ok && k1 > k0) ok = vkc_write(ly->ikey, (size_t)k0 * ch->ID, iv.keys + (size_t)k0 * ch->ID, (size_t)(k1 - k0) * ch->ID * sizeof(float));
+ size_t ir = (size_t)8 * 2 * ch->ID;
+ if (ok && !ly->iring_ok)
+ ok = !coli_v4_compressor_view(iv.compressor, &cv) && cv.rows == 8 && cv.projection == 2 * ch->ID &&
+ vkc_write(ly->iring, 0, cv.kv, ir * sizeof(float)) && vkc_write(ly->iring, ir, cv.score, ir * sizeof(float));
+ }
+ if (!ok) goto lost;
+ for (int q = q_lo; q < start; q++) ch->slot_pos[q % ch->Wd] = q;
+ }
+ for (int s = 0; s < nr; s++) ch->slot_pos[(pb + s) % ch->Wd] = pb + s; /* this chunk's rows go in below */
+ int pending = 0;
+ for (int i = 0; i < L && ok; i++) {
+ V4cLayer *ly = &ch->ly[i];
+ if (pending) { /* the FFN branch of the layer before */
+ ok = v4c_join(ch, nr);
+ if (ok && taps && i - 1 >= T0) ok = vkc_copy(ch->taps, (size_t)(i - 1 - T0) * nr * HD, ch->xs, 0, (size_t)nr * HD);
+ pending = 0;
+ }
+ ok = ok && v4c_pre(ch, ly->fna, ly->o_hca, ch->hpa, ly->o_an, nr) && v4c_attention(ch, i, &f) &&
+ v4c_post(ch, ch->hpa, nr) && v4c_pre(ch, ly->fnf, ly->o_hcf, ch->hpf, ly->o_fn, nr) &&
+ vkc_copy(ch->h2d, 0, ch->nrm, 0, (size_t)nr * D) && vkc_submit(1); /* A1 */
+ if (!ok) break;
+ /* A2: the shared expert, while the host computes the routed experts */
+ ok = vkc_begin() && v4c_shared(ch, ly, nr) && vkc_submit(0);
+ if (!ok) break;
+ double t1 = spec_now();
+ if (pool) coli_v4_expert_store_prefill_pool(experts, i);
+ if (coli_v4_moe_routed(ch->host_routed, &engine->dense_resident.layers[i], config, experts,
+ (const float *)vkc_ptr(ch->h2d), tokens + c0, nr, error, error_size)) {
+ if (pool) coli_v4_expert_store_prefill_pool(experts, -1);
+ vkc_finish();
+ for (int k = 0; k < ch->Wd; k++) ch->slot_pos[k] = -1; /* some layers wrote this chunk's rows */
+ v4c_lower(start);
+ free(cs); free(wl); free(cos_); free(sin_);
+ return -1;
+ }
+ memcpy(vkc_ptr(ch->routed), ch->host_routed, (size_t)nr * D * sizeof(float));
+ ch->host_ms += (spec_now() - t1) * 1e3;
+ ok = vkc_begin();
+ pending = 1;
+ }
+ if (ok && pending) {
+ ok = v4c_join(ch, nr);
+ if (ok && taps && L - 1 >= T0) ok = vkc_copy(ch->taps, (size_t)(L - 1 - T0) * nr * HD, ch->xs, 0, (size_t)nr * HD);
+ }
+ ok = ok && vkc_copy(ch->xd, 0, ch->xs, 0, (size_t)nr * HD);
+ if (ok && last) { /* what the host's state gets back */
+ for (int i = 0; i < L && ok; i++) {
+ V4cLayer *ly = &ch->ly[i];
+ VkcRegion *rg = malloc((size_t)(nq > 0 ? nq : 1) * sizeof *rg);
+ if (!rg) { ok = 0; break; }
+ for (int q = q0; q < E; q++) rg[q - q0] = (VkcRegion){ly->pw + (size_t)(q - q0) * hd, (size_t)(q % ch->Wd) * hd, (size_t)hd};
+ ok = vkc_copy_regions(ch->pull, ly->win, rg, nq);
+ free(rg);
+ if (!ok || ly->ratio <= 0) continue;
+ int r = ly->ratio, g0 = start / r, np = E / r - g0;
+ size_t rf = (size_t)ly->crows * ly->cproj, cp = (size_t)ly->cproj;
+ ok = vkc_copy(ch->pull, ly->pc, ly->ckv, (size_t)g0 * hd, (size_t)np * hd);
+ if (ok && r == 4) ok = vkc_copy(ch->pull, ly->pr, ly->ring, 0, 2 * rf);
+ else if (ok) {
+ VkcRegion *rr = malloc((size_t)2 * ly->ns * sizeof *rr);
+ if (!rr) { ok = 0; break; }
+ for (int j = 0; j < ly->ns; j++) {
+ size_t slot = (size_t)((ly->q1 + j) % r);
+ rr[2 * j] = (VkcRegion){ly->pr + j * cp, slot * cp, cp};
+ rr[2 * j + 1] = (VkcRegion){ly->pr + ((size_t)ly->ns + j) * cp, rf + slot * cp, cp};
+ }
+ ok = vkc_copy_regions(ch->pull, ly->ring, rr, 2 * ly->ns);
+ free(rr);
+ }
+ if (!ok || !ly->idx) continue;
+ int k0 = start / 4, nk = E / 4 - k0;
+ ok = vkc_copy(ch->pull, ly->pk, ly->ikey, (size_t)k0 * ch->ID, (size_t)nk * ch->ID) &&
+ vkc_copy(ch->pull, ly->pir, ly->iring, 0, (size_t)2 * 8 * 2 * ch->ID);
+ }
+ }
+ ok = ok && vkc_submit(1);
+ if (!ok) goto lost;
+ memcpy(next + (size_t)c0 * HD, vkc_ptr(ch->xd), (size_t)nr * HD * sizeof(float));
+ if (taps) {
+ const float *tp = (const float *)vkc_ptr(ch->taps);
+ for (int t = T0 < 0 ? 0 : T0; t < L; t++)
+ for (int s = 0; s < nr; s++)
+ v4_mainh_tap(c, t, tp + ((size_t)(t - T0) * nr + s) * HD, (int64_t)pb + s);
+ }
+ }
+ if (pool) coli_v4_expert_store_prefill_pool(experts, -1);
+ /* every chunk is through: the host's state as the CPU would have left it */
+ {
+ const float *pl = (const float *)vkc_ptr(ch->pull);
+ for (int i = 0; i < L; i++) {
+ V4cLayer *ly = &ch->ly[i];
+ ColiV4AttentionView v;
+ coli_v4_attention_view(attention[i], &v);
+ for (int q = q0; q < E; q++) memcpy(v.kv + (size_t)(q % W) * hd, pl + ly->pw + (size_t)(q - q0) * hd, (size_t)hd * sizeof(float));
+ if (ly->ratio <= 0) continue;
+ int r = ly->ratio, g0 = start / r, np = E / r - g0;
+ if (coli_v4_attention_reserve(attention[i], E / r)) goto host_oom;
+ coli_v4_attention_view(attention[i], &v);
+ memcpy(v.compressed + (size_t)g0 * hd, pl + ly->pc, (size_t)np * hd * sizeof(float));
+ coli_v4_attention_set_count(attention[i], E / r);
+ ColiV4CompressorView cv;
+ coli_v4_compressor_view(v.compressor, &cv);
+ size_t rf = (size_t)ly->crows * ly->cproj, cp = (size_t)ly->cproj;
+ if (r == 4) {
+ memcpy(cv.kv, pl + ly->pr, rf * sizeof(float));
+ memcpy(cv.score, pl + ly->pr + rf, rf * sizeof(float));
+ } else for (int j = 0; j < ly->ns; j++) {
+ size_t slot = (size_t)((ly->q1 + j) % r);
+ memcpy(cv.kv + slot * cp, pl + ly->pr + j * cp, cp * sizeof(float));
+ memcpy(cv.score + slot * cp, pl + ly->pr + ((size_t)ly->ns + j) * cp, cp * sizeof(float));
+ }
+ if (!ly->idx) continue;
+ ColiV4IndexerView iv;
+ int k0 = start / 4, nk = E / 4 - k0;
+ if (coli_v4_indexer_reserve(v.indexer, E / 4)) goto host_oom;
+ coli_v4_indexer_view(v.indexer, &iv);
+ memcpy(iv.keys + (size_t)k0 * ch->ID, pl + ly->pk, (size_t)nk * ch->ID * sizeof(float));
+ coli_v4_indexer_set_count(v.indexer, E / 4);
+ coli_v4_compressor_view(iv.compressor, &cv);
+ size_t ir = (size_t)8 * 2 * ch->ID;
+ memcpy(cv.kv, pl + ly->pir, ir * sizeof(float));
+ memcpy(cv.score, pl + ly->pir + ir, ir * sizeof(float));
+ }
+ ch->win_valid = E; ch->E = E;
+ for (int i = 0; i < L; i++) {
+ V4cLayer *ly = &ch->ly[i];
+ if (ly->ratio > 0) { ly->kv_valid = E / ly->ratio; ly->ring_ok = 1; }
+ if (ly->idx) { ly->ik_valid = E / 4; ly->iring_ok = 1; }
+ }
+ }
+ *state_ptr = next; *next_ptr = state;
+ free(cs); free(wl); free(cos_); free(sin_);
+ ch->forwards++;
+ return 1;
+host_oom: /* the host state half written: as a failed CPU forward leaves it, an error */
+ free(cs); free(wl); free(cos_); free(sin_);
+ v4c_lower(0);
+ if (error && error_size) snprintf(error, error_size, "out of memory writing the chained forward's state back");
+ return -1;
+lost: /* a frame failed: the device is gone (or would not take a command); the CPU takes over */
+ free(cs); free(wl); free(cos_); free(sin_);
+ if (pool) coli_v4_expert_store_prefill_pool(experts, -1);
+ if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost(); }
+ g_v4c_mode = 0; ch->failed = 1;
+ v4c_lower(0);
+ fprintf(stderr, "[VK] deepseek_v4 chain: the device was lost; the CPU runs this forward again and from here on "
+ "(the host's state is current: there is nothing to rebuild)\n");
+ return 0;
+}
+
+static void v4c_report(void) {
+ V4Chain *ch = g_v4c;
+ if (!ch || !ch->ok || !ch->forwards) return;
+ VkcStats st; vkc_stats(&st);
+ fprintf(stderr, "[VK] deepseek_v4 chain: %llu forwards, %llu frames (%llu ops, %llu matmuls, %llu tiled GEMM), "
+ "%.1f ms waiting for the device, %.1f ms of routed experts on the host, %.1f MiB on the device\n",
+ ch->forwards, st.frames, st.ops, st.matmuls, st.gemms, st.wait_ms, ch->host_ms, st.dev_bytes / 1048576.0);
+ vkc_prof_print();
+}
+
+/* COLI_VK_CHAIN at startup, before the tier sizes itself (the trunk's device copies count
+ * as used): the decision, the pipelines, the resident layers and their tensors. */
+static void v4c_start(const ColiV4Engine *cengine) {
+ ColiV4Engine *engine = (ColiV4Engine *)cengine;
+ if (!g_v4_vk_ready || !engine) return;
+ int tier_on = vkt_wanted() && engine->experts && !engine->experts->gpu && engine->config.n_routed_experts > 0;
+ int on = coli_vk_chain_decide("deepseek_v4", tier_on, COLI_VK_CHAIN_UNMEASURED);
+ const char *no = NULL;
+ if (on && !(g_v4c_inited = vkc_init())) no = "the chain's pipelines did not come up";
+ if (on && !no && !(vkc_mla_ready() && vkc_mhc_ready() && vkc_dsv4_ready()))
+ no = "the MLA, mHC or DeepSeek shaders are missing (chain_hgemv, chain_mhc, chain_dsv4)";
+ if (no) fprintf(stderr, "[VK] deepseek_v4: %s: the dense chain stays off\n", no);
+ if (!on || no) return;
+ if (!v4c_setup(engine)) return;
+ g_v4c_mode = on;
+}
+/* After the tier's: at exit the chain goes before the device. */
+static void v4c_atexit(void) {
+ if (g_v4c_inited) atexit(vkc_shutdown);
+}
+/* v4_vk_close, before the matrices go: the line, then every frame through. */
+static void v4c_close(void) {
+ v4c_report();
+ if (g_v4c_inited && !vkc_lost()) vkc_finish();
+}
diff --git a/c/deepseek_v4_internal.h b/c/deepseek_v4_internal.h
index b9111bad..fbe5b428 100644
--- a/c/deepseek_v4_internal.h
+++ b/c/deepseek_v4_internal.h
@@ -664,6 +664,43 @@ uint64_t coli_v4_expert_store_record_bytes(ColiExpertStore *store);
int coli_v4_expert_store_read_private(ColiExpertStore *store, ColiExpertKey key,
unsigned char *buffer, ColiExpertView *view);
extern int (*coli_v4_expert_store_device_tier)(int layer, int expert);
+
+/* The dense chain (deepseek_v4_chain.h, COLI_VK_CHAIN): a layer's attention state as its
+ * units keep it, so the chain can mirror it on the device and write back what a forward
+ * changed, as the CPU would have left it. The views point into the live state.
+ * attention the window ring [window][head_dim] (position % window), the compressed
+ * rows [capacity][head_dim] and their count, the compressor and the
+ * indexer (NULL on a layer without them, or before the layer first ran:
+ * coli_v4_window_attention_prepare makes them);
+ * compressor its ring: kv and score rows [rows][projection] (rows = ratio, twice that
+ * with the overlap of ratio 4; projection = head_dim, twice that likewise);
+ * indexer its keys [capacity][head_dim] and their count, and its compressor.
+ * reserve makes room for `rows` rows (the contents kept, the count untouched); set_count
+ * sets the count (at most the capacity). */
+typedef struct {
+ float *kv, *compressed;
+ int window, head_dim, ratio, compressed_count, compressed_capacity;
+ ColiDeepSeekV4CompressorState *compressor;
+ ColiDeepSeekV4Indexer *indexer;
+} ColiV4AttentionView;
+int coli_v4_attention_view(ColiDeepSeekV4WindowAttentionState *state, ColiV4AttentionView *view);
+int coli_v4_attention_reserve(ColiDeepSeekV4WindowAttentionState *state, int rows);
+int coli_v4_attention_set_count(ColiDeepSeekV4WindowAttentionState *state, int count);
+typedef struct { float *kv, *score; int rows, projection, ratio, head_dim, rotate_fp4; } ColiV4CompressorView;
+int coli_v4_compressor_view(ColiDeepSeekV4CompressorState *state, ColiV4CompressorView *view);
+typedef struct { float *keys; int count, capacity, head_dim; ColiDeepSeekV4CompressorState *compressor; } ColiV4IndexerView;
+int coli_v4_indexer_view(ColiDeepSeekV4Indexer *state, ColiV4IndexerView *view);
+int coli_v4_indexer_reserve(ColiDeepSeekV4Indexer *state, int rows);
+int coli_v4_indexer_set_count(ColiDeepSeekV4Indexer *state, int count);
+/* The MoE of `rows` normalized FFN rows without the shared expert: each row's routed
+ * experts summed in the CPU block's order (the store, the Vulkan tier), the sum left as
+ * it stands before the CPU adds the shared expert and rounds; the chain adds the shared
+ * expert it ran on the device. tokens: the rows' ids (the hash router). 0, or -1 with
+ * the reason in error. */
+int coli_v4_moe_routed(float *routed, const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, ColiExpertStore *store,
+ const float *inputs, const int *tokens, int rows,
+ char *error, size_t error_size);
#endif
#ifdef __cplusplus