Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
deepseek_v4: the dense chain on the device (COLI_VK_CHAIN=1)
JustVuggea64d5ca4937
diff --git a/c/Makefile.deepseek-v4 b/c/Makefile.deepseek-v4index f8b2823e..54529a14 100644--- a/c/Makefile.deepseek-v4+++ b/c/Makefile.deepseek-v4@@ -240,6 +240,13 @@ V4_VK_SPV = shaders/qmatmul.spv shaders/qmatmul_gate_up.spv \shaders/attention_absorb.spv shaders/rmsnorm.spv \shaders/qmatmul_gemm.spv shaders/qmatmul_coop.spv \shaders/expert_act.spv shaders/expert_act_v4.spv+# the dense chain (vk_chain.c, deepseek_v4_chain.h: COLI_VK_CHAIN) and its shaders+V4_VK_OBJ += vk_chain_v4.o+V4_VK_SPV += shaders/chain_norm.spv shaders/chain_rope.spv shaders/chain_attn.spv \+ shaders/chain_dnconv.spv shaders/chain_dnrec.spv shaders/chain_ew.spv \+ shaders/chain_qsa.spv shaders/chain_ple.spv shaders/chain_gemv.spv \+ shaders/chain_mla.spv shaders/chain_hgemv.spv shaders/chain_dsa.spv \+ shaders/chain_kda.spv shaders/chain_mhc.spv shaders/chain_dsv4.spvendifinclude Makefile.deepseek-v4.units@@ -315,6 +322,8 @@ backend_vulkan_v4.o: backend_vulkan.c backend_vulkan.h $(V4_FLAGS_STAMP)$(CC) $(CFLAGS) -c backend_vulkan.c -o $@vk_tier_v4.o: vk_tier.c vk_tier.h backend_vulkan.h tier.h $(V4_FLAGS_STAMP)$(CC) $(CFLAGS) -c vk_tier.c -o $@+vk_chain_v4.o: vk_chain.c vk_chain.h vk_alloc.h backend_vulkan.h $(V4_FLAGS_STAMP)+ $(CC) $(CFLAGS) -c vk_chain.c -o $@shaders/%.spv: shaders/%.comp@command -v $(GLSLC) >/dev/null 2>&1 || { echo "glslc not found: install shaderc (for VK=1)" >&2; exit 1; }$(GLSLC) --target-env=vulkan1.2 $< -o $@@@ -332,7 +341,7 @@ $(REGISTRY_OBJ): expert_store_registry.c expert_store_registry.h expert_store.h$(TARGET_OBJS) $(TEST_UNIT_OBJS): %.o: deepseek_v4.c deepseek_v4.h \deepseek_v4_internal.h deepseek_v4_dspark.inc st.h json.h compat.h tensor.h quant.h fp8_format.h sse41_kernels.h \- route_trace.h vk_tier.h \+ route_trace.h vk_tier.h deepseek_v4_chain.h vk_chain.h \native_quant.h native_quant_batch.h native_quant_dual.h \native_quant_fp4_rows16.h expert_store_registry.h $(V4_FLAGS_STAMP)$(CC) $(CFLAGS) -D$* -c deepseek_v4.c -o $@@@ -384,4 +393,4 @@ deepseek-v4-clean:rm -f $(V4_BINARY) $(V4_OBJS) $(TEST_UNIT_OBJS) $(V4_HOT_TEST_OBJ) \$(V4_BATCH_TEST_OBJ) \$(V4_SERVE_TEST) test_expert_store_registry $(V4_FLAGS_STAMP) \- $(V4_CUDA_FLAGS_STAMP) backend_vulkan_v4.o vk_tier_v4.o+ $(V4_CUDA_FLAGS_STAMP) backend_vulkan_v4.o vk_tier_v4.o vk_chain_v4.odiff --git a/c/deepseek_v4.c b/c/deepseek_v4.cindex 308611e9..7f3b38df 100644--- a/c/deepseek_v4.c+++ b/c/deepseek_v4.c@@ -2210,6 +2210,36 @@ int coli_v4_attention_window_token_ref(return attention_token_impl(output, state, weights, config, input, position,error, error_size);}++#ifdef COLI_VULKAN+/* The dense chain's view of this state (deepseek_v4_internal.h). */+int coli_v4_attention_view(ColiDeepSeekV4WindowAttentionState *state,+ ColiV4AttentionView *view) {+ if (!state || !view) return -1;+ *view = (ColiV4AttentionView){state->kv, state->compressed, state->window_size,+ state->head_dim, state->ratio, state->compressed_count,+ state->compressed_capacity, state->compressor,+ state->indexer};+ return 0;+}+int coli_v4_attention_reserve(ColiDeepSeekV4WindowAttentionState *state, int rows) {+ if (!state || rows < 0 || (rows && !state->compressed)) return -1;+ if (rows <= state->compressed_capacity) return 0;+ int capacity = state->compressed_capacity > 0 ? state->compressed_capacity : 16;+ while (capacity < rows) capacity *= 2;+ float *grown = realloc(state->compressed,+ (size_t)capacity * state->head_dim * sizeof(*grown));+ if (!grown) return -1;+ state->compressed = grown;+ state->compressed_capacity = capacity;+ return 0;+}+int coli_v4_attention_set_count(ColiDeepSeekV4WindowAttentionState *state, int count) {+ if (!state || count < 0 || count > state->compressed_capacity) return -1;+ state->compressed_count = count;+ return 0;+}+#endif#endif /* COLI_V4_UNIT_ATTENTION */#ifdef COLI_V4_UNIT_ATTENTION_BATCH@@ -3482,6 +3512,18 @@ static int compressor_pool_and_emit(ColiDeepSeekV4CompressorState *state,*produced = 1;return 0;}++#ifdef COLI_VULKAN+/* The dense chain's view of the ring (deepseek_v4_internal.h). */+int coli_v4_compressor_view(ColiDeepSeekV4CompressorState *state,+ ColiV4CompressorView *view) {+ if (!state || !view) return -1;+ *view = (ColiV4CompressorView){state->kv_state, state->score_state, state->state_rows,+ state->projection_dim, state->ratio, state->head_dim,+ state->rotate_fp4};+ return 0;+}+#endif#endif /* COLI_V4_UNIT_COMPRESSOR */#ifdef COLI_V4_UNIT_INDEXER@@ -4186,6 +4228,34 @@ const float *coli_v4_indexer_compressed_values(int coli_v4_indexer_compressed_count(const ColiDeepSeekV4Indexer *state) {return state ? state->count : 0;}++#ifdef COLI_VULKAN+/* The dense chain's view of the keys (deepseek_v4_internal.h). */+int coli_v4_indexer_view(ColiDeepSeekV4Indexer *state, ColiV4IndexerView *view) {+ if (!state || !view) return -1;+ *view = (ColiV4IndexerView){state->compressed, state->count, state->capacity,+ state->config->index_head_dim, state->compressor};+ return 0;+}+int coli_v4_indexer_reserve(ColiDeepSeekV4Indexer *state, int rows) {+ if (!state || rows < 0) return -1;+ if (rows <= state->capacity) return 0;+ int dimension = state->config->index_head_dim, capacity = state->capacity > 0 ? state->capacity : 16;+ while (capacity < rows) capacity *= 2;+ float *grown = realloc(state->compressed, (size_t)capacity * dimension * sizeof(*grown));+ if (!grown) return -1;+ memset(grown + (size_t)state->capacity * dimension, 0,+ (size_t)(capacity - state->capacity) * dimension * sizeof(*grown));+ state->compressed = grown;+ state->capacity = capacity;+ return 0;+}+int coli_v4_indexer_set_count(ColiDeepSeekV4Indexer *state, int count) {+ if (!state || count < 0 || count > state->capacity) return -1;+ state->count = count;+ return 0;+}+#endif#endif /* COLI_V4_UNIT_INDEXER */#ifdef COLI_V4_UNIT_SPARSE_ATTENTION@@ -4280,6 +4350,15 @@ static int moe_fail(const char *format, ...) {va_end(arguments);return -1;}+#ifdef COLI_VULKAN+/* The dense chain (deepseek_v4_chain.h) runs the shared expert on the device and adds+ * it there: while coli_v4_moe_routed runs, the MoE below leaves the shared expert out+ * and returns the routed sum as it stands before that add (no rounding). */+static __thread int v4_moe_routed_only;+#define V4_SHARED_HERE (!v4_moe_routed_only)+#else+#define V4_SHARED_HERE 1+#endif/* ######## deepseek_v4_block_hybrid.c ######## *//* Accepted decode pipeline plus batched causal attention for prompt prefill. *//* ---- begin include deepseek_v4_block_pipeline.c ---- */@@ -5065,7 +5144,7 @@ static int v4_vk_token_finish(V4VkToken *vk, float *output, ColiExpertStore *stofor (int i = 0; !result && i < d; i++) output[i] += redo[i];}}- if (!result)+ if (!result && V4_SHARED_HERE)for (int i = 0; i < d; i++) output[i] = coli_bf16_round(output[i] + shared_output[i]);free(redo); free(dev); free(rows);return result;@@ -5236,7 +5315,7 @@ static int moe_token_pipeline(float *output,fp8_view(&w3, weights, "ffn.shared_experts.w3")))result = moe_fail("layer %d: the shared expert's fp8 tensors are missing",weights->plan.layer);- if (!result) result = coli_v4_shared_expert_forward_ref(+ if (!result && V4_SHARED_HERE) result = coli_v4_shared_expert_forward_ref(shared_output, &w1, &w2, &w3, input, config->swiglu_limit);if (!result) memset(output, 0, (size_t)d * sizeof(*output));@@ -5535,7 +5614,7 @@ static int moe_token_pipeline(float *output,if (!result)for (int i = 0; i < d; i++) output[i] += expert_output[i];}- if (!result)+ if (!result && V4_SHARED_HERE)for (int i = 0; i < d; i++)output[i] = coli_bf16_round(output[i] + shared_output[i]);#if defined(COLI_VULKAN) && defined(COLI_V4_EXPERIMENTAL_DUAL_EXPERT_LOADER)@@ -6095,7 +6174,9 @@ static int v4_moe_batch_union(fp8_view(&w3, weights, "ffn.shared_experts.w3")))result = moe_fail("layer %d: the shared expert's fp8 tensors are missing",weights->plan.layer);- if (!result && batch > 1 && v4_shared_batch_enabled())+ if (!V4_SHARED_HERE) {+ /* the dense chain adds the shared expert itself */+ } else if (!result && batch > 1 && v4_shared_batch_enabled())result = v4_shared_expert_forward_batch_ref(shared, &w1, &w2, &w3, inputs, batch, config->swiglu_limit);else@@ -6196,7 +6277,7 @@ static int v4_moe_batch_union(coli_expert_release(store, &view);}#endif- for (int item = 0; !result && item < batch; item++)+ for (int item = 0; !result && V4_SHARED_HERE && item < batch; item++)for (int column = 0; column < d; column++)outputs[(size_t)item * d + column] = coli_bf16_round(outputs[(size_t)item * d + column] +@@ -6234,6 +6315,31 @@ static double v4_block_prof_now(void) {return ts.tv_sec + ts.tv_nsec / 1e9;}+#ifdef COLI_VULKAN+/* The dense chain's MoE step (deepseek_v4_internal.h): the routes of coli_v4_block_window_+ * batch_ref and of the decode block (one row), without the shared expert. */+int coli_v4_moe_routed(float *routed, const ColiDeepSeekV4LayerWeights *weights,+ const ColiDeepSeekV4Config *config, ColiExpertStore *store,+ const float *inputs, const int *tokens, int rows,+ char *error, size_t error_size) {+ if (!routed || !weights || !config || !store || !inputs || !tokens || rows < 1 || rows > 128)+ return set_error(error, error_size, "invalid chained MoE arguments");+ int d = config->hidden_size, result = 0;+ v4_moe_routed_only = 1;+ if (rows > 1 && v4_expert_union_enabled())+ result = v4_moe_batch_union(routed, weights, config, store, inputs, tokens, rows);+ else+ for (int item = 0; !result && item < rows; item++)+ result = moe_token_pipeline(routed + (size_t)item * d, weights, config, store,+ inputs + (size_t)item * d, tokens[item]);+ v4_moe_routed_only = 0;+ if (!result) return 0;+ if (moe_reason()[0])+ return set_error(error, error_size, "chained block failed in MoE: %s", moe_reason());+ return set_error(error, error_size, "chained block failed in MoE");+}+#endif+int coli_v4_block_window_batch_ref(float *outputs_hc, ColiDeepSeekV4WindowAttentionState *attention,const ColiDeepSeekV4LayerWeights *weights,@@ -12628,6 +12734,13 @@ static void v4_vk_tier_report(const char *scope, unsigned long long hits,}#endif+#ifdef COLI_VULKAN+/* the dense chain (deepseek_v4_chain.h, included below) */+static void v4c_start(const ColiV4Engine *engine);+static void v4c_atexit(void);+static void v4c_close(void);+#endif+static void v4_vk_open(const ColiV4Engine *engine) {#ifdef COLI_VULKANg_v4_vk_thread = pthread_self();@@ -12637,7 +12750,9 @@ static void v4_vk_open(const ColiV4Engine *engine) {int tier = vkt_wanted() && engine->experts && !engine->experts->gpu &&engine->config.n_routed_experts > 0;g_v4_vk_ready = coli_vk_init_env_tier("deepseek_v4", tier);+ v4c_start(engine); /* COLI_VK_CHAIN: the trunk on the device, before the tier sizes itself */v4_vk_tier_start(engine);+ v4c_atexit(); /* after the tier's: the chain goes before the device */if (g_v4_vk_ready && !vkt_ready() && !coli_vk_dense())coli_vk_dense_decide("deepseek_v4", 0, 1); /* no tier after all: the trunk to the device */if (g_v4_vk_ready && coli_vk_dense()) coli_v4_vk_matmul = v4_vk_matmul_impl; /* COLI_VK_DENSE=0: no hook */@@ -12651,6 +12766,7 @@ static void v4_vk_open(const ColiV4Engine *engine) {static void v4_vk_close(void) {#ifdef COLI_VULKANif (!g_v4_vk_ready) return;+ v4c_close(); /* the chain's line, its frames through before the matrices go */if (coli_v4_vk_tier) { /* the tier's run line, then nothing reaches it any more */ColiExpertStoreStats st = {0};ColiExpertStore *store = coli_v4_vk_tier->store;@@ -12733,6 +12849,18 @@ static int head_argmax(ColiV4Engine *engine, const float *hidden,* logprobs=k, docs/brio.md) needs the whole row, so the row is computed here* once and the argmax is a scan over it. Same head_bf16_dot per row, same scan* order: the token picked and its logit do not change. */+#ifdef COLI_VULKAN+/* DUMP=<path> (VK=1 builds): every logits row the head computes, for the Vulkan gates */+static void v4_dump_logits(const float *scores, size_t count) {+ static FILE *dump;+ static int dump_init;+ if (!dump_init) { const char *d = getenv("DUMP"); dump_init = 1; if (d && *d) dump = fopen(d, "wb"); }+ if (dump) { fwrite(scores, sizeof(float), count, dump); fflush(dump); }+}+#define V4_DUMP_LOGITS(scores, count) v4_dump_logits((scores), (count))+#else+#define V4_DUMP_LOGITS(scores, count) ((void)0)+#endifstatic int head_scores_impl(ColiV4Engine *engine, const float *hidden,const ColiSafetensorsIndex *index,const ColiDeepSeekV4Config *config, float *scores) {@@ -12752,14 +12880,17 @@ static int head_scores_impl(ColiV4Engine *engine, const float *hidden,if (resident) {#ifdef COLI_VULKANif (coli_v4_vk_matmul &&- coli_v4_vk_matmul(11, resident, NULL, 0, vocab, d, scores, hidden, 1) == 0)+ coli_v4_vk_matmul(11, resident, NULL, 0, vocab, d, scores, hidden, 1) == 0) {+ V4_DUMP_LOGITS(scores, (size_t)vocab);return 0;+ }#endif#pragma omp parallel for schedule(static)for (int row = 0; row < vocab; row++) {const uint16_t *weight = resident + (size_t)row * d;scores[row] = head_bf16_dot(weight, hidden, d);}+ V4_DUMP_LOGITS(scores, (size_t)vocab);return 0;}/* Low-memory fallback: stream small row tiles exactly as before. */@@ -12783,6 +12914,7 @@ static int head_scores_impl(ColiV4Engine *engine, const float *hidden,}}free(raw);+ V4_DUMP_LOGITS(scores, (size_t)vocab);return 0;}/* First maximum in vocabulary order: the tie-break head_argmax always had. */@@ -12859,6 +12991,7 @@ static int head_argmax_batch(ColiV4Engine *engine, const float *hidden,scores[(size_t)item * vocab + row] = head_bf16_dot(weight, hidden + (size_t)item * d, d);}+ V4_DUMP_LOGITS(scores, (size_t)batch * vocab);for (int item = 0; item < batch; item++) {int winner = -1;float maximum = -FLT_MAX;@@ -13153,6 +13286,10 @@ static const float *v4_mainh_get(int64_t position) {#include "deepseek_v4_dspark.inc"+#ifdef COLI_VULKAN+#include "deepseek_v4_chain.h" /* the layers as a dense chain on the device (COLI_VK_CHAIN) */+#endif+static int v4_prefill_pool_enabled(void) {static int enabled = -1;if (enabled < 0) {@@ -13206,6 +13343,14 @@ static int target_batch_impl(ColiV4Engine *engine, float **state_ptr, float **ne* does not change the caller's semantic distinction: speculative decode* always passes use_prefill_pool=0. */int pool_experts = use_prefill_pool && v4_prefill_pool_enabled();+#ifdef COLI_VULKAN+ { /* the dense chain runs every layer on the device; 0: the CPU runs them, as below */+ int chained = v4c_forward(engine, state_ptr, next_ptr, attention, config, experts, tokens,+ start, batch, pool_experts, error, error_size);+ if (chained) return chained > 0 ? 0 : -1;+ v4c_cpu_step(start);+ }+#endiffor (int layer_id = 0; layer_id < config->num_hidden_layers; layer_id++) {ColiDeepSeekV4LayerWeights layer;if (coli_v4_layer_load(engine, &layer, config, index, layer_id,@@ -13303,6 +13448,14 @@ static int target_token_impl(ColiV4Engine *engine, float **state_ptr, float **nechar *error, size_t error_size) {float *state = *state_ptr, *next = *next_ptr;if (load_embedding(state, index, config, token)) return -1;+#ifdef COLI_VULKAN+ { /* the dense chain, as in target_batch */+ int chained = v4c_forward(engine, state_ptr, next_ptr, attention, config, experts, &token,+ position, 1, 0, error, error_size);+ if (chained) return chained > 0 ? 0 : -1;+ v4c_cpu_step(position);+ }+#endiffor (int layer_id = 0; layer_id < config->num_hidden_layers; layer_id++) {ColiDeepSeekV4LayerWeights layer;if (coli_v4_layer_load(engine, &layer, config, index, layer_id,@@ -14028,6 +14181,9 @@ static int v4_ckpt_restore(ColiV4Session *session, int prompt_count) {fprintf(stderr, "v4_ckpt restore failed layer=%d\n", layer);return 0; /* partial restore is harmless: caller full-resets */}+#ifdef COLI_VULKAN+ v4c_reset(); /* a snapshot from another history: the chain's copies go */+#endifkv_prefix_clear(&session->fed);kv_prefix_record(&session->fed, v4_ckpt_slots[best].ids, 0, best_len);v4_ckpt_slots[best].used = ++v4_ckpt_clock;@@ -14199,6 +14355,9 @@ int coli_v4_session_generate(ColiV4Session *session,#endiffor (int layer = 0; layer < config->num_hidden_layers; layer++)coli_v4_window_attention_reset(session->attention[layer]);+#ifdef COLI_VULKAN+ v4c_reset();+#endifif (coli_v4_full_dspark_wanted) v4_ds_reset_history();/* Plan a capture: the longest common prefix of two successive fresh* prompts IS the stable system prefix. Snapshot there mid-prefill. */@@ -14521,6 +14680,9 @@ int coli_v4_session_generate(ColiV4Session *session,batch, 0, NULL, NULL, error, error_size)) {(void)spec_attention_restore(attention, snapshots, config->num_hidden_layers);+#ifdef COLI_VULKAN+ v4c_restored(old_last + 1);+#endifspec_attention_free(snapshots,config->num_hidden_layers);kv_prefix_taint(&session->fed);@@ -14542,6 +14704,9 @@ int coli_v4_session_generate(ColiV4Session *session,if (!heads_ok) {(void)spec_attention_restore(attention, snapshots, config->num_hidden_layers);+#ifdef COLI_VULKAN+ v4c_restored(old_last + 1);+#endifspec_attention_free(snapshots,config->num_hidden_layers);kv_prefix_taint(&session->fed);@@ -14595,6 +14760,9 @@ int coli_v4_session_generate(ColiV4Session *session,* Restore the exact snapshot and replay only inputs that* really correspond to emitted outputs. */if (retained < batch) {+#ifdef COLI_VULKAN+ v4c_restored(old_last + 1); /* the rows past it rejected */+#endifif (spec_attention_restore(attention, snapshots,config->num_hidden_layers)) {@@ -15378,6 +15546,7 @@ static int v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,#endif#ifdef COLI_VULKANv4_vk_tier_report("turn", (unsigned long long)hits, (unsigned long long)misses);+ v4c_report(); /* the dense chain's line, when it ran */#endifcoli_v4_expert_store_emit_hits(engine->experts);coli_v4_expert_store_emit_emap(engine->experts);@@ -15778,6 +15947,9 @@ int main(int argc, char **argv) {(size_t)generated_count * sizeof(int));for (int layer = 0; layer < config.num_hidden_layers; layer++)coli_v4_window_attention_reset(attention[layer]);+#ifdef COLI_VULKAN+ v4c_reset();+#endif/* Rebuild tf_pred for the fixture (argmax at each position). */size_t hd_tf = (size_t)config.hc_mult * config.hidden_size;tf_state = malloc((size_t)full_count * hd_tf * sizeof(float));diff --git a/c/deepseek_v4_chain.h b/c/deepseek_v4_chain.hnew file mode 100644index 00000000..e59e0ced--- /dev/null+++ b/c/deepseek_v4_chain.h@@ -0,0 +1,872 @@+/* deepseek_v4_chain.h -- DeepSeek V4's layers as a dense chain on the Vulkan device+ * (vk_chain.h). Included once by deepseek_v4.c's GENERATE_STATS unit in a COLI_VULKAN+ * build, after the per-matrix path (v4_vk_*): the chain's matrices are that path's+ * device copies, looked up in the same map, one copy per matrix. target_batch and+ * target_token hand it their forwards (v4c_forward); COLI_VK_CHAIN decides+ * (coli_vk_chain_decide, this engine's integrated GPU default COLI_VK_CHAIN_UNMEASURED:+ * off, not measured on a V4 checkpoint).+ *+ * The residual is hc_mult streams per position (mHC); each site collapses them with the+ * mix it computes itself (hc_pre). DeepSeek V4 rounds to bf16 after nearly every step+ * and rounds the input of every fp8 matrix to E4M3 per 128 (the CPU kernel's activation):+ * the chain makes each of those roundings where the CPU makes it (vkc_dsv4_round), so+ * the device and the CPU differ only by the order of the sums and the device's exp and+ * sqrt, which the roundings mostly absorb (a value that lands on a bf16 boundary can go+ * the other way, and an E4M3 block then rounds on another step: docs/vulkan.md has what+ * that does to the logits). Every matrix takes the per-row GEMV, so a row's bits do not+ * depend on how a forward is cut into chunks, nor on prompt against decode against a+ * verify, as on the CPU. What runs where, per layer, for one block of rows (decode: one+ * row; a prompt in chunks of the CPU's prefill chunk):+ * device, frame A1: the previous layer's FFN branch (the routed sum from the host plus+ * the shared expert, to bf16) written back into the streams (mHC post, bf16); the+ * attention site: hc_attn_fn, the split with Sinkhorn, the collapse, bf16, the+ * RMSNorm, bf16; the attention: the input to E4M3, wq_a, bf16, q_norm, bf16; on a+ * compressed layer the compressor (wkv and wgate in bf16 on the f32 input, the ring+ * with its position bias, the pooled rows to bf16, their norm, bf16, RoPE at the+ * group's first position, the no-position part to E4M3 per 64); on a ratio-4 layer+ * the indexer (its own compressor into the keys: norm, RoPE, the Hadamard transform,+ * E2M1 per 32; the queries: indexer wq_b on the q latent, bf16, RoPE, Hadamard,+ * E2M1; weights_proj; the relu-weighted scores and the top-k in score order); wq_b,+ * bf16, the per-head RMS without weight, bf16; wkv, bf16, kv_norm, bf16; RoPE (YaRN's+ * table on compressed layers) with bf16 on the rotated part; the key's no-position+ * part to E4M3 per 64, bf16; the new rows into the window ring; the sparse attention+ * with the sink over the window and the compressed rows (V4's bf16 weights and+ * output), the inverse RoPE, bf16; wo_a per group (each group's input to E4M3),+ * bf16, wo_b, bf16; mHC post, bf16; the FFN site's split, collapse, norm. Then the+ * frame is waited for.+ * host: the routed experts (coli_v4_moe_routed: the bf16 or the hash router, the+ * expert tier's batch and the CPU's share, summed in the CPU block's order).+ * device, frame A2 (not waited for): the shared expert (the input to E4M3, w1 and w3,+ * V4's SwiGLU between bf16 roundings, the input to E4M3, w2, bf16).+ * The final streams come back to the host, which runs final_hidden and the head as+ * before; DSpark's taps of the last three layers' streams too (v4_mainh_tap).+ *+ * State, and who owns it: the host's stays canonical. What a forward changes there (the+ * window ring, the compressed rows and their count, each compressor's ring, the+ * indexer's keys and count and its compressor's ring) is written back once the+ * forward's last frame is through, as the CPU would have left it, so a lost device, a+ * rejected draft (spec_attention_restore), a prefix checkpoint, a reset or a CPU forward+ * never has to rebuild anything. The device mirrors it: the window ring holds window ++ * chunk rows (no row of a chunk overwrites one an earlier row still reads) behind a+ * watermark win_valid, a row below it valid unless a later position took its slot (a+ * rejected draft longer than a chunk does: slot_pos says); the compressed rows and the+ * indexer's keys of each layer behind kv_valid / ik_valid; each ring behind a flag.+ * Every host write lowers them: a CPU+ * forward (v4c_cpu_step), a restored snapshot (v4c_restored), a reset or a checkpoint+ * restore (v4c_reset), a forward that does not continue where the chain left off, an+ * attention state the mirrors do not describe. A lost device (COLI_VK_CHAIN_FAULT=n+ * fakes it): the forward runs again on the CPU from its input, which the chain leaves+ * untouched until every chunk is through, and the CPU runs from there.+ *+ * The chain declines (the CPU runs, the watermarks follow) when the dense layers are not+ * resident (a low-memory plan reloads them per forward; the --oracle path's own copies+ * too), under the CUDA tier, for prompts only when COLI_VK_CHAIN=2 and the forward has+ * two rows or fewer, for a forward whose compressed rows (window plus every row of a+ * layer without an indexer) pass the attention's 3072 entries, and for a model its+ * shaders do not take: head_dim above 1024, an indexer head that is not a power of two+ * (the Hadamard transform) or above 4096 floats, a top-k above 4096, more than 8+ * streams. */+#include "vk_chain.h"++typedef struct {+ int ratio, idx, kind; /* compress ratio (0 none), an indexer, its list */+ int crows, cproj; /* the compressor's ring: rows and floats a row */+ size_t o_an, o_fn, o_qn, o_kn, o_sink, o_hca, o_hcf, o_cn, o_ape, o_icn, o_iape;+ ColiVkTensor *fna, *fnf, *wq_a, *wq_b, *wkv, *wo_b, *sh1, *sh2, *sh3, **wo_a;+ ColiVkTensor *cwkv, *cwg, *iwkv, *iwg, *iwq, *iwp;+ VkcBuf *win, *ckv, *ring, *ikey, *iring;+ int ccap, kv_valid, ring_ok, icap, ik_valid, iring_ok;+ size_t pw, pc, pr, pk, pir; /* this forward's places in the pull buffer */+ int ns, q1; /* the ring rows it pulls: all (the overlap), or the slots of q1.. */+} V4cLayer;++typedef struct {+ int ok, failed, told;+ ColiV4Engine *engine;+ int L, D, H, HD, nm, hr, nh, hd, rd, QL, og, ol, gw, W, Wd, IH, ID, I, K, rmax, rows, nkind;+ int kratio[COLI_V4_MAX_LAYERS + 1]; /* the ratio of each list kind (0: the indexer's) */+ V4cLayer *ly;+ VkcBuf *prm;+ ColiDeepSeekV4WindowAttentionState **attn; /* the states the mirrors describe */+ int E, win_valid; /* the position after the last chained forward */+ int *slot_pos; /* the position each device ring row holds (-1 none) */+ size_t wcap; /* score columns a row */+ VkcBuf *xs, *xn, *mix, *hpa, *hpf, *col, *nrm, *nq, *qa, *qr, *qrq, *q, *kv, *heads, *grp, *br;+ VkcBuf *craw, *cscr, *icraw, *icscr, *iq, *ihw, *isc, *list, *cs, *sq, *gs, *us, *hs, *ds;+ VkcBuf *h2d, *routed, *xd, *taps, *pull;+ float *host_routed;+ unsigned long long forwards;+ double host_ms;+} V4Chain;++static V4Chain *g_v4c;+static int g_v4c_mode = 0; /* COLI_VK_CHAIN as decided */+static int g_v4c_inited = 0;++/* Rows per chunk: the CPU's prefill chunk (V4_PREFILL_CHUNK, 128 at most), so the host's+ * MoE sees the batches the CPU block would; COLI_VK_CHAIN_ROWS lowers it. */+static int v4c_rows(void) {+ static int rows;+ if (!rows) {+ const char *c = getenv("V4_PREFILL_CHUNK"), *e = getenv("COLI_VK_CHAIN_ROWS");+ int cw = c ? atoi(c) : 128;+ if (cw < 1 || cw > 128) cw = 128;+ int v = e && *e ? atoi(e) : cw;+ rows = v < 1 ? 1 : v > cw ? cw : v;+ }+ return rows;+}+static int v4c_res(VkcBuf **b, size_t floats, int kind) { return vkc_reserve(b, (floats ? floats : 1) * sizeof(float), kind); }+static const void *v4c_val(const ColiDeepSeekV4LayerWeights *w, const char *suffix) {+ char name[COLI_V4_MAX_TENSOR_NAME];+ snprintf(name, sizeof name, "layers.%d.%s", w->plan.layer, suffix);+ return coli_v4_layer_data(w, name, NULL);+}++/* A resident matrix's device copy, the per-matrix path's own (v4_vk_matmul_impl's map+ * and layout): fp8 as fmt 12 (rows unpacked from the rows8 tiles, each 128x128 block+ * scale written out for its rows), bf16 as fmt 11, f32 as fmt 10 (the mHC mixes, which+ * only the chain multiplies). */+static ColiVkTensor *v4c_tensor(int fmt, const void *data, const float *scales, int rows8, int rows, int columns) {+ if (!g_v4_vk_ready || !data || rows < 1 || columns < 1) return NULL;+ if (fmt == 12 && (!scales || columns % 128 || (rows8 && rows % 8))) return NULL;+ V4VkEntry *e = v4_vk_find(data, fmt, rows, columns);+ if (e && e->tensor) return e->tensor;+ if (e && e->refused) return NULL;+ size_t bytes = (size_t)rows * columns * (fmt == 10 ? 4 : fmt == 11 ? 2 : 1);+ if (!v4_vk_resident(data, bytes)) return NULL;+ if (!e && !(e = v4_vk_insert(data, fmt, rows, columns))) return NULL;+ const unsigned char *weights = data;+ unsigned char *unpacked = NULL;+ float *expanded = NULL;+ if (fmt == 12) {+ int groups = columns / 128;+ expanded = malloc((size_t)rows * groups * sizeof(float));+ if (rows8) unpacked = malloc((size_t)rows * columns);+ if (!expanded || (rows8 && !unpacked)) { free(expanded); free(unpacked); return NULL; }+ for (int row = 0; row < rows; row++)+ memcpy(expanded + (size_t)row * groups, scales + (size_t)(row / 128) * groups, (size_t)groups * sizeof(float));+ if (rows8) {+ const unsigned char *packed = data;+ for (int tile = 0; tile < rows / 8; tile++)+ for (int column = 0; column < columns; column++)+ for (int lane = 0; lane < 8; lane++)+ unpacked[((size_t)tile * 8 + lane) * columns + column] = packed[((size_t)tile * columns + column) * 8 + lane];+ weights = unpacked;+ }+ }+ int ok = coli_vk_tensor_ensure(&e->tensor, weights, expanded, fmt, columns, rows, fmt == 12 ? 128 : 0);+ free(unpacked); free(expanded);+ if (!ok && !e->tensor) e->refused = 1;+ return ok ? e->tensor : NULL;+}+/* an fp8 matrix "<prefix>.weight" with its ".scale", a group of rows of it when rows > 0 */+static ColiVkTensor *v4c_fp8(const ColiDeepSeekV4LayerWeights *w, const char *prefix, int group, int rows) {+ char name[COLI_V4_MAX_TENSOR_NAME], sname[COLI_V4_MAX_TENSOR_NAME];+ const ColiDeepSeekV4TensorSpec *ws = NULL;+ snprintf(name, sizeof name, "layers.%d.%s.weight", w->plan.layer, prefix);+ snprintf(sname, sizeof sname, "layers.%d.%s.scale", w->plan.layer, prefix);+ const unsigned char *data = coli_v4_layer_data(w, name, &ws);+ const float *scales = coli_v4_layer_data(w, sname, NULL);+ if (!data || !scales || !ws || ws->rank != 2 || ws->dtype != COLI_ST_F8_E4M3) return NULL;+ int O = (int)ws->shape[0], I = (int)ws->shape[1];+ if (rows <= 0) return v4c_tensor(12, data, scales, ws->packed_rows8, O, I);+ /* the CPU's group view (attention_token_impl): rows group*rows.., its own scale rows */+ int scale_columns = (I + 127) / 128, scale_rows = (rows + 127) / 128;+ return v4c_tensor(12, data + (size_t)group * rows * I, scales + (size_t)group * scale_rows * scale_columns,+ ws->packed_rows8, rows, I);+}+static ColiVkTensor *v4c_plain(const ColiDeepSeekV4LayerWeights *w, const char *suffix, int fmt) {+ char name[COLI_V4_MAX_TENSOR_NAME];+ const ColiDeepSeekV4TensorSpec *ws = NULL;+ snprintf(name, sizeof name, "layers.%d.%s", w->plan.layer, suffix);+ const void *data = coli_v4_layer_data(w, name, &ws);+ if (!data || !ws || ws->rank != 2) return NULL;+ return v4c_tensor(fmt, data, NULL, 0, (int)ws->shape[0], (int)ws->shape[1]);+}++/* ---- the host's writes: the device's copies follow ------------------------------------- */+static void v4c_lower(int pos) {+ V4Chain *ch = g_v4c;+ if (!ch || !ch->ok) return;+ if (pos < 0) pos = 0;+ if (ch->win_valid > pos) ch->win_valid = pos;+ for (int i = 0; i < ch->L; i++) {+ V4cLayer *ly = &ch->ly[i];+ if (ly->ratio > 0 && ly->kv_valid > pos / ly->ratio) ly->kv_valid = pos / ly->ratio;+ if (ly->idx && ly->ik_valid > pos / 4) ly->ik_valid = pos / 4;+ ly->ring_ok = ly->iring_ok = 0;+ }+ ch->E = -1;+}+static void v4c_cpu_step(int start) { v4c_lower(start); } /* a CPU forward from start */+static void v4c_restored(int pos) { v4c_lower(pos); } /* the state restored to pos, the rows before it kept */+static void v4c_reset(void) { v4c_lower(0); } /* a reset, or a state from elsewhere */++/* ---- setup ----------------------------------------------------------------------------- */+static const char *v4c_unsupported(const ColiDeepSeekV4Config *c) {+ int any_idx = 0;+ for (int i = 0; i < c->num_hidden_layers; i++) any_idx |= c->compress_ratios[i] == 4;+ if (c->head_dim > 1024 || c->head_dim < 2 || c->qk_rope_head_dim < 2 || (c->qk_rope_head_dim & 1) ||+ c->qk_rope_head_dim > c->head_dim) return "an attention geometry its shaders do not take";+ if (c->hc_mult < 1 || c->hc_mult > 8 || c->hc_sinkhorn_iters < 1) return "hyper-connections its shaders do not take";+ if (c->o_groups < 1 || c->num_attention_heads % c->o_groups) return "an output grouping it does not take";+ if (c->sliding_window < 1 || c->sliding_window > 3072) return "a window its attention does not take";+ if (any_idx && (c->index_n_heads < 1 || c->index_n_heads > 4096 || c->index_head_dim > 4096 ||+ c->index_head_dim < c->qk_rope_head_dim || (c->index_head_dim & (c->index_head_dim - 1)) ||+ c->index_topk < 1 || c->index_topk > 4096 || c->sliding_window + c->index_topk > 3072))+ return "an indexer its shaders do not take";+ for (int i = 0; i < c->num_hidden_layers; i++)+ if (c->compress_ratios[i] < 0) return "a compression ratio it does not take";+ return NULL;+}++static int v4c_setup(ColiV4Engine *engine) {+ const ColiDeepSeekV4Config *c = &engine->config;+ const char *why = v4c_unsupported(c);+ if (!why && !engine->runtime.dense_resident) why = "the dense layers are not resident (a low-memory plan reloads them per forward)";+ if (!why && (engine->gpu.enabled || (engine->experts && engine->experts->gpu))) why = "the CUDA tier is on and keeps its place";+ if (why) { fprintf(stderr, "[VK] deepseek_v4 chain: %s; the CPU runs the layers\n", why); return 0; }+ int L = c->num_hidden_layers;+ char err[256] = {0};+ const ColiSafetensorsIndex *index = coli_v4_engine_target_index(engine);+ for (int i = 0; i < L; i++) { /* the resident layers, loaded now */+ ColiDeepSeekV4LayerWeights w;+ if (coli_v4_layer_load(engine, &w, c, index, i, err, sizeof err) || !engine->dense_resident.ready[i]) {+ fprintf(stderr, "[VK] deepseek_v4 chain: layer %d did not load (%s); the CPU runs the layers\n", i, err);+ return 0;+ }+ }+ V4Chain *ch = calloc(1, sizeof *ch);+ if (!ch || !(ch->ly = calloc((size_t)L, sizeof *ch->ly))) { free(ch); return 0; }+ g_v4c = ch;+ ch->engine = engine; ch->L = L; ch->D = c->hidden_size; ch->H = c->hc_mult; ch->HD = ch->H * ch->D;+ ch->nm = (2 + ch->H) * ch->H; ch->hr = 2 * ch->H + ch->H * ch->H;+ ch->nh = c->num_attention_heads; ch->hd = c->head_dim; ch->rd = c->qk_rope_head_dim; ch->QL = c->q_lora_rank;+ ch->og = c->o_groups; ch->ol = c->o_lora_rank; ch->gw = (ch->nh / ch->og) * ch->hd;+ ch->W = c->sliding_window; ch->Wd = ch->W + v4c_rows();+ if (!(ch->slot_pos = malloc((size_t)ch->Wd * sizeof(int)))) return 0;+ for (int k = 0; k < ch->Wd; k++) ch->slot_pos[k] = -1;+ ch->IH = c->index_n_heads > 0 ? c->index_n_heads : 1; ch->ID = c->index_head_dim > 0 ? c->index_head_dim : 2;+ ch->I = c->moe_intermediate_size; ch->rmax = 1; ch->E = -1; ch->nkind = 1;+ int D = ch->D, nm = ch->nm;+ for (int i = 0; i < L; i++) {+ V4cLayer *ly = &ch->ly[i];+ ly->ratio = c->compress_ratios[i]; ly->idx = ly->ratio == 4;+ if (ly->ratio > ch->rmax) ch->rmax = ly->ratio;+ ly->crows = ly->ratio == 4 ? 8 : ly->ratio; ly->cproj = (ly->ratio == 4 ? 2 : 1) * ch->hd;+ if (ly->ratio > 0 && !ly->idx) {+ int k = 1;+ while (k < ch->nkind && ch->kratio[k] != ly->ratio) k++;+ if (k == ch->nkind) ch->kratio[ch->nkind++] = ly->ratio;+ ly->kind = k;+ }+ }+ /* the parameter arena: norms decoded from bf16, sinks, hc scales and bases, the rings' biases */+ size_t n = 0;+ for (int i = 0; i < L; i++) {+ V4cLayer *ly = &ch->ly[i];+ ly->o_an = n; n += D; ly->o_fn = n; n += D; ly->o_qn = n; n += ch->QL; ly->o_kn = n; n += ch->hd;+ ly->o_sink = n; n += ch->nh; ly->o_hca = n; n += 3 + nm; ly->o_hcf = n; n += 3 + nm;+ if (ly->ratio > 0) { ly->o_cn = n; n += ch->hd; ly->o_ape = n; n += (size_t)ly->ratio * ly->cproj; }+ if (ly->idx) { ly->o_icn = n; n += ch->ID; ly->o_iape = n; n += (size_t)4 * 2 * ch->ID; }+ }+ float *a = calloc(n ? n : 1, sizeof(float));+ int ok = a != NULL;+ for (int i = 0; ok && i < L; i++) {+ V4cLayer *ly = &ch->ly[i];+ const ColiDeepSeekV4LayerWeights *w = &engine->dense_resident.layers[i];+ const uint16_t *an = v4c_val(w, "attn_norm.weight"), *fn = v4c_val(w, "ffn_norm.weight"),+ *qn = v4c_val(w, "attn.q_norm.weight"), *kn = v4c_val(w, "attn.kv_norm.weight");+ const float *sink = v4c_val(w, "attn.attn_sink"), *hsa = v4c_val(w, "hc_attn_scale"), *hba = v4c_val(w, "hc_attn_base"),+ *hsf = v4c_val(w, "hc_ffn_scale"), *hbf = v4c_val(w, "hc_ffn_base");+ ok = an && fn && qn && kn && sink && hsa && hba && hsf && hbf;+ for (int k = 0; ok && k < D; k++) { a[ly->o_an + k] = coli_bf16_decode(an[k]); a[ly->o_fn + k] = coli_bf16_decode(fn[k]); }+ for (int k = 0; ok && k < ch->QL; k++) a[ly->o_qn + k] = coli_bf16_decode(qn[k]);+ for (int k = 0; ok && k < ch->hd; k++) a[ly->o_kn + k] = coli_bf16_decode(kn[k]);+ if (ok) {+ memcpy(a + ly->o_sink, sink, (size_t)ch->nh * sizeof(float));+ memcpy(a + ly->o_hca, hsa, 3 * sizeof(float)); memcpy(a + ly->o_hca + 3, hba, (size_t)nm * sizeof(float));+ memcpy(a + ly->o_hcf, hsf, 3 * sizeof(float)); memcpy(a + ly->o_hcf + 3, hbf, (size_t)nm * sizeof(float));+ }+ if (ok && ly->ratio > 0) {+ const uint16_t *cn = v4c_val(w, "attn.compressor.norm.weight");+ const float *ape = v4c_val(w, "attn.compressor.ape");+ ok = cn && ape;+ for (int k = 0; ok && k < ch->hd; k++) a[ly->o_cn + k] = coli_bf16_decode(cn[k]);+ if (ok) memcpy(a + ly->o_ape, ape, (size_t)ly->ratio * ly->cproj * sizeof(float));+ }+ if (ok && ly->idx) {+ const uint16_t *icn = v4c_val(w, "attn.indexer.compressor.norm.weight");+ const float *iape = v4c_val(w, "attn.indexer.compressor.ape");+ ok = icn && iape;+ for (int k = 0; ok && k < ch->ID; k++) a[ly->o_icn + k] = coli_bf16_decode(icn[k]);+ if (ok) memcpy(a + ly->o_iape, iape, (size_t)8 * ch->ID * sizeof(float));+ }+ }+ ok = ok && (ch->prm = vkc_buf((n ? n : 1) * sizeof(float), VKC_DEV)) && vkc_begin() &&+ vkc_write(ch->prm, 0, a, n * sizeof(float)) && vkc_submit(1);+ free(a);+ /* the tensors and the per-layer state on the device */+ for (int i = 0; ok && i < L; i++) {+ V4cLayer *ly = &ch->ly[i];+ const ColiDeepSeekV4LayerWeights *w = &engine->dense_resident.layers[i];+ ly->wo_a = calloc((size_t)ch->og, sizeof(void *));+ ok = ly->wo_a && (ly->fna = v4c_plain(w, "hc_attn_fn", 10)) && (ly->fnf = v4c_plain(w, "hc_ffn_fn", 10)) &&+ (ly->wq_a = v4c_fp8(w, "attn.wq_a", 0, 0)) && (ly->wq_b = v4c_fp8(w, "attn.wq_b", 0, 0)) &&+ (ly->wkv = v4c_fp8(w, "attn.wkv", 0, 0)) && (ly->wo_b = v4c_fp8(w, "attn.wo_b", 0, 0)) &&+ (ly->sh1 = v4c_fp8(w, "ffn.shared_experts.w1", 0, 0)) && (ly->sh2 = v4c_fp8(w, "ffn.shared_experts.w2", 0, 0)) &&+ (ly->sh3 = v4c_fp8(w, "ffn.shared_experts.w3", 0, 0));+ for (int g = 0; ok && g < ch->og; g++) ok = (ly->wo_a[g] = v4c_fp8(w, "attn.wo_a", g, ch->ol)) != NULL;+ if (ok && ly->ratio > 0)+ ok = (ly->cwkv = v4c_plain(w, "attn.compressor.wkv.weight", 11)) && (ly->cwg = v4c_plain(w, "attn.compressor.wgate.weight", 11)) &&+ (ly->ring = vkc_buf((size_t)2 * ly->crows * ly->cproj * sizeof(float), VKC_DEV));+ if (ok && ly->idx)+ ok = (ly->iwkv = v4c_plain(w, "attn.indexer.compressor.wkv.weight", 11)) &&+ (ly->iwg = v4c_plain(w, "attn.indexer.compressor.wgate.weight", 11)) &&+ (ly->iwp = v4c_plain(w, "attn.indexer.weights_proj.weight", 11)) && (ly->iwq = v4c_fp8(w, "attn.indexer.wq_b", 0, 0)) &&+ (ly->iring = vkc_buf((size_t)2 * 8 * 2 * ch->ID * sizeof(float), VKC_DEV));+ if (ok) ok = (ly->win = vkc_buf((size_t)ch->Wd * ch->hd * sizeof(float), VKC_DEV)) != NULL;+ }+ if (!ok) { fprintf(stderr, "[VK] deepseek_v4 chain: a matrix or the state did not reach the device; the CPU runs the layers\n"); return 0; }+ ch->ok = 1;+ int ncomp = 0, nidx = 0;+ for (int i = 0; i < L; i++) { ncomp += ch->ly[i].ratio > 0; nidx += ch->ly[i].idx; }+ size_t bytes = 0, tensors = 0;+ coli_vk_mem_info(&bytes, &tensors);+ fprintf(stderr, "[VK] deepseek_v4 chain: %d layers on the device (%d compressing, %d indexing), %d streams, "+ "%.1f MiB of parameters, %zu matrices (%.1f MiB) on the device\n",+ L, ncomp, nidx, ch->H, n * 4 / 1048576.0, tensors, bytes / 1048576.0);+ return 1;+}++/* room on the device for `rows` compressed rows (and keys) of layer i; growing drops them */+static int v4c_cache(V4Chain *ch, V4cLayer *ly, int rows) {+ if (ly->ratio > 0 && ly->ccap < rows) {+ int cap = 64; while (cap < rows) cap *= 2;+ vkc_free(ly->ckv);+ ly->ckv = vkc_buf((size_t)cap * ch->hd * sizeof(float), VKC_DEV);+ ly->kv_valid = 0; ly->ccap = ly->ckv ? cap : 0;+ if (!ly->ckv) return 0;+ }+ if (ly->idx && ly->icap < rows) {+ int cap = 64; while (cap < rows) cap *= 2;+ vkc_free(ly->ikey);+ ly->ikey = vkc_buf((size_t)cap * ch->ID * sizeof(float), VKC_DEV);+ ly->ik_valid = 0; ly->icap = ly->ikey ? cap : 0;+ if (!ly->ikey) return 0;+ }+ return 1;+}++static int v4c_scratch(V4Chain *ch, int rows, int E, int LR) {+ size_t r = (size_t)rows;+ int cp = 2 * ch->hd, taps = coli_v4_full_dspark_wanted ? 3 : 1;+ ch->wcap = (size_t)(E / 4 > 1 ? E / 4 : 1);+ int ok = v4c_res(&ch->xs, r * ch->HD, VKC_DEV) && v4c_res(&ch->xn, r * ch->HD, VKC_DEV) &&+ v4c_res(&ch->mix, r * ch->nm, VKC_DEV) && v4c_res(&ch->hpa, r * ch->hr, VKC_DEV) &&+ v4c_res(&ch->hpf, r * ch->hr, VKC_DEV) && v4c_res(&ch->col, r * ch->D, VKC_DEV) &&+ v4c_res(&ch->nrm, r * ch->D, VKC_DEV) && v4c_res(&ch->nq, r * ch->D, VKC_DEV) &&+ v4c_res(&ch->qa, r * ch->QL, VKC_DEV) && v4c_res(&ch->qr, r * ch->QL, VKC_DEV) &&+ v4c_res(&ch->qrq, r * ch->QL, VKC_DEV) && v4c_res(&ch->q, r * ch->nh * ch->hd, VKC_DEV) &&+ v4c_res(&ch->kv, r * ch->hd, VKC_DEV) && v4c_res(&ch->heads, r * ch->nh * ch->hd, VKC_DEV) &&+ v4c_res(&ch->grp, r * ch->og * ch->ol, VKC_DEV) && v4c_res(&ch->br, r * ch->D, VKC_DEV) &&+ v4c_res(&ch->craw, r * cp, VKC_DEV) && v4c_res(&ch->cscr, r * cp, VKC_DEV) &&+ v4c_res(&ch->icraw, r * 2 * ch->ID, VKC_DEV) && v4c_res(&ch->icscr, r * 2 * ch->ID, VKC_DEV) &&+ v4c_res(&ch->iq, r * ch->IH * ch->ID, VKC_DEV) && v4c_res(&ch->ihw, r * ch->IH, VKC_DEV) &&+ v4c_res(&ch->isc, r * ch->wcap, VKC_DEV) && v4c_res(&ch->list, (size_t)ch->nkind * r * LR, VKC_DEV) &&+ v4c_res(&ch->cs, (2 * r + ch->rmax) * ch->rd, VKC_DEV) && v4c_res(&ch->sq, r * ch->D, VKC_DEV) &&+ v4c_res(&ch->gs, r * ch->I, VKC_DEV) && v4c_res(&ch->us, r * ch->I, VKC_DEV) &&+ v4c_res(&ch->hs, r * ch->I, VKC_DEV) && v4c_res(&ch->ds, r * ch->D, VKC_DEV) &&+ v4c_res(&ch->h2d, r * ch->D, VKC_DOWN) && v4c_res(&ch->routed, r * ch->D, VKC_UP) &&+ v4c_res(&ch->xd, r * ch->HD, VKC_DOWN) && v4c_res(&ch->taps, (size_t)taps * r * ch->HD, VKC_DOWN);+ if (!ok) return 0;+ if (ch->rows < rows) {+ float *hr = realloc(ch->host_routed, r * ch->D * sizeof(float));+ if (!hr) return 0;+ ch->host_routed = hr; ch->rows = rows;+ }+ return 1;+}++/* ---- one layer's pieces ----------------------------------------------------------------- */+static int v4c_round(VkcBuf *x, VkcBuf *y, int kind, int nseg, int per_row, int len, int block, int flags,+ int x_off, int x_row, int x_seg, int y_off, int y_row, int y_seg) {+ VkcDsRound p = {kind, nseg, per_row, len, block, flags, x_off, x_row, x_seg, y_off, y_row, y_seg,+ kind == VKC_DS_HADAMARD ? 1.0f / sqrtf((float)len) : 0.0f};+ return vkc_dsv4_round(x, y, &p);+}+/* n rows of len floats at off, row apart, to bf16 in place */+static int v4c_bf16(VkcBuf *x, int n, int len, int off, int row) {+ return v4c_round(x, x, VKC_DS_BF16, n, 1, len, 0, 0, off, row, 0, off, row, 0);+}+/* n contiguous rows of len floats to E4M3 per 128 (an fp8 matrix's input) */+static int v4c_e4m3(VkcBuf *x, VkcBuf *y, int n, int len) {+ return v4c_round(x, y, VKC_DS_E4M3, n, 1, len, 128, 0, 0, len, 0, 0, len, 0);+}+static int v4c_norm(V4Chain *ch, VkcBuf *x, int xo, int xrow, size_t wo, VkcBuf *y, int yo, int yrow, int rows, int D, int flags) {+ VkcNorm p = {rows, D, 1, xo, xrow, D, yo, yrow, D, (int)wo, 0, flags, ch->engine->config.rms_norm_eps, 1.f};+ return vkc_norm(x, ch->prm, y, &p);+}+/* RoPE on interleaved pairs, the last rd floats of each segment, then bf16 */+static int v4c_rope(V4Chain *ch, VkcBuf *x, int nseg, int per_row, int x_off, int x_row, int x_seg,+ int cs_off, int cs_row, int inverse) {+ VkcDsRope p = {nseg, per_row, ch->rd, x_off, x_row, x_seg, cs_off, cs_row, inverse, 1};+ return vkc_dsv4_rope(x, ch->cs, &p);+}+/* mHC's entry for a site (hc_pre with its own mix): the mix, the split into hp, the+ * collapse, bf16, the norm, bf16, into nrm */+static int v4c_pre(V4Chain *ch, ColiVkTensor *fn, size_t hco, VkcBuf *hp, size_t lno, int n) {+ const ColiDeepSeekV4Config *c = &ch->engine->config;+ int H = ch->H, D = ch->D, HD = ch->HD;+ VkcMhc sp = {n, H, D, c->hc_sinkhorn_iters, 0, HD, 0, ch->nm, 0, ch->hr, 0, D, (int)hco, 0, c->rms_norm_eps, c->hc_eps, 0.f};+ return vkc_matmul(fn, ch->xs, 0, ch->mix, 0, n) && vkc_mhc(VKC_MHC_SPLIT, ch->xs, ch->mix, hp, ch->prm, NULL, &sp) &&+ vkc_mhc(VKC_MHC_COLLAPSE, ch->xs, NULL, hp, NULL, ch->col, &sp) && v4c_bf16(ch->col, n, D, 0, D) &&+ v4c_norm(ch, ch->col, 0, D, lno, ch->nrm, 0, D, n, D, 0) && v4c_bf16(ch->nrm, n, D, 0, D);+}+/* mHC's exit: xn = comb(xs) + post * br, bf16, then the two swap */+static int v4c_post(V4Chain *ch, VkcBuf *hp, int n) {+ const ColiDeepSeekV4Config *c = &ch->engine->config;+ VkcMhc po = {n, ch->H, ch->D, c->hc_sinkhorn_iters, 0, ch->HD, 0, ch->D, 0, ch->hr, 0, ch->HD, 0, 0, c->rms_norm_eps, c->hc_eps, 0.f};+ if (!vkc_mhc(VKC_MHC_POST, ch->xs, ch->br, hp, NULL, ch->xn, &po) || !v4c_bf16(ch->xn, n, ch->HD, 0, ch->HD)) return 0;+ VkcBuf *t = ch->xs; ch->xs = ch->xn; ch->xn = t;+ return 1;+}++/* The forward being recorded: positions, rows, the RoPE tables' places in ch->cs (the+ * window table's row 0 is position pb; the compress table's row 0 is pb - rmax + 1). */+typedef struct { int start, n, pb, nr, csW, csC, LR, s_ident; } V4cFwd;++/* the compressor's ring for nr rows into out (rows of D floats), then the produced rows'+ * pooled-latent pipeline: bf16, norm, bf16, RoPE at the group's first position, and the+ * attention's E4M3 per 64 on the no-position part (or the indexer's Hadamard and E2M1) */+static int v4c_compress(V4Chain *ch, const V4cFwd *f, ColiVkTensor *wkv, ColiVkTensor *wg, VkcBuf *kvb, VkcBuf *scb,+ VkcBuf *ring, size_t ape, VkcBuf *out, int r, int D, size_t norm, int fp4) {+ int nr = f->nr, pb = f->pb, P = r == 4 ? 2 * D : D, g0 = pb / r, np = (pb + nr) / r - g0, rd = ch->rd;+ VkcDsComp cp = {nr, pb, r, P, D, r == 4, 0, P, 0, P, (int)ape, 0, D, 0};+ if (!vkc_matmul(wkv, ch->nrm, 0, kvb, 0, nr) || !vkc_matmul(wg, ch->nrm, 0, scb, 0, nr) ||+ !vkc_dsv4_compress(kvb, scb, ring, ch->prm, out, &cp)) return 0;+ if (np <= 0) return 1;+ int o = g0 * D, cs = f->csC + (g0 * r - (pb - ch->rmax + 1)) * rd;+ if (!v4c_bf16(out, np, D, o, D) || !v4c_norm(ch, out, o, D, norm, out, o, D, np, D, 0) || !v4c_bf16(out, np, D, o, D) ||+ !v4c_rope(ch, out, np, 1, o + D - rd, D, 0, cs, r * rd, 0)) return 0;+ if (!fp4) return v4c_round(out, out, VKC_DS_E4M3, np, 1, D - rd, 64, 1, o, D, 0, o, D, 0);+ return v4c_round(out, out, VKC_DS_HADAMARD, np, 1, D, 0, 0, o, D, 0, o, D, 0) &&+ v4c_round(out, out, VKC_DS_E2M1, np, 1, D, 32, 1, o, D, 0, o, D, 0);+}++/* attention_token_impl for nr rows on the device, into ch->br */+static int v4c_attention(V4Chain *ch, int i, const V4cFwd *f) {+ const ColiDeepSeekV4Config *c = &ch->engine->config;+ V4cLayer *ly = &ch->ly[i];+ int hd = ch->hd, nh = ch->nh, rd = ch->rd, QL = ch->QL, D = ch->D, nr = f->nr, pb = f->pb, qrow = nh * hd;+ int tcs = ly->ratio > 0 ? f->csC + (ch->rmax - 1) * rd : f->csW; /* the table row of position pb */+ int ok = v4c_e4m3(ch->nrm, ch->nq, nr, D) &&+ vkc_matmul(ly->wq_a, ch->nq, 0, ch->qa, 0, nr) && v4c_bf16(ch->qa, nr, QL, 0, QL) &&+ v4c_norm(ch, ch->qa, 0, QL, ly->o_qn, ch->qr, 0, QL, nr, QL, 0) && v4c_bf16(ch->qr, nr, QL, 0, QL) &&+ v4c_e4m3(ch->qr, ch->qrq, nr, QL);+ if (ok && ly->ratio > 0)+ ok = v4c_compress(ch, f, ly->cwkv, ly->cwg, ch->craw, ch->cscr, ly->ring, ly->o_ape, ly->ckv, ly->ratio, hd, ly->o_cn, 0);+ if (ok && ly->idx) { /* the indexer: keys, queries, scores, top-k */+ int IH = ch->IH, ID = ch->ID, width = (pb + nr) / 4, K = c->index_topk;+ ok = v4c_compress(ch, f, ly->iwkv, ly->iwg, ch->icraw, ch->icscr, ly->iring, ly->o_iape, ly->ikey, 4, ID, ly->o_icn, 1) &&+ vkc_matmul(ly->iwq, ch->qrq, 0, ch->iq, 0, nr) && v4c_bf16(ch->iq, nr, IH * ID, 0, IH * ID) &&+ v4c_rope(ch, ch->iq, nr * IH, IH, ID - rd, IH * ID, ID, tcs, rd, 0) &&+ v4c_round(ch->iq, ch->iq, VKC_DS_HADAMARD, nr * IH, IH, ID, 0, 0, 0, IH * ID, ID, 0, IH * ID, ID) &&+ v4c_round(ch->iq, ch->iq, VKC_DS_E2M1, nr * IH, IH, ID, 32, 1, 0, IH * ID, ID, 0, IH * ID, ID) &&+ vkc_matmul(ly->iwp, ch->nrm, 0, ch->ihw, 0, nr);+ if (ok && width > 0) {+ VkcDsScore sp = {nr, pb, 4, IH, ID, width, 0, IH * ID, 0, IH, 0, ID, 0, (int)ch->wcap,+ 1.0f / sqrtf((float)(ID * IH)), 0, 0};+ VkcDsTopk tp = {nr, width, K, (int)ch->wcap, ch->W, f->LR, ch->Wd, 1};+ ok = vkc_dsv4_score(ch->iq, ch->ihw, ly->ikey, NULL, ch->isc, &sp) && vkc_dsv4_topk(ch->isc, ch->list, &tp);+ if (ok && f->s_ident > 0) { /* V4_IDX_IDENTITY: index order while all fit */+ VkcDsTopk ti = tp; ti.S = f->s_ident; ti.order = 0;+ ok = vkc_dsv4_topk(ch->isc, ch->list, &ti);+ }+ }+ }+ ok = ok && vkc_matmul(ly->wq_b, ch->qrq, 0, ch->q, 0, nr) && v4c_bf16(ch->q, nr, qrow, 0, qrow);+ if (ok) { /* the per-head RMS without weight, bf16 */+ VkcNorm p = {nr * nh, hd, nh, 0, qrow, hd, 0, qrow, hd, 0, 0, VKC_NORM_NOW, c->rms_norm_eps, 1.f};+ ok = vkc_norm(ch->q, ch->prm, ch->q, &p) && v4c_bf16(ch->q, nr, qrow, 0, qrow);+ }+ ok = ok && vkc_matmul(ly->wkv, ch->nq, 0, ch->kv, 0, nr) && v4c_bf16(ch->kv, nr, hd, 0, hd) &&+ v4c_norm(ch, ch->kv, 0, hd, ly->o_kn, ch->kv, 0, hd, nr, hd, 0) && v4c_bf16(ch->kv, nr, hd, 0, hd) &&+ v4c_rope(ch, ch->q, nr * nh, nh, hd - rd, qrow, hd, tcs, rd, 0) &&+ v4c_rope(ch, ch->kv, nr, 1, hd - rd, hd, 0, tcs, rd, 0) &&+ v4c_round(ch->kv, ch->kv, VKC_DS_E4M3, nr, 1, hd - rd, 64, 1, 0, hd, 0, 0, hd, 0);+ if (!ok) return 0;+ VkcRegion *rg = malloc((size_t)nr * sizeof *rg); /* the new rows into the window ring */+ if (!rg) return 0;+ for (int s = 0; s < nr; s++) rg[s] = (VkcRegion){(size_t)((pb + s) % ch->Wd) * hd, (size_t)s * hd, (size_t)hd};+ ok = vkc_copy_regions(ly->win, ch->kv, rg, nr);+ free(rg);+ int cnt = ch->W;+ if (ly->idx) cnt += c->index_topk;+ else if (ly->ratio > 0) cnt += (pb + nr) / ly->ratio;+ int kind = ly->ratio > 0 && !ly->idx ? ly->kind : 0;+ VkcDsAttn a = {nr, nh, hd, cnt, kind * nr * f->LR, f->LR, ch->Wd, 0, 0, 0, qrow, 0, qrow, (int)ly->o_sink, 1,+ 1.0f / sqrtf((float)hd)};+ ok = ok && vkc_dsv4_attn(ch->q, ly->win, ly->ratio > 0 ? ly->ckv : NULL, ch->list, ch->prm, ch->heads, &a) &&+ v4c_rope(ch, ch->heads, nr * nh, nh, hd - rd, qrow, hd, tcs, rd, 1) &&+ v4c_round(ch->heads, ch->heads, VKC_DS_E4M3, nr, 1, qrow, 128, 0, 0, qrow, 0, 0, qrow, 0);+ int og = ch->og, ol = ch->ol, gw = ch->gw;+ if (ok && og == 1) ok = vkc_matmul(ly->wo_a[0], ch->heads, 0, ch->grp, 0, nr);+ for (int g = 0; ok && og > 1 && g < og; g++) {+ VkcHgemv h = {0, nr, 1, ol, ol, 0, g * gw, qrow, 0, g * ol, og * ol, 0, 0, 0, 0};+ ok = vkc_mla_hgemv(ly->wo_a[g], ch->heads, ch->grp, NULL, &h);+ }+ return ok && v4c_bf16(ch->grp, nr, og * ol, 0, og * ol) && v4c_e4m3(ch->grp, ch->grp, nr, og * ol) &&+ vkc_matmul(ly->wo_b, ch->grp, 0, ch->br, 0, nr) && v4c_bf16(ch->br, nr, D, 0, D);+}++/* the shared expert: E4M3 input, w1 and w3, V4's SwiGLU, E4M3, w2, bf16 into ds */+static int v4c_shared(V4Chain *ch, V4cLayer *ly, int nr) {+ float lim = ch->engine->config.swiglu_limit;+ VkcDsSwiglu sw = {nr * ch->I, 0, 0, 0, lim > 0.0f ? lim : 0.0f};+ return v4c_e4m3(ch->nrm, ch->sq, nr, ch->D) && vkc_matmul(ly->sh1, ch->sq, 0, ch->gs, 0, nr) &&+ vkc_matmul(ly->sh3, ch->sq, 0, ch->us, 0, nr) && vkc_dsv4_swiglu(ch->gs, ch->us, ch->hs, &sw) &&+ v4c_e4m3(ch->hs, ch->hs, nr, ch->I) && vkc_matmul(ly->sh2, ch->hs, 0, ch->ds, 0, nr) &&+ v4c_bf16(ch->ds, nr, ch->D, 0, ch->D);+}+/* the FFN branch joins: br = bf16(routed + shared), then mHC's exit */+static int v4c_join(V4Chain *ch, int nr) {+ VkcEw add = {VKC_EW_ADD, nr * ch->D, ch->D, 1, 0, 1, 0, 0, 0, 0, 0, 1.f};+ return vkc_ew(ch->br, ch->routed, ch->ds, NULL, NULL, &add) && v4c_bf16(ch->br, nr, ch->D, 0, ch->D) &&+ v4c_post(ch, ch->hpf, nr);+}++static int v4c_read_count(ColiDeepSeekV4WindowAttentionState *s, int *count, int *keys) {+ ColiV4AttentionView v;+ if (coli_v4_attention_view(s, &v)) return 0;+ *count = v.compressed_count;+ *keys = 0;+ if (v.indexer) { ColiV4IndexerView iv; if (coli_v4_indexer_view(v.indexer, &iv)) return 0; *keys = iv.count; }+ return 1;+}++/* Every layer for the n rows of streams in *state_ptr (positions start..), the final+ * streams into *state_ptr (swapped with *next_ptr, as the CPU's layer loop leaves them).+ * 1: done; 0: not taken (the CPU runs the layers, nothing changed); -1: the MoE failed+ * (error set, the host state as before the forward). */+static int v4c_forward(ColiV4Engine *engine, float **state_ptr, float **next_ptr,+ ColiDeepSeekV4WindowAttentionState **attention, const ColiDeepSeekV4Config *config,+ ColiExpertStore *experts, const int *tokens, int start, int n, int pool,+ char *error, size_t error_size) {+ V4Chain *ch = g_v4c;+ if (!g_v4c_mode || !ch || !ch->ok || ch->failed || !engine || engine != ch->engine || !attention || n < 1) return 0;+ if (g_v4c_mode == COLI_VK_CHAIN_PREFILL && n <= 2) return 0;+ if (vkc_lost()) { ch->failed = 1; g_v4c_mode = 0; return 0; }+ const ColiDeepSeekV4Config *c = config;+ int L = ch->L, D = ch->D, HD = ch->HD, hd = ch->hd, rd = ch->rd, W = ch->W, E = start + n;+ int K = 0; /* the compressed slots of a list row */+ for (int i = 0; i < L; i++) {+ V4cLayer *ly = &ch->ly[i];+ int k = ly->idx ? c->index_topk : ly->ratio > 0 ? E / ly->ratio : 0;+ if (k > K) K = k;+ }+ if (W + K > 3072) {+ if (!ch->told) fprintf(stderr, "[VK] deepseek_v4 chain: %d compressed rows at position %d pass the attention's "+ "3072 entries; the CPU runs the layers from here\n", K, E);+ ch->told = 1;+ return 0;+ }+ char err[256] = {0};+ for (int i = 0; i < L; i++) /* the compressors and the indexer exist on the host */+ if (ch->ly[i].ratio > 0 &&+ coli_v4_window_attention_prepare(attention[i], &engine->dense_resident.layers[i], config, err, sizeof err))+ return 0;+ if (attention != ch->attn) { v4c_lower(0); ch->attn = attention; }+ if (start != ch->E) v4c_lower(start);+ for (int i = 0; i < L; i++) { /* the host's state is where the forward starts */+ V4cLayer *ly = &ch->ly[i];+ int cnt, keys;+ if (ly->ratio <= 0) continue;+ if (!v4c_read_count(attention[i], &cnt, &keys) || cnt != start / ly->ratio || (ly->idx && keys != start / 4)) {+ if (!ch->told) fprintf(stderr, "[VK] deepseek_v4 chain: layer %d holds %d compressed rows at position %d; "+ "the CPU runs this forward\n", i, cnt, start);+ ch->told = 1;+ return 0;+ }+ }+ int rows = n < v4c_rows() ? n : v4c_rows(), LR = W + (K > 0 ? K : 1);+ int ok = v4c_scratch(ch, rows, E, LR);+ for (int i = 0; ok && i < L; i++) ok = v4c_cache(ch, &ch->ly[i], E / (ch->ly[i].ratio > 0 ? ch->ly[i].ratio : 1) + 1);+ /* what comes back at the end: the window rows, each layer's new compressed rows and+ * keys, the rings (a ring of ratio r without the overlap: the slots this forward wrote) */+ size_t pn = 0;+ int q0 = E - W > start ? E - W : start, nq = E - q0;+ for (int i = 0; ok && i < L; i++) {+ V4cLayer *ly = &ch->ly[i];+ ly->pw = pn; pn += (size_t)nq * hd;+ if (ly->ratio <= 0) continue;+ int r = ly->ratio;+ ly->pc = pn; pn += (size_t)(E / r - start / r) * hd;+ ly->q1 = E - r > start ? E - r : start; /* without the overlap only the slots written change */+ ly->ns = r == 4 ? ly->crows : E - ly->q1;+ ly->pr = pn; pn += (size_t)2 * ly->ns * ly->cproj;+ if (!ly->idx) continue;+ ly->pk = pn; pn += (size_t)(E / 4 - start / 4) * ch->ID;+ ly->pir = pn; pn += (size_t)2 * 8 * 2 * ch->ID;+ }+ ok = ok && v4c_res(&ch->pull, pn, VKC_DOWN);+ float *cs = malloc((size_t)(2 * rows + ch->rmax) * rd * sizeof(float));+ int *wl = malloc((size_t)ch->nkind * rows * LR * sizeof(int));+ float *cos_ = malloc((size_t)rd / 2 * sizeof(float)), *sin_ = malloc((size_t)rd / 2 * sizeof(float));+ if (!ok || !cs || !wl || !cos_ || !sin_) {+ free(cs); free(wl); free(cos_); free(sin_);+ fprintf(stderr, "[VK] deepseek_v4 chain: device memory for %d rows at %d positions refused; the CPU runs the layers\n", rows, E);+ ch->failed = 1; g_v4c_mode = 0;+ return 0;+ }+ /* every matrix on the per-row GEMV: a row's bits do not depend on the chunk it sits+ * in (a prompt, a verify, a decode step, a prefix reused or computed cold), as on the+ * CPU, whose batched and per-token kernels give the same bits */+ vkc_gemm_rows(0);+ static int ident = -1;+ if (ident < 0) { const char *e = getenv("V4_IDX_IDENTITY"); ident = e && *e == '1'; }+ float *state = *state_ptr, *next = *next_ptr;+ int T0 = L - 3, taps = coli_v4_full_dspark_wanted; /* DSpark's targets: the last three layers */+ for (int c0 = 0; c0 < n; c0 += rows) {+ int nr = n - c0 < rows ? n - c0 : rows, pb = start + c0, last = c0 + nr == n;+ V4cFwd f = {start, n, pb, nr, 0, nr * rd, LR, 0};+ if (ident) for (int s = 0; s < nr && (pb + s + 1) / 4 <= c->index_topk; s++) f.s_ident = s + 1;+ /* this chunk's RoPE rows, the CPU's own tables: the window table (the plain base)+ * from position pb, the compress table (YaRN, compress_rope_theta) from pb - rmax + 1 */+ for (int s = 0; s < nr + ch->rmax - 1; s++) {+ for (int t = 0; t < 2; t++) {+ if (t == 0 && s >= nr) continue;+ int p = t == 0 ? pb + s : pb - ch->rmax + 1 + s;+ float *dst = cs + (t == 0 ? f.csW : f.csC) + (size_t)s * rd;+ if (p < 0 || coli_v4_rope_position(cos_, sin_, rd, p, t ? c->original_max_position_embeddings : 0,+ t ? c->compress_rope_theta : c->rope_theta, c->rope_factor,+ c->rope_beta_fast, c->rope_beta_slow)) {+ memset(dst, 0, (size_t)rd * sizeof(float));+ continue;+ }+ for (int k = 0; k < rd / 2; k++) { dst[2 * k] = cos_[k]; dst[2 * k + 1] = sin_[k]; }+ }+ }+ /* each list kind's rows: the window (the last W positions a row reaches, oldest+ * first, as device ring rows), then the compressed rows: the indexer's top-k slots+ * (written on the device), or every row a layer of that ratio has made, in order */+ for (int k = 0; k < ch->nkind; k++)+ for (int s = 0; s < nr; s++) {+ int p = pb + s, *row = wl + ((size_t)k * nr + s) * LR;+ for (int j = 0; j < W; j++) { int q = p - W + 1 + j; row[j] = q >= 0 ? q % ch->Wd : -1; }+ for (int j = W; j < LR; j++) {+ int g = j - W;+ row[j] = k > 0 && g < (p + 1) / ch->kratio[k] ? ch->Wd + g : -1;+ }+ }+ if (!vkc_begin() || !vkc_write(ch->xs, 0, state + (size_t)c0 * HD, (size_t)nr * HD * sizeof(float)) ||+ !vkc_write(ch->list, 0, wl, (size_t)ch->nkind * nr * LR * sizeof(int)) ||+ !vkc_write(ch->cs, 0, cs, (size_t)(f.csC + (nr + ch->rmax - 1) * rd) * sizeof(float))) goto lost;+ if (c0 == 0) { /* the device's copies made the host's, as the forward needs them */+ int q_lo = start - W + 1 < 0 ? 0 : start - W + 1;+ for (int i = 0; ok && i < L; i++) {+ V4cLayer *ly = &ch->ly[i];+ ColiV4AttentionView v;+ if (coli_v4_attention_view(attention[i], &v)) goto lost;+ for (int q = q_lo; ok && q < start; q++)+ if (q >= ch->win_valid || ch->slot_pos[q % ch->Wd] != q) /* stale, or its row taken since */+ ok = vkc_write(ly->win, (size_t)(q % ch->Wd) * hd, v.kv + (size_t)(q % W) * hd, (size_t)hd * sizeof(float));+ if (ly->ratio <= 0) continue;+ int t0 = ly->kv_valid, t1 = start / ly->ratio;+ if (ok && t1 > t0) ok = vkc_write(ly->ckv, (size_t)t0 * hd, v.compressed + (size_t)t0 * hd, (size_t)(t1 - t0) * hd * sizeof(float));+ ColiV4CompressorView cv;+ size_t rf = (size_t)ly->crows * ly->cproj;+ if (ok && !ly->ring_ok)+ ok = !coli_v4_compressor_view(v.compressor, &cv) && cv.rows == ly->crows && cv.projection == ly->cproj &&+ vkc_write(ly->ring, 0, cv.kv, rf * sizeof(float)) && vkc_write(ly->ring, rf, cv.score, rf * sizeof(float));+ if (!ly->idx) continue;+ ColiV4IndexerView iv;+ ok = ok && !coli_v4_indexer_view(v.indexer, &iv);+ int k0 = ly->ik_valid, k1 = start / 4;+ if (ok && k1 > k0) ok = vkc_write(ly->ikey, (size_t)k0 * ch->ID, iv.keys + (size_t)k0 * ch->ID, (size_t)(k1 - k0) * ch->ID * sizeof(float));+ size_t ir = (size_t)8 * 2 * ch->ID;+ if (ok && !ly->iring_ok)+ ok = !coli_v4_compressor_view(iv.compressor, &cv) && cv.rows == 8 && cv.projection == 2 * ch->ID &&+ vkc_write(ly->iring, 0, cv.kv, ir * sizeof(float)) && vkc_write(ly->iring, ir, cv.score, ir * sizeof(float));+ }+ if (!ok) goto lost;+ for (int q = q_lo; q < start; q++) ch->slot_pos[q % ch->Wd] = q;+ }+ for (int s = 0; s < nr; s++) ch->slot_pos[(pb + s) % ch->Wd] = pb + s; /* this chunk's rows go in below */+ int pending = 0;+ for (int i = 0; i < L && ok; i++) {+ V4cLayer *ly = &ch->ly[i];+ if (pending) { /* the FFN branch of the layer before */+ ok = v4c_join(ch, nr);+ if (ok && taps && i - 1 >= T0) ok = vkc_copy(ch->taps, (size_t)(i - 1 - T0) * nr * HD, ch->xs, 0, (size_t)nr * HD);+ pending = 0;+ }+ ok = ok && v4c_pre(ch, ly->fna, ly->o_hca, ch->hpa, ly->o_an, nr) && v4c_attention(ch, i, &f) &&+ v4c_post(ch, ch->hpa, nr) && v4c_pre(ch, ly->fnf, ly->o_hcf, ch->hpf, ly->o_fn, nr) &&+ vkc_copy(ch->h2d, 0, ch->nrm, 0, (size_t)nr * D) && vkc_submit(1); /* A1 */+ if (!ok) break;+ /* A2: the shared expert, while the host computes the routed experts */+ ok = vkc_begin() && v4c_shared(ch, ly, nr) && vkc_submit(0);+ if (!ok) break;+ double t1 = spec_now();+ if (pool) coli_v4_expert_store_prefill_pool(experts, i);+ if (coli_v4_moe_routed(ch->host_routed, &engine->dense_resident.layers[i], config, experts,+ (const float *)vkc_ptr(ch->h2d), tokens + c0, nr, error, error_size)) {+ if (pool) coli_v4_expert_store_prefill_pool(experts, -1);+ vkc_finish();+ for (int k = 0; k < ch->Wd; k++) ch->slot_pos[k] = -1; /* some layers wrote this chunk's rows */+ v4c_lower(start);+ free(cs); free(wl); free(cos_); free(sin_);+ return -1;+ }+ memcpy(vkc_ptr(ch->routed), ch->host_routed, (size_t)nr * D * sizeof(float));+ ch->host_ms += (spec_now() - t1) * 1e3;+ ok = vkc_begin();+ pending = 1;+ }+ if (ok && pending) {+ ok = v4c_join(ch, nr);+ if (ok && taps && L - 1 >= T0) ok = vkc_copy(ch->taps, (size_t)(L - 1 - T0) * nr * HD, ch->xs, 0, (size_t)nr * HD);+ }+ ok = ok && vkc_copy(ch->xd, 0, ch->xs, 0, (size_t)nr * HD);+ if (ok && last) { /* what the host's state gets back */+ for (int i = 0; i < L && ok; i++) {+ V4cLayer *ly = &ch->ly[i];+ VkcRegion *rg = malloc((size_t)(nq > 0 ? nq : 1) * sizeof *rg);+ if (!rg) { ok = 0; break; }+ for (int q = q0; q < E; q++) rg[q - q0] = (VkcRegion){ly->pw + (size_t)(q - q0) * hd, (size_t)(q % ch->Wd) * hd, (size_t)hd};+ ok = vkc_copy_regions(ch->pull, ly->win, rg, nq);+ free(rg);+ if (!ok || ly->ratio <= 0) continue;+ int r = ly->ratio, g0 = start / r, np = E / r - g0;+ size_t rf = (size_t)ly->crows * ly->cproj, cp = (size_t)ly->cproj;+ ok = vkc_copy(ch->pull, ly->pc, ly->ckv, (size_t)g0 * hd, (size_t)np * hd);+ if (ok && r == 4) ok = vkc_copy(ch->pull, ly->pr, ly->ring, 0, 2 * rf);+ else if (ok) {+ VkcRegion *rr = malloc((size_t)2 * ly->ns * sizeof *rr);+ if (!rr) { ok = 0; break; }+ for (int j = 0; j < ly->ns; j++) {+ size_t slot = (size_t)((ly->q1 + j) % r);+ rr[2 * j] = (VkcRegion){ly->pr + j * cp, slot * cp, cp};+ rr[2 * j + 1] = (VkcRegion){ly->pr + ((size_t)ly->ns + j) * cp, rf + slot * cp, cp};+ }+ ok = vkc_copy_regions(ch->pull, ly->ring, rr, 2 * ly->ns);+ free(rr);+ }+ if (!ok || !ly->idx) continue;+ int k0 = start / 4, nk = E / 4 - k0;+ ok = vkc_copy(ch->pull, ly->pk, ly->ikey, (size_t)k0 * ch->ID, (size_t)nk * ch->ID) &&+ vkc_copy(ch->pull, ly->pir, ly->iring, 0, (size_t)2 * 8 * 2 * ch->ID);+ }+ }+ ok = ok && vkc_submit(1);+ if (!ok) goto lost;+ memcpy(next + (size_t)c0 * HD, vkc_ptr(ch->xd), (size_t)nr * HD * sizeof(float));+ if (taps) {+ const float *tp = (const float *)vkc_ptr(ch->taps);+ for (int t = T0 < 0 ? 0 : T0; t < L; t++)+ for (int s = 0; s < nr; s++)+ v4_mainh_tap(c, t, tp + ((size_t)(t - T0) * nr + s) * HD, (int64_t)pb + s);+ }+ }+ if (pool) coli_v4_expert_store_prefill_pool(experts, -1);+ /* every chunk is through: the host's state as the CPU would have left it */+ {+ const float *pl = (const float *)vkc_ptr(ch->pull);+ for (int i = 0; i < L; i++) {+ V4cLayer *ly = &ch->ly[i];+ ColiV4AttentionView v;+ coli_v4_attention_view(attention[i], &v);+ for (int q = q0; q < E; q++) memcpy(v.kv + (size_t)(q % W) * hd, pl + ly->pw + (size_t)(q - q0) * hd, (size_t)hd * sizeof(float));+ if (ly->ratio <= 0) continue;+ int r = ly->ratio, g0 = start / r, np = E / r - g0;+ if (coli_v4_attention_reserve(attention[i], E / r)) goto host_oom;+ coli_v4_attention_view(attention[i], &v);+ memcpy(v.compressed + (size_t)g0 * hd, pl + ly->pc, (size_t)np * hd * sizeof(float));+ coli_v4_attention_set_count(attention[i], E / r);+ ColiV4CompressorView cv;+ coli_v4_compressor_view(v.compressor, &cv);+ size_t rf = (size_t)ly->crows * ly->cproj, cp = (size_t)ly->cproj;+ if (r == 4) {+ memcpy(cv.kv, pl + ly->pr, rf * sizeof(float));+ memcpy(cv.score, pl + ly->pr + rf, rf * sizeof(float));+ } else for (int j = 0; j < ly->ns; j++) {+ size_t slot = (size_t)((ly->q1 + j) % r);+ memcpy(cv.kv + slot * cp, pl + ly->pr + j * cp, cp * sizeof(float));+ memcpy(cv.score + slot * cp, pl + ly->pr + ((size_t)ly->ns + j) * cp, cp * sizeof(float));+ }+ if (!ly->idx) continue;+ ColiV4IndexerView iv;+ int k0 = start / 4, nk = E / 4 - k0;+ if (coli_v4_indexer_reserve(v.indexer, E / 4)) goto host_oom;+ coli_v4_indexer_view(v.indexer, &iv);+ memcpy(iv.keys + (size_t)k0 * ch->ID, pl + ly->pk, (size_t)nk * ch->ID * sizeof(float));+ coli_v4_indexer_set_count(v.indexer, E / 4);+ coli_v4_compressor_view(iv.compressor, &cv);+ size_t ir = (size_t)8 * 2 * ch->ID;+ memcpy(cv.kv, pl + ly->pir, ir * sizeof(float));+ memcpy(cv.score, pl + ly->pir + ir, ir * sizeof(float));+ }+ ch->win_valid = E; ch->E = E;+ for (int i = 0; i < L; i++) {+ V4cLayer *ly = &ch->ly[i];+ if (ly->ratio > 0) { ly->kv_valid = E / ly->ratio; ly->ring_ok = 1; }+ if (ly->idx) { ly->ik_valid = E / 4; ly->iring_ok = 1; }+ }+ }+ *state_ptr = next; *next_ptr = state;+ free(cs); free(wl); free(cos_); free(sin_);+ ch->forwards++;+ return 1;+host_oom: /* the host state half written: as a failed CPU forward leaves it, an error */+ free(cs); free(wl); free(cos_); free(sin_);+ v4c_lower(0);+ if (error && error_size) snprintf(error, error_size, "out of memory writing the chained forward's state back");+ return -1;+lost: /* a frame failed: the device is gone (or would not take a command); the CPU takes over */+ free(cs); free(wl); free(cos_); free(sin_);+ if (pool) coli_v4_expert_store_prefill_pool(experts, -1);+ if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost(); }+ g_v4c_mode = 0; ch->failed = 1;+ v4c_lower(0);+ fprintf(stderr, "[VK] deepseek_v4 chain: the device was lost; the CPU runs this forward again and from here on "+ "(the host's state is current: there is nothing to rebuild)\n");+ return 0;+}++static void v4c_report(void) {+ V4Chain *ch = g_v4c;+ if (!ch || !ch->ok || !ch->forwards) return;+ VkcStats st; vkc_stats(&st);+ fprintf(stderr, "[VK] deepseek_v4 chain: %llu forwards, %llu frames (%llu ops, %llu matmuls, %llu tiled GEMM), "+ "%.1f ms waiting for the device, %.1f ms of routed experts on the host, %.1f MiB on the device\n",+ ch->forwards, st.frames, st.ops, st.matmuls, st.gemms, st.wait_ms, ch->host_ms, st.dev_bytes / 1048576.0);+ vkc_prof_print();+}++/* COLI_VK_CHAIN at startup, before the tier sizes itself (the trunk's device copies count+ * as used): the decision, the pipelines, the resident layers and their tensors. */+static void v4c_start(const ColiV4Engine *cengine) {+ ColiV4Engine *engine = (ColiV4Engine *)cengine;+ if (!g_v4_vk_ready || !engine) return;+ int tier_on = vkt_wanted() && engine->experts && !engine->experts->gpu && engine->config.n_routed_experts > 0;+ int on = coli_vk_chain_decide("deepseek_v4", tier_on, COLI_VK_CHAIN_UNMEASURED);+ const char *no = NULL;+ if (on && !(g_v4c_inited = vkc_init())) no = "the chain's pipelines did not come up";+ if (on && !no && !(vkc_mla_ready() && vkc_mhc_ready() && vkc_dsv4_ready()))+ no = "the MLA, mHC or DeepSeek shaders are missing (chain_hgemv, chain_mhc, chain_dsv4)";+ if (no) fprintf(stderr, "[VK] deepseek_v4: %s: the dense chain stays off\n", no);+ if (!on || no) return;+ if (!v4c_setup(engine)) return;+ g_v4c_mode = on;+}+/* After the tier's: at exit the chain goes before the device. */+static void v4c_atexit(void) {+ if (g_v4c_inited) atexit(vkc_shutdown);+}+/* v4_vk_close, before the matrices go: the line, then every frame through. */+static void v4c_close(void) {+ v4c_report();+ if (g_v4c_inited && !vkc_lost()) vkc_finish();+}diff --git a/c/deepseek_v4_internal.h b/c/deepseek_v4_internal.hindex b9111bad..fbe5b428 100644--- a/c/deepseek_v4_internal.h+++ b/c/deepseek_v4_internal.h@@ -664,6 +664,43 @@ uint64_t coli_v4_expert_store_record_bytes(ColiExpertStore *store);int coli_v4_expert_store_read_private(ColiExpertStore *store, ColiExpertKey key,unsigned char *buffer, ColiExpertView *view);extern int (*coli_v4_expert_store_device_tier)(int layer, int expert);++/* The dense chain (deepseek_v4_chain.h, COLI_VK_CHAIN): a layer's attention state as its+ * units keep it, so the chain can mirror it on the device and write back what a forward+ * changed, as the CPU would have left it. The views point into the live state.+ * attention the window ring [window][head_dim] (position % window), the compressed+ * rows [capacity][head_dim] and their count, the compressor and the+ * indexer (NULL on a layer without them, or before the layer first ran:+ * coli_v4_window_attention_prepare makes them);+ * compressor its ring: kv and score rows [rows][projection] (rows = ratio, twice that+ * with the overlap of ratio 4; projection = head_dim, twice that likewise);+ * indexer its keys [capacity][head_dim] and their count, and its compressor.+ * reserve makes room for `rows` rows (the contents kept, the count untouched); set_count+ * sets the count (at most the capacity). */+typedef struct {+ float *kv, *compressed;+ int window, head_dim, ratio, compressed_count, compressed_capacity;+ ColiDeepSeekV4CompressorState *compressor;+ ColiDeepSeekV4Indexer *indexer;+} ColiV4AttentionView;+int coli_v4_attention_view(ColiDeepSeekV4WindowAttentionState *state, ColiV4AttentionView *view);+int coli_v4_attention_reserve(ColiDeepSeekV4WindowAttentionState *state, int rows);+int coli_v4_attention_set_count(ColiDeepSeekV4WindowAttentionState *state, int count);+typedef struct { float *kv, *score; int rows, projection, ratio, head_dim, rotate_fp4; } ColiV4CompressorView;+int coli_v4_compressor_view(ColiDeepSeekV4CompressorState *state, ColiV4CompressorView *view);+typedef struct { float *keys; int count, capacity, head_dim; ColiDeepSeekV4CompressorState *compressor; } ColiV4IndexerView;+int coli_v4_indexer_view(ColiDeepSeekV4Indexer *state, ColiV4IndexerView *view);+int coli_v4_indexer_reserve(ColiDeepSeekV4Indexer *state, int rows);+int coli_v4_indexer_set_count(ColiDeepSeekV4Indexer *state, int count);+/* The MoE of `rows` normalized FFN rows without the shared expert: each row's routed+ * experts summed in the CPU block's order (the store, the Vulkan tier), the sum left as+ * it stands before the CPU adds the shared expert and rounds; the chain adds the shared+ * expert it ran on the device. tokens: the rows' ids (the hash router). 0, or -1 with+ * the reason in error. */+int coli_v4_moe_routed(float *routed, const ColiDeepSeekV4LayerWeights *weights,+ const ColiDeepSeekV4Config *config, ColiExpertStore *store,+ const float *inputs, const int *tokens, int rows,+ char *error, size_t error_size);#endif#ifdef __cplusplus