Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

v4: forward-port DSpark drafting and verification

DrewZ433a2f91e5a2
diff --git a/c/deepseek_v4_dspark.c b/c/deepseek_v4_dspark.c
new file mode 100644
index 00000000..3c0a3f80
--- /dev/null
+++ b/c/deepseek_v4_dspark.c
@@ -0,0 +1,4027 @@
+/* Amalgamated deepseek_v4_dspark.c — GLM-style source; compile with -DCOLI_V4_UNIT_* per object */
+/* Umbrella API: deepseek_v4_dspark.h (included by units) */
+
+#ifdef COLI_V4_UNIT_CONFIG_DSPARK_COMPAT
+/* ######## deepseek_v4_config_dspark_compat.c ######## */
+#include "deepseek_v4_dspark.h"
+/* ---- begin inlined deepseek_v4_config_dspark_compat.c ---- */
+#define coli_v4_config_parse coli_v4_config_parse_strict
+#define coli_v4_config_load coli_v4_config_load_strict
+/* ---- begin include deepseek_v4_config.c ---- */
+#include "deepseek_v4_internal.h"
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "json.h"
+
+static int set_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list arguments;
+ va_start(arguments, format);
+ vsnprintf(error, size, format, arguments);
+ va_end(arguments);
+ }
+ return -1;
+}
+
+static int required_int(jval *root, const char *name, int *output,
+ char *error, size_t error_size) {
+ jval *value = json_get(root, name);
+ if (!value || value->t != J_NUM)
+ return set_error(error, error_size, "missing integer config field: %s", name);
+ *output = (int)value->num;
+ return 0;
+}
+
+static int required_float(jval *root, const char *name, float *output,
+ char *error, size_t error_size) {
+ jval *value = json_get(root, name);
+ if (!value || value->t != J_NUM)
+ return set_error(error, error_size, "missing numeric config field: %s", name);
+ *output = (float)value->num;
+ return 0;
+}
+
+static int require_string(jval *root, const char *name, const char *expected,
+ char *error, size_t error_size) {
+ jval *value = json_get(root, name);
+ if (!value || value->t != J_STR || strcmp(value->str, expected))
+ return set_error(error, error_size, "unsupported %s (expected %s)",
+ name, expected);
+ return 0;
+}
+
+int coli_v4_config_parse(ColiDeepSeekV4Config *config, const char *json,
+ char *error, size_t error_size) {
+ if (!config || !json)
+ return set_error(error, error_size, "invalid DeepSeek-V4 config arguments");
+ memset(config, 0, sizeof(*config));
+ char *arena = NULL;
+ jval *root = json_parse(json, &arena);
+ if (!root || root->t != J_OBJ) {
+ free(arena);
+ return set_error(error, error_size, "DeepSeek-V4 config is not an object");
+ }
+ int failed =
+ require_string(root, "model_type", "deepseek_v4", error, error_size) ||
+ require_string(root, "expert_dtype", "fp4", error, error_size) ||
+ require_string(root, "scoring_func", "sqrtsoftplus", error, error_size) ||
+ require_string(root, "topk_method", "noaux_tc", error, error_size) ||
+ required_int(root, "hidden_size", &config->hidden_size, error, error_size) ||
+ required_int(root, "num_hidden_layers", &config->num_hidden_layers, error, error_size) ||
+ required_int(root, "num_attention_heads", &config->num_attention_heads, error, error_size) ||
+ required_int(root, "head_dim", &config->head_dim, error, error_size) ||
+ required_int(root, "q_lora_rank", &config->q_lora_rank, error, error_size) ||
+ required_int(root, "qk_rope_head_dim", &config->qk_rope_head_dim, error, error_size) ||
+ required_int(root, "o_groups", &config->o_groups, error, error_size) ||
+ required_int(root, "o_lora_rank", &config->o_lora_rank, error, error_size) ||
+ required_int(root, "sliding_window", &config->sliding_window, error, error_size) ||
+ required_int(root, "index_n_heads", &config->index_n_heads, error, error_size) ||
+ required_int(root, "index_head_dim", &config->index_head_dim, error, error_size) ||
+ required_int(root, "index_topk", &config->index_topk, error, error_size) ||
+ required_int(root, "n_routed_experts", &config->n_routed_experts, error, error_size) ||
+ required_int(root, "num_experts_per_tok", &config->num_experts_per_tok, error, error_size) ||
+ required_int(root, "n_shared_experts", &config->n_shared_experts, error, error_size) ||
+ required_int(root, "moe_intermediate_size", &config->moe_intermediate_size, error, error_size) ||
+ required_int(root, "num_hash_layers", &config->num_hash_layers, error, error_size) ||
+ required_int(root, "num_nextn_predict_layers", &config->num_nextn_predict_layers, error, error_size) ||
+ required_int(root, "hc_mult", &config->hc_mult, error, error_size) ||
+ required_int(root, "hc_sinkhorn_iters", &config->hc_sinkhorn_iters, error, error_size) ||
+ required_int(root, "vocab_size", &config->vocab_size, error, error_size) ||
+ required_int(root, "max_position_embeddings", &config->max_position_embeddings, error, error_size) ||
+ required_float(root, "rms_norm_eps", &config->rms_norm_eps, error, error_size) ||
+ required_float(root, "hc_eps", &config->hc_eps, error, error_size) ||
+ required_float(root, "routed_scaling_factor", &config->routed_scaling_factor, error, error_size) ||
+ required_float(root, "swiglu_limit", &config->swiglu_limit, error, error_size) ||
+ required_float(root, "rope_theta", &config->rope_theta, error, error_size) ||
+ required_float(root, "compress_rope_theta", &config->compress_rope_theta, error, error_size);
+ if (failed) {
+ free(arena);
+ return -1;
+ }
+ jval *rope = json_get(root, "rope_scaling");
+ if (!rope || rope->t != J_OBJ ||
+ required_int(rope, "original_max_position_embeddings",
+ &config->original_max_position_embeddings, error, error_size) ||
+ required_int(rope, "beta_fast", &config->rope_beta_fast, error, error_size) ||
+ required_int(rope, "beta_slow", &config->rope_beta_slow, error, error_size) ||
+ required_float(rope, "factor", &config->rope_factor, error, error_size)) {
+ free(arena);
+ return -1;
+ }
+ jval *ratios = json_get(root, "compress_ratios");
+ if (!ratios || ratios->t != J_ARR || ratios->len < 1 ||
+ ratios->len > COLI_V4_MAX_LAYERS) {
+ free(arena);
+ return set_error(error, error_size, "invalid compress_ratios");
+ }
+ config->compress_ratio_count = ratios->len;
+ for (int index = 0; index < ratios->len; index++) {
+ if (ratios->kids[index]->t != J_NUM) {
+ free(arena);
+ return set_error(error, error_size, "non-numeric compress ratio");
+ }
+ config->compress_ratios[index] = (int)ratios->kids[index]->num;
+ }
+ jval *quantization = json_get(root, "quantization_config");
+ if (!quantization || quantization->t != J_OBJ ||
+ require_string(quantization, "fmt", "e4m3", error, error_size) ||
+ require_string(quantization, "scale_fmt", "ue8m0", error, error_size)) {
+ free(arena);
+ return -1;
+ }
+ if (config->hidden_size < 1 || config->num_hidden_layers < 1 ||
+ config->num_attention_heads < 1 || config->n_routed_experts < 1 ||
+ config->num_experts_per_tok < 1 ||
+ config->num_experts_per_tok > config->n_routed_experts ||
+ config->n_shared_experts != 1 || config->hc_mult < 1 ||
+ config->compress_ratio_count != config->num_hidden_layers +
+ config->num_nextn_predict_layers) {
+ free(arena);
+ return set_error(error, error_size, "inconsistent DeepSeek-V4 config dimensions");
+ }
+ free(arena);
+ return 0;
+}
+
+int coli_v4_config_load(ColiDeepSeekV4Config *config, const char *model_dir,
+ char *error, size_t error_size) {
+ if (!config || !model_dir)
+ return set_error(error, error_size, "invalid DeepSeek-V4 config path");
+ size_t path_length = strlen(model_dir) + sizeof("/config.json");
+ char *path = malloc(path_length);
+ if (!path) return set_error(error, error_size, "out of memory building config path");
+ snprintf(path, path_length, "%s/config.json", model_dir);
+ FILE *stream = fopen(path, "rb");
+ if (!stream) {
+ int result = set_error(error, error_size, "cannot open %s", path);
+ free(path);
+ return result;
+ }
+ fseek(stream, 0, SEEK_END);
+ long length = ftell(stream);
+ rewind(stream);
+ if (length < 1) {
+ fclose(stream);
+ free(path);
+ return set_error(error, error_size, "empty config: %s", model_dir);
+ }
+ char *text = malloc((size_t)length + 1);
+ if (!text || fread(text, 1, (size_t)length, stream) != (size_t)length) {
+ free(text);
+ fclose(stream);
+ free(path);
+ return set_error(error, error_size, "cannot read config: %s", model_dir);
+ }
+ text[length] = 0;
+ fclose(stream);
+ int result = coli_v4_config_parse(config, text, error, error_size);
+ free(text);
+ free(path);
+ return result;
+}
+/* ---- end include deepseek_v4_config.c ---- */
+
+#undef coli_v4_config_load
+#undef coli_v4_config_parse
+
+int coli_v4_config_parse(ColiDeepSeekV4Config *config, const char *json,
+ char *error, size_t error_size) {
+ int result = coli_v4_config_parse_strict(config, json, error, error_size);
+ if (!result) return 0;
+ if (!config || !json || !strstr(json, "\"dspark_block_size\"") ||
+ config->num_hidden_layers < 1 ||
+ config->compress_ratio_count <= config->num_hidden_layers)
+ return result;
+ int inferred = config->compress_ratio_count - config->num_hidden_layers;
+ if (inferred < 1 || inferred > COLI_V4_DSPARK_MAX_STAGES)
+ return result;
+ config->num_nextn_predict_layers = inferred;
+ if (config->num_experts_per_tok < 1 ||
+ config->num_experts_per_tok > config->n_routed_experts ||
+ config->n_shared_experts != 1 || config->hc_mult < 1)
+ return result;
+ if (error && error_size) error[0] = 0;
+ return 0;
+}
+
+int coli_v4_config_load(ColiDeepSeekV4Config *config, const char *model_dir,
+ char *error, size_t error_size) {
+ if (!config || !model_dir)
+ return set_error(error, error_size, "invalid DeepSeek-V4 config path");
+ size_t path_length = strlen(model_dir) + sizeof("/config.json");
+ char *path = malloc(path_length);
+ if (!path) return set_error(error, error_size, "out of memory building config path");
+ snprintf(path, path_length, "%s/config.json", model_dir);
+ FILE *stream = fopen(path, "rb");
+ if (!stream) { free(path); return set_error(error, error_size,
+ "cannot open config"); }
+ fseek(stream, 0, SEEK_END); long length = ftell(stream); rewind(stream);
+ char *text = length > 0 ? malloc((size_t)length + 1) : NULL;
+ if (!text || fread(text, 1, (size_t)length, stream) != (size_t)length) {
+ free(text); fclose(stream); free(path);
+ return set_error(error, error_size, "cannot read config");
+ }
+ text[length] = 0; fclose(stream);
+ int result = coli_v4_config_parse(config, text, error, error_size);
+ free(text); free(path); return result;
+}
+/* ---- end inlined deepseek_v4_config_dspark_compat.c ---- */
+#endif /* COLI_V4_UNIT_CONFIG_DSPARK_COMPAT */
+
+#ifdef COLI_V4_UNIT_DSPARK
+/* ######## deepseek_v4_dspark.c ######## */
+#include <string.h>
+
+static char *dspark_second_dot(const char *text, int character) {
+ char *first = strchr(text, character);
+ return first ? strchr(first + 1, character) : NULL;
+}
+
+#define strchr dspark_second_dot
+/* ---- begin inlined deepseek_v4_dspark.c ---- */
+#include "deepseek_v4_dspark.h"
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "json.h"
+#include "safetensors_index.h"
+
+static int dspark_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list arguments; va_start(arguments, format);
+ vsnprintf(error, size, format, arguments); va_end(arguments);
+ }
+ return -1;
+}
+
+static char *read_config(const char *model_dir) {
+ size_t length = strlen(model_dir) + sizeof("/config.json");
+ char *path = malloc(length);
+ if (!path) return NULL;
+ snprintf(path, length, "%s/config.json", model_dir);
+ FILE *stream = fopen(path, "rb"); free(path);
+ if (!stream) return NULL;
+ fseek(stream, 0, SEEK_END); long bytes = ftell(stream); rewind(stream);
+ char *text = bytes > 0 ? malloc((size_t)bytes + 1) : NULL;
+ if (!text || fread(text, 1, (size_t)bytes, stream) != (size_t)bytes) {
+ free(text); fclose(stream); return NULL;
+ }
+ text[bytes] = 0; fclose(stream); return text;
+}
+
+static int json_integer(jval *root, const char *name, int *output) {
+ jval *value = json_get(root, name);
+ if (!value || value->t != J_NUM) return -1;
+ *output = (int)value->num; return 0;
+}
+
+int coli_v4_dspark_layer_plan(ColiDeepSeekV4LayerPlan *plan,
+ const ColiDeepSeekV4Config *config, int stage,
+ char *error, size_t error_size) {
+ if (!plan || !config || stage < 0 || stage >= COLI_V4_DSPARK_MAX_STAGES)
+ return dspark_error(error, error_size, "invalid DSpark layer plan");
+ ColiDeepSeekV4Config common = *config;
+ common.num_hash_layers = 0;
+ common.compress_ratios[0] = 0;
+ if (coli_v4_layer_plan(plan, &common, 0, error, error_size)) return -1;
+ plan->layer = stage;
+ plan->compression_ratio = 0;
+ plan->uses_hash_router = 0;
+ plan->has_compressor = 0;
+ plan->has_indexer = 0;
+ for (size_t i = 0; i < plan->tensor_count; i++) {
+ const char *suffix = strchr(plan->tensors[i].name, '.');
+ if (!suffix) return dspark_error(error, error_size,
+ "invalid common layer tensor name");
+ char rewritten[COLI_V4_MAX_TENSOR_NAME];
+ int written = snprintf(rewritten, sizeof(rewritten), "mtp.%d%s",
+ stage, suffix);
+ if (written < 0 || (size_t)written >= sizeof(rewritten))
+ return dspark_error(error, error_size, "DSpark tensor name too long");
+ strcpy(plan->tensors[i].name, rewritten);
+ }
+ return 0;
+}
+
+static int require_tensor(const ColiSafetensorsIndex *index, const char *name,
+ ColiSafetensorsDType dtype, int rank,
+ const int64_t *shape, uint64_t *bytes,
+ char *error, size_t error_size) {
+ const ColiSafetensorsTensor *tensor = coli_st_find(index, name);
+ if (!tensor) return dspark_error(error, error_size,
+ "missing DSpark tensor: %s", name);
+ if (tensor->dtype != dtype || tensor->rank != rank)
+ return dspark_error(error, error_size,
+ "DSpark dtype/rank mismatch: %s", name);
+ for (int i = 0; i < rank; i++)
+ if (tensor->shape[i] != shape[i])
+ return dspark_error(error, error_size,
+ "DSpark shape mismatch: %s", name);
+ *bytes += tensor->nbytes; return 0;
+}
+
+int coli_v4_dspark_inspect(const char *model_dir,
+ const ColiDeepSeekV4Config *config,
+ ColiDeepSeekV4DSparkManifest *manifest,
+ char *error, size_t error_size) {
+ if (!model_dir || !config || !manifest)
+ return dspark_error(error, error_size, "invalid DSpark inspect arguments");
+ memset(manifest, 0, sizeof(*manifest));
+ char *text = read_config(model_dir), *arena = NULL;
+ jval *root = text ? json_parse(text, &arena) : NULL;
+ if (!root || root->t != J_OBJ ||
+ json_integer(root, "dspark_block_size", &manifest->block_size) ||
+ json_integer(root, "dspark_noise_token_id", &manifest->noise_token_id) ||
+ json_integer(root, "dspark_markov_rank", &manifest->markov_rank)) {
+ free(arena); free(text);
+ return dspark_error(error, error_size, "missing DSpark config fields");
+ }
+ jval *targets = json_get(root, "dspark_target_layer_ids");
+ if (!targets || targets->t != J_ARR || targets->len < 1 ||
+ targets->len > COLI_V4_DSPARK_MAX_TARGETS) {
+ free(arena); free(text);
+ return dspark_error(error, error_size, "invalid DSpark target layers");
+ }
+ manifest->target_count = targets->len;
+ for (int i = 0; i < targets->len; i++) {
+ if (targets->kids[i]->t != J_NUM) {
+ free(arena); free(text);
+ return dspark_error(error, error_size, "invalid DSpark target layer");
+ }
+ manifest->target_layer_ids[i] = (int)targets->kids[i]->num;
+ }
+ free(arena); free(text);
+
+ ColiSafetensorsIndex *index = NULL;
+ if (coli_st_index_open(&index, model_dir, error, error_size)) return -1;
+ char name[160];
+ for (int stage = 0; stage < COLI_V4_DSPARK_MAX_STAGES; stage++) {
+ snprintf(name, sizeof(name), "mtp.%d.attn_norm.weight", stage);
+ if (!coli_st_find(index, name)) break;
+ manifest->stage_count++;
+ ColiDeepSeekV4LayerPlan plan;
+ ColiDeepSeekV4LayerStats stats;
+ if (coli_v4_dspark_layer_plan(&plan, config, stage, error, error_size) ||
+ coli_v4_layer_validate(&plan, index, &stats, error, error_size)) {
+ coli_st_index_close(index); return -1;
+ }
+ manifest->common_stage_bytes[stage] = stats.total_bytes;
+ }
+ if (manifest->stage_count < 1) {
+ coli_st_index_close(index);
+ return dspark_error(error, error_size, "checkpoint has no DSpark stages");
+ }
+ int64_t main_weight[] = {config->hidden_size,
+ (int64_t)config->hidden_size * manifest->target_count};
+ int64_t main_scale[] = {(config->hidden_size + 127) / 128,
+ (main_weight[1] + 127) / 128};
+ int64_t hidden[] = {config->hidden_size};
+ int last = manifest->stage_count - 1;
+ int64_t markov[] = {config->vocab_size, manifest->markov_rank};
+ int64_t confidence[] = {1, config->hidden_size + manifest->markov_rank};
+ snprintf(name, sizeof(name), "mtp.0.main_proj.weight");
+ int failed = require_tensor(index, name, COLI_ST_F8_E4M3, 2, main_weight,
+ &manifest->special_bytes, error, error_size);
+ snprintf(name, sizeof(name), "mtp.0.main_proj.scale");
+ failed |= require_tensor(index, name, COLI_ST_F8_E8M0, 2, main_scale,
+ &manifest->special_bytes, error, error_size);
+ failed |= require_tensor(index, "mtp.0.main_norm.weight", COLI_ST_BF16, 1,
+ hidden, &manifest->special_bytes, error, error_size);
+ snprintf(name, sizeof(name), "mtp.%d.norm.weight", last);
+ failed |= require_tensor(index, name, COLI_ST_BF16, 1, hidden,
+ &manifest->special_bytes, error, error_size);
+ snprintf(name, sizeof(name), "mtp.%d.markov_head.markov_w1.weight", last);
+ failed |= require_tensor(index, name, COLI_ST_BF16, 2, markov,
+ &manifest->special_bytes, error, error_size);
+ snprintf(name, sizeof(name), "mtp.%d.markov_head.markov_w2.weight", last);
+ failed |= require_tensor(index, name, COLI_ST_BF16, 2, markov,
+ &manifest->special_bytes, error, error_size);
+ snprintf(name, sizeof(name), "mtp.%d.confidence_head.proj.weight", last);
+ failed |= require_tensor(index, name, COLI_ST_BF16, 2, confidence,
+ &manifest->special_bytes, error, error_size);
+ coli_st_index_close(index);
+ return failed ? -1 : 0;
+}
+/* ---- end inlined deepseek_v4_dspark.c ---- */
+
+#undef strchr
+#endif /* COLI_V4_UNIT_DSPARK */
+
+#ifdef COLI_V4_UNIT_DSPARK_MEMORY
+/* ######## deepseek_v4_dspark_memory.c ######## */
+#define coli_v4_dspark_memory_plan coli_v4_dspark_memory_plan_unmargined
+/* ---- begin inlined deepseek_v4_dspark_memory.c ---- */
+#include "deepseek_v4_dspark.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#include "deepseek_v4_dspark.h"
+#include "safetensors_index.h"
+
+#define DSPARK_MIB UINT64_C(1048576)
+
+int coli_v4_dspark_memory_plan(const char *model_dir,
+ const ColiDeepSeekV4Config *config,
+ ColiV4DSparkMemoryPlan *plan,
+ char *error, size_t error_size) {
+ if (!model_dir || !config || !plan) return -1;
+ memset(plan, 0, sizeof(*plan));
+ ColiDeepSeekV4DSparkManifest manifest;
+ if (coli_v4_dspark_inspect(model_dir, config, &manifest,
+ error, error_size)) return -1;
+ plan->stages = manifest.stage_count;
+ plan->expert_slots_per_stage = config->num_experts_per_tok;
+ plan->resident_heads_bytes = manifest.special_bytes;
+ for (int i = 0; i < manifest.stage_count; i++)
+ if (manifest.common_stage_bytes[i] > plan->streamed_stage_bytes)
+ plan->streamed_stage_bytes = manifest.common_stage_bytes[i];
+ ColiSafetensorsIndex *index = NULL;
+ if (coli_st_index_open(&index, model_dir, error, error_size)) return -1;
+ static const char *parts[] = {
+ "mtp.0.ffn.experts.0.w1.weight", "mtp.0.ffn.experts.0.w1.scale",
+ "mtp.0.ffn.experts.0.w2.weight", "mtp.0.ffn.experts.0.w2.scale",
+ "mtp.0.ffn.experts.0.w3.weight", "mtp.0.ffn.experts.0.w3.scale",
+ };
+ for (size_t i = 0; i < sizeof(parts) / sizeof(parts[0]); i++) {
+ const ColiSafetensorsTensor *tensor = coli_st_find(index, parts[i]);
+ if (!tensor) { coli_st_index_close(index); return -1; }
+ plan->expert_record_bytes += tensor->nbytes;
+ }
+ coli_st_index_close(index);
+ plan->minimum_expert_cache_bytes = plan->expert_record_bytes *
+ (uint64_t)plan->stages * config->num_experts_per_tok;
+ uint64_t capture = (uint64_t)64 * manifest.target_count * config->hc_mult *
+ config->hidden_size * sizeof(float);
+ uint64_t query = (uint64_t)64 * config->hc_mult * config->hidden_size *
+ sizeof(float) * 2;
+ uint64_t kv = (uint64_t)manifest.stage_count *
+ (config->sliding_window + 64) * config->head_dim * sizeof(float);
+ plan->working_bytes = 64 * DSPARK_MIB + capture + query + kv;
+ plan->incremental_reserve_bytes = plan->resident_heads_bytes +
+ plan->streamed_stage_bytes + plan->minimum_expert_cache_bytes +
+ plan->working_bytes;
+ return 0;
+}
+/* ---- end inlined deepseek_v4_dspark_memory.c ---- */
+
+#undef coli_v4_dspark_memory_plan
+
+int coli_v4_dspark_memory_plan(const char *model_dir,
+ const ColiDeepSeekV4Config *config,
+ ColiV4DSparkMemoryPlan *plan,
+ char *error, size_t error_size) {
+ int result = coli_v4_dspark_memory_plan_unmargined(
+ model_dir, config, plan, error, error_size);
+ if (result) return result;
+ /* Native FP8/FP4 batch kernels and OpenMP allocate short-lived per-thread
+ buffers outside the explicit query/KV arrays. Keep this model-size-
+ independent margin in the RAM tier plan so low-memory hosts do not lend
+ the same bytes to the target expert cache. */
+ uint64_t allocator_margin = 256 * DSPARK_MIB;
+ if (UINT64_MAX - plan->working_bytes < allocator_margin ||
+ UINT64_MAX - plan->incremental_reserve_bytes < allocator_margin)
+ return -1;
+ plan->working_bytes += allocator_margin;
+ plan->incremental_reserve_bytes += allocator_margin;
+ return 0;
+}
+#endif /* COLI_V4_UNIT_DSPARK_MEMORY */
+
+#ifdef COLI_V4_UNIT_DSPARK_RUNTIME_RESIDENT
+/* ######## deepseek_v4_dspark_runtime_resident.c ######## */
+#define coli_v4_dspark_layer_load coli_v4_dspark_layer_reference_load
+/* ---- begin include deepseek_v4_dspark_runtime.c ---- */
+#include "deepseek_v4_dspark.h"
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static int runtime_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list arguments; va_start(arguments, format);
+ vsnprintf(error, size, format, arguments); va_end(arguments);
+ }
+ return -1;
+}
+
+static void add_stats(ColiDeepSeekV4LayerStats *stats,
+ const ColiSafetensorsTensor *tensor) {
+ stats->tensor_count++; stats->total_bytes += tensor->nbytes;
+ switch (tensor->dtype) {
+ case COLI_ST_BF16: stats->bf16_bytes += tensor->nbytes; break;
+ case COLI_ST_F32: stats->f32_bytes += tensor->nbytes; break;
+ case COLI_ST_F8_E4M3: stats->fp8_weight_bytes += tensor->nbytes; break;
+ case COLI_ST_F8_E8M0: stats->fp8_scale_bytes += tensor->nbytes; break;
+ case COLI_ST_I64: stats->i64_bytes += tensor->nbytes; break;
+ default: break;
+ }
+}
+
+int coli_v4_dspark_layer_load(ColiV4Engine *engine,
+ ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const ColiSafetensorsIndex *index, int stage,
+ char *error, size_t error_size) {
+ (void)engine;
+ if (!weights || !config || !index)
+ return runtime_error(error, error_size, "invalid DSpark layer load");
+ memset(weights, 0, sizeof(*weights));
+ if (coli_v4_dspark_layer_plan(&weights->plan, config, stage,
+ error, error_size)) return -1;
+ for (size_t i = 0; i < weights->plan.tensor_count; i++) {
+ ColiDeepSeekV4TensorSpec *spec = &weights->plan.tensors[i];
+ char actual[COLI_V4_MAX_TENSOR_NAME];
+ strcpy(actual, spec->name);
+ const ColiSafetensorsTensor *tensor = coli_st_find(index, actual);
+ if (!tensor || tensor->dtype != spec->dtype ||
+ tensor->rank != spec->rank) {
+ coli_v4_layer_free(NULL, weights);
+ return runtime_error(error, error_size,
+ "invalid DSpark tensor: %s", actual);
+ }
+ for (int dimension = 0; dimension < spec->rank; dimension++)
+ if (tensor->shape[dimension] != spec->shape[dimension]) {
+ coli_v4_layer_free(NULL, weights);
+ return runtime_error(error, error_size,
+ "DSpark shape mismatch: %s", actual);
+ }
+ weights->data[i] = malloc((size_t)tensor->nbytes);
+ if (!weights->data[i] ||
+ coli_st_read_tensor(index, tensor, weights->data[i])) {
+ coli_v4_layer_free(NULL, weights);
+ return runtime_error(error, error_size,
+ "cannot load DSpark tensor: %s", actual);
+ }
+ add_stats(&weights->stats, tensor);
+ const char *suffix = actual + strlen("mtp.0");
+ int prefix = snprintf(spec->name, sizeof(spec->name),
+ "layers.%d", stage);
+ size_t suffix_length = strlen(suffix);
+ if (prefix < 0 || (size_t)prefix + suffix_length >= sizeof(spec->name)) {
+ coli_v4_layer_free(NULL, weights);
+ return runtime_error(error, error_size,
+ "DSpark runtime tensor name is too long");
+ }
+ memcpy(spec->name + prefix, suffix, suffix_length + 1);
+ }
+ return 0;
+}
+/* ---- end include deepseek_v4_dspark_runtime.c ---- */
+
+#undef coli_v4_dspark_layer_load
+
+#include "deepseek_v4_dspark.h"
+#include "deepseek_v4_internal.h"
+
+static int dspark_stages_resident(ColiV4Engine *engine) {
+ return engine && engine->runtime.dspark_resident;
+}
+
+int coli_v4_dspark_layer_load(ColiV4Engine *engine,
+ ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const ColiSafetensorsIndex *index, int stage,
+ char *error, size_t error_size) {
+ if (!dspark_stages_resident(engine))
+ return coli_v4_dspark_layer_reference_load(
+ engine, weights, config, index, stage, error, error_size);
+ if (!engine || !weights || !config || !index || stage < 0 ||
+ stage >= COLI_V4_DSPARK_MAX_STAGES) return -1;
+ if (engine->dspark_resident.index &&
+ engine->dspark_resident.index != index) {
+ if (error && error_size)
+ snprintf(error, error_size,
+ "resident DSpark stages cannot switch model instances");
+ return -1;
+ }
+ engine->dspark_resident.index = index;
+ if (!engine->dspark_resident.ready[stage]) {
+ if (coli_v4_dspark_layer_reference_load(
+ engine, &engine->dspark_resident.layers[stage], config, index,
+ stage, error, error_size)) return -1;
+ engine->dspark_resident.ready[stage] = 1;
+ engine->dspark_resident.total_bytes +=
+ engine->dspark_resident.layers[stage].stats.total_bytes;
+ fprintf(stderr, "dspark_stage_resident stage=%d total=%.3fGiB\n",
+ stage, engine->dspark_resident.total_bytes / 1073741824.0);
+ }
+ *weights = engine->dspark_resident.layers[stage]; return 0;
+}
+
+void coli_v4_dspark_layer_release(ColiV4Engine *engine,
+ ColiDeepSeekV4LayerWeights *weights) {
+ if (!weights) return;
+ int stage = weights->plan.layer;
+ if (dspark_stages_resident(engine) && engine && stage >= 0 &&
+ stage < COLI_V4_DSPARK_MAX_STAGES &&
+ engine->dspark_resident.ready[stage] &&
+ weights->data[0] == engine->dspark_resident.layers[stage].data[0]) {
+ memset(weights, 0, sizeof(*weights)); return;
+ }
+ coli_v4_layer_free(engine, weights);
+}
+#endif /* COLI_V4_UNIT_DSPARK_RUNTIME_RESIDENT */
+
+#ifdef COLI_V4_UNIT_DSPARK_EXPERT_STORE
+/* ######## deepseek_v4_dspark_expert_store.c ######## */
+#include <stdarg.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "deepseek_v4_dspark.h"
+
+static int dspark_store_snprintf_v2(char *output, size_t size,
+ const char *format, ...) {
+ char rewritten[256];
+ /* strlen("layers.%d.ffn.experts.") is 22. Comparing 23 bytes also
+ * compares the literal NUL with the following '%' in the real format,
+ * preventing the mtp namespace rewrite and loading target experts. */
+ if (!strncmp(format, "layers.%d.ffn.experts.", 22))
+ snprintf(rewritten, sizeof(rewritten), "mtp.%s", format + 7);
+ else
+ snprintf(rewritten, sizeof(rewritten), "%s", format);
+ va_list arguments; va_start(arguments, format);
+ int result = vsnprintf(output, size, rewritten, arguments);
+ va_end(arguments); return result;
+}
+
+#define snprintf dspark_store_snprintf_v2
+#define coli_deepseek_v4_expert_store_open \
+ coli_deepseek_v4_dspark_expert_store_open
+/* ---- begin include deepseek_v4_expert_store.c ---- */
+#define _GNU_SOURCE
+#include "deepseek_v4_internal.h"
+
+#include <assert.h>
+#include <pthread.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "safetensors_index.h"
+
+#ifdef COLI_V4_EXPERIMENTAL_PREFETCH_BATCH
+int coli_st_prefetch_many(
+ const ColiSafetensorsIndex *index, const int *shards,
+ const uint64_t *offsets, const size_t *lengths, size_t count);
+#endif
+
+enum { V4_W1 = 0, V4_W2 = 1, V4_W3 = 2, V4_MATRIX_COUNT = 3 };
+
+typedef struct {
+ const ColiSafetensorsTensor *weight[V4_MATRIX_COUNT];
+ const ColiSafetensorsTensor *scale[V4_MATRIX_COUNT];
+ int shard;
+ uint64_t scale_offset;
+ uint64_t scale_bytes;
+ uint64_t weight_offset;
+ uint64_t weight_bytes;
+ uint64_t record_bytes;
+} V4ExpertRecord;
+
+typedef struct {
+ int expert;
+ unsigned references;
+ uint64_t used;
+ unsigned char *slab;
+} V4ExpertSlot;
+
+typedef struct {
+ ColiSafetensorsIndex *index;
+ int layers;
+ int experts_per_layer;
+ int slots_per_layer;
+ uint64_t record_bytes;
+ V4ExpertRecord *records;
+ V4ExpertSlot *slots;
+ uint64_t clock;
+ unsigned active_leases;
+ ColiExpertStoreStats stats;
+ pthread_mutex_t mutex;
+} V4ExpertStoreState;
+
+static int set_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list args;
+ va_start(args, format);
+ vsnprintf(error, size, format, args);
+ va_end(args);
+ }
+ return -1;
+}
+
+static int compare_tensors(const void *left, const void *right) {
+ const ColiSafetensorsTensor *const *a = left;
+ const ColiSafetensorsTensor *const *b = right;
+ return ((*a)->offset > (*b)->offset) - ((*a)->offset < (*b)->offset);
+}
+
+static int contiguous_group(const ColiSafetensorsTensor *const input[3],
+ int *shard, uint64_t *offset, uint64_t *bytes) {
+ const ColiSafetensorsTensor *parts[3] = {input[0], input[1], input[2]};
+ qsort(parts, 3, sizeof(parts[0]), compare_tensors);
+ if (parts[0]->shard != parts[1]->shard || parts[1]->shard != parts[2]->shard ||
+ parts[0]->offset + parts[0]->nbytes != parts[1]->offset ||
+ parts[1]->offset + parts[1]->nbytes != parts[2]->offset)
+ return -1;
+ *shard = parts[0]->shard;
+ *offset = parts[0]->offset;
+ *bytes = parts[2]->offset + parts[2]->nbytes - parts[0]->offset;
+ return 0;
+}
+
+static int validate_matrix(const ColiSafetensorsTensor *weight,
+ const ColiSafetensorsTensor *scale) {
+ if (!weight || !scale || weight->dtype != COLI_ST_I8 ||
+ scale->dtype != COLI_ST_F8_E8M0 || weight->rank != 2 || scale->rank != 2 ||
+ weight->shape[0] != scale->shape[0] || weight->shape[1] <= 0 ||
+ scale->shape[1] <= 0)
+ return -1;
+ int64_t logical_columns = weight->shape[1] * 2;
+ return scale->shape[1] * 32 == logical_columns ? 0 : -1;
+}
+
+static int build_record(V4ExpertStoreState *state, int layer, int expert,
+ V4ExpertRecord *record, char *error, size_t error_size) {
+ static const char *matrix_names[V4_MATRIX_COUNT] = {"w1", "w2", "w3"};
+ char name[160];
+ memset(record, 0, sizeof(*record));
+ for (int matrix = 0; matrix < V4_MATRIX_COUNT; matrix++) {
+ snprintf(name, sizeof(name), "layers.%d.ffn.experts.%d.%s.weight",
+ layer, expert, matrix_names[matrix]);
+ record->weight[matrix] = coli_st_find(state->index, name);
+ snprintf(name, sizeof(name), "layers.%d.ffn.experts.%d.%s.scale",
+ layer, expert, matrix_names[matrix]);
+ record->scale[matrix] = coli_st_find(state->index, name);
+ if (validate_matrix(record->weight[matrix], record->scale[matrix]) != 0)
+ return set_error(error, error_size,
+ "invalid native FP4 expert matrix: layer=%d expert=%d %s",
+ layer, expert, matrix_names[matrix]);
+ }
+ int scale_shard = -1, weight_shard = -1;
+ if (contiguous_group(record->scale, &scale_shard, &record->scale_offset,
+ &record->scale_bytes) != 0 ||
+ contiguous_group(record->weight, &weight_shard, &record->weight_offset,
+ &record->weight_bytes) != 0 || scale_shard != weight_shard)
+ return set_error(error, error_size,
+ "expert is not two contiguous ranges: layer=%d expert=%d",
+ layer, expert);
+ record->shard = scale_shard;
+ record->record_bytes = record->scale_bytes + record->weight_bytes;
+ return 0;
+}
+
+static V4ExpertRecord *get_record(V4ExpertStoreState *state, ColiExpertKey key) {
+ if (key.layer < 0 || key.layer >= state->layers || key.expert < 0 ||
+ key.expert >= state->experts_per_layer)
+ return NULL;
+ return &state->records[(size_t)key.layer * state->experts_per_layer + key.expert];
+}
+
+static V4ExpertSlot *layer_slots(V4ExpertStoreState *state, int layer) {
+ return state->slots + (size_t)layer * state->slots_per_layer;
+}
+
+static void fill_tensor_view(ColiTensorView *view,
+ const V4ExpertRecord *record,
+ const V4ExpertSlot *slot, int matrix) {
+ const ColiSafetensorsTensor *weight = record->weight[matrix];
+ const ColiSafetensorsTensor *scale = record->scale[matrix];
+ memset(view, 0, sizeof(*view));
+ view->format = COLI_TENSOR_FP4_NATIVE_BLOCK;
+ view->scale_format = COLI_SCALE_UE8M0;
+ view->data = slot->slab + record->scale_bytes +
+ (weight->offset - record->weight_offset);
+ view->scales = slot->slab + (scale->offset - record->scale_offset);
+ view->data_bytes = (size_t)weight->nbytes;
+ view->scale_bytes = (size_t)scale->nbytes;
+ view->rows = weight->shape[0];
+ view->columns = weight->shape[1] * 2;
+ view->block_rows = 1;
+ view->block_columns = 32;
+}
+
+static int lookup(ColiExpertStore *store, ColiExpertKey key,
+ ColiExpertView *view) {
+ if (!store || !store->state || !view) {
+ if (view) memset(view, 0, sizeof(*view));
+ return -1;
+ }
+ V4ExpertStoreState *state = store->state;
+ V4ExpertRecord *record = get_record(state, key);
+ if (!record) {
+ memset(view, 0, sizeof(*view));
+ return -1;
+ }
+ pthread_mutex_lock(&state->mutex);
+ state->stats.requests++;
+ V4ExpertSlot *slots = layer_slots(state, key.layer);
+ V4ExpertSlot *slot = NULL;
+ for (int i = 0; i < state->slots_per_layer; i++) {
+ if (slots[i].slab && slots[i].expert == key.expert) {
+ slot = &slots[i];
+ state->stats.hits++;
+ break;
+ }
+ }
+ if (!slot) {
+ for (int i = 0; i < state->slots_per_layer; i++) {
+ if (!slots[i].references && (!slot || !slots[i].slab ||
+ (slot->slab && slots[i].used < slot->used)))
+ slot = &slots[i];
+ }
+ if (!slot) {
+ pthread_mutex_unlock(&state->mutex);
+ memset(view, 0, sizeof(*view));
+ return -1;
+ }
+ if (!slot->slab) {
+ slot->slab = malloc((size_t)state->record_bytes);
+ if (!slot->slab) {
+ pthread_mutex_unlock(&state->mutex);
+ memset(view, 0, sizeof(*view));
+ return -1;
+ }
+ state->stats.resident_bytes += state->record_bytes;
+ }
+ /* A short read must never expose a partially overwritten old slot. */
+ slot->expert = -1;
+ if (coli_st_read_at(state->index, record->shard, record->scale_offset,
+ (size_t)record->scale_bytes, slot->slab) != 0 ||
+ coli_st_read_at(state->index, record->shard, record->weight_offset,
+ (size_t)record->weight_bytes,
+ slot->slab + record->scale_bytes) != 0) {
+ pthread_mutex_unlock(&state->mutex);
+ memset(view, 0, sizeof(*view));
+ return -1;
+ }
+ slot->expert = key.expert;
+ state->stats.misses++;
+ state->stats.bytes_read += record->record_bytes;
+ }
+ slot->references++;
+ state->active_leases++;
+ slot->used = ++state->clock;
+ memset(view, 0, sizeof(*view));
+ view->key = key;
+ fill_tensor_view(&view->gate, record, slot, V4_W1);
+ fill_tensor_view(&view->down, record, slot, V4_W2);
+ fill_tensor_view(&view->up, record, slot, V4_W3);
+ view->lease = slot;
+ pthread_mutex_unlock(&state->mutex);
+ return 0;
+}
+
+static void release(ColiExpertStore *store, ColiExpertView *view) {
+ if (!store || !store->state || !view || !view->lease) {
+ if (view) memset(view, 0, sizeof(*view));
+ return;
+ }
+ V4ExpertStoreState *state = store->state;
+ V4ExpertSlot *slot = view->lease;
+ pthread_mutex_lock(&state->mutex);
+ if (slot->references) slot->references--;
+ if (state->active_leases) state->active_leases--;
+ pthread_mutex_unlock(&state->mutex);
+ memset(view, 0, sizeof(*view));
+}
+
+static int prefetch(ColiExpertStore *store, const ColiExpertKey *keys,
+ size_t count) {
+ if (!store || !store->state || (!keys && count)) return 0;
+ V4ExpertStoreState *state = store->state;
+ int accepted = 0;
+#ifdef COLI_V4_EXPERIMENTAL_PREFETCH_BATCH
+ size_t capacity = count * 2, ranges = 0;
+ int *shards = malloc(capacity * sizeof(*shards));
+ uint64_t *offsets = malloc(capacity * sizeof(*offsets));
+ size_t *lengths = malloc(capacity * sizeof(*lengths));
+ int candidates = 0;
+ if ((!shards || !offsets || !lengths) && capacity) {
+ free(lengths); free(offsets); free(shards); return 0;
+ }
+ pthread_mutex_lock(&state->mutex);
+ for (size_t i = 0; i < count; i++) {
+ V4ExpertRecord *record = get_record(state, keys[i]);
+ if (!record) continue;
+ int resident = 0;
+ V4ExpertSlot *slots = layer_slots(state, keys[i].layer);
+ for (int slot = 0; slot < state->slots_per_layer; slot++)
+ if (slots[slot].slab && slots[slot].expert == keys[i].expert) {
+ resident = 1; break;
+ }
+ if (resident) continue;
+ shards[ranges] = record->shard;
+ offsets[ranges] = record->scale_offset;
+ lengths[ranges++] = (size_t)record->scale_bytes;
+ shards[ranges] = record->shard;
+ offsets[ranges] = record->weight_offset;
+ lengths[ranges++] = (size_t)record->weight_bytes;
+ candidates++;
+ }
+ pthread_mutex_unlock(&state->mutex);
+ if (candidates && !coli_st_prefetch_many(
+ state->index, shards, offsets, lengths, ranges))
+ accepted = candidates;
+ free(lengths); free(offsets); free(shards);
+#else
+ for (size_t i = 0; i < count; i++) {
+ V4ExpertRecord *record = get_record(state, keys[i]);
+ if (!record) continue;
+ if (coli_st_prefetch_at(state->index, record->shard, record->scale_offset,
+ (size_t)record->scale_bytes) == 0 &&
+ coli_st_prefetch_at(state->index, record->shard, record->weight_offset,
+ (size_t)record->weight_bytes) == 0)
+ accepted++;
+ }
+#endif
+ pthread_mutex_lock(&state->mutex);
+ state->stats.prefetched += (uint64_t)accepted;
+ pthread_mutex_unlock(&state->mutex);
+ return accepted;
+}
+
+static void stats(const ColiExpertStore *store, ColiExpertStoreStats *output) {
+ if (!store || !store->state || !output) return;
+ V4ExpertStoreState *state = store->state;
+ pthread_mutex_lock(&state->mutex);
+ *output = state->stats;
+ pthread_mutex_unlock(&state->mutex);
+}
+
+static void destroy(ColiExpertStore *store) {
+ if (!store) return;
+ V4ExpertStoreState *state = store->state;
+ if (state) {
+ assert(state->active_leases == 0 && "destroy with active expert leases");
+ for (int i = 0; i < state->layers * state->slots_per_layer; i++)
+ free(state->slots[i].slab);
+ pthread_mutex_destroy(&state->mutex);
+ coli_st_index_close(state->index);
+ free(state->records);
+ free(state->slots);
+ free(state);
+ }
+ free(store);
+}
+
+int coli_deepseek_v4_expert_store_open(
+ const ColiDeepSeekV4ExpertStoreOptions *options, ColiExpertStore **output,
+ char *error, size_t error_size) {
+ static const ColiExpertStoreOps operations = {
+ lookup, release, prefetch, stats, destroy
+ };
+ if (!options || !output || !options->model_dir || options->layers < 1 ||
+ options->experts_per_layer < 1 || !options->cache_bytes)
+ return set_error(error, error_size, "invalid DeepSeek-V4 ExpertStore options");
+ *output = NULL;
+ ColiExpertStore *store = calloc(1, sizeof(*store));
+ V4ExpertStoreState *state = calloc(1, sizeof(*state));
+ if (!store || !state) {
+ free(store);
+ free(state);
+ return set_error(error, error_size, "out of memory creating ExpertStore");
+ }
+ pthread_mutex_init(&state->mutex, NULL);
+ state->layers = options->layers;
+ state->experts_per_layer = options->experts_per_layer;
+ if (coli_st_index_open(&state->index, options->model_dir, error, error_size) != 0)
+ goto fail;
+ size_t record_count = (size_t)state->layers * state->experts_per_layer;
+ state->records = calloc(record_count, sizeof(*state->records));
+ if (!state->records) {
+ set_error(error, error_size, "out of memory creating expert manifest");
+ goto fail;
+ }
+ for (int layer = 0; layer < state->layers; layer++) {
+ for (int expert = 0; expert < state->experts_per_layer; expert++) {
+ V4ExpertRecord *record = &state->records[
+ (size_t)layer * state->experts_per_layer + expert];
+ if (build_record(state, layer, expert, record, error, error_size) != 0)
+ goto fail;
+ if (!state->record_bytes) state->record_bytes = record->record_bytes;
+ if (record->record_bytes != state->record_bytes) {
+ set_error(error, error_size, "non-uniform expert size at layer=%d expert=%d",
+ layer, expert);
+ goto fail;
+ }
+ }
+ }
+ state->slots_per_layer = (int)(options->cache_bytes /
+ ((uint64_t)state->layers * state->record_bytes));
+ int minimum_slots = state->experts_per_layer < 6
+ ? state->experts_per_layer : 6;
+ if (state->slots_per_layer < minimum_slots) {
+ set_error(error, error_size,
+ "cache budget cannot hold %d active experts per layer "
+ "(need %llu bytes)", minimum_slots,
+ (unsigned long long)((uint64_t)state->layers * minimum_slots *
+ state->record_bytes));
+ goto fail;
+ }
+ if (state->slots_per_layer > state->experts_per_layer)
+ state->slots_per_layer = state->experts_per_layer;
+ state->slots = calloc((size_t)state->layers * state->slots_per_layer,
+ sizeof(*state->slots));
+ if (!state->slots) {
+ set_error(error, error_size, "out of memory creating expert cache slots");
+ goto fail;
+ }
+ for (int i = 0; i < state->layers * state->slots_per_layer; i++)
+ state->slots[i].expert = -1;
+ state->stats.capacity_bytes = (uint64_t)state->layers *
+ state->slots_per_layer * state->record_bytes;
+ store->ops = &operations;
+ store->state = state;
+ *output = store;
+ return 0;
+
+fail:
+ if (state->slots) free(state->slots);
+ free(state->records);
+ coli_st_index_close(state->index);
+ pthread_mutex_destroy(&state->mutex);
+ free(state);
+ free(store);
+ return -1;
+}
+/* ---- end include deepseek_v4_expert_store.c ---- */
+
+#undef coli_deepseek_v4_expert_store_open
+#undef snprintf
+#endif /* COLI_V4_UNIT_DSPARK_EXPERT_STORE */
+
+#ifdef COLI_V4_UNIT_DSPARK_ATTENTION
+/* ######## deepseek_v4_dspark_attention.c ######## */
+/* ---- begin inlined deepseek_v4_dspark_attention.c ---- */
+#include "deepseek_v4_dspark.h"
+
+#include <math.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "deepseek_v4_internal.h"
+#include "native_quant.h"
+#include "native_quant_batch.h"
+
+struct ColiV4DSparkAttentionState {
+ int window_size;
+ int head_dim;
+ int valid;
+ float *kv;
+ int *positions;
+};
+
+static int ds_attn_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list arguments; va_start(arguments, format);
+ vsnprintf(error, size, format, arguments); va_end(arguments);
+ }
+ return -1;
+}
+
+static const void *ds_layer_data(const ColiDeepSeekV4LayerWeights *weights,
+ const char *suffix,
+ const ColiDeepSeekV4TensorSpec **spec) {
+ char name[COLI_V4_MAX_TENSOR_NAME];
+ snprintf(name, sizeof(name), "layers.%d.%s", weights->plan.layer, suffix);
+ return coli_v4_layer_data(weights, name, spec);
+}
+
+static int ds_fp8_view(ColiTensorView *view,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const char *prefix) {
+ char name[128]; const ColiDeepSeekV4TensorSpec *ws = NULL, *ss = NULL;
+ snprintf(name, sizeof(name), "%s.weight", prefix);
+ const void *data = ds_layer_data(weights, name, &ws);
+ snprintf(name, sizeof(name), "%s.scale", prefix);
+ const void *scales = ds_layer_data(weights, name, &ss);
+ if (!data || !scales || !ws || !ss || ws->rank != 2 || ss->rank != 2)
+ return -1;
+ memset(view, 0, sizeof(*view));
+ view->format = COLI_TENSOR_FP8_E4M3_BLOCK;
+ view->scale_format = COLI_SCALE_UE8M0;
+ view->data = data; view->scales = scales;
+ view->rows = ws->shape[0]; view->columns = ws->shape[1];
+ view->data_bytes = (size_t)view->rows * view->columns;
+ view->scale_bytes = (size_t)ss->shape[0] * ss->shape[1];
+ view->block_rows = 128; view->block_columns = 128;
+ return 0;
+}
+
+int coli_v4_dspark_attention_create(ColiV4DSparkAttentionState **output,
+ const ColiDeepSeekV4Config *config) {
+ if (!output || !config || config->sliding_window < 1 || config->head_dim < 1)
+ return -1;
+ *output = NULL;
+ ColiV4DSparkAttentionState *state = calloc(1, sizeof(*state));
+ if (!state) return -1;
+ state->window_size = config->sliding_window;
+ state->head_dim = config->head_dim;
+ state->kv = calloc((size_t)state->window_size * state->head_dim,
+ sizeof(*state->kv));
+ state->positions = malloc((size_t)state->window_size *
+ sizeof(*state->positions));
+ if (!state->kv || !state->positions) {
+ coli_v4_dspark_attention_destroy(state); return -1;
+ }
+ coli_v4_dspark_attention_reset(state); *output = state; return 0;
+}
+
+void coli_v4_dspark_attention_reset(ColiV4DSparkAttentionState *state) {
+ if (!state) return;
+ state->valid = 0;
+ memset(state->kv, 0, (size_t)state->window_size * state->head_dim *
+ sizeof(*state->kv));
+ for (int i = 0; i < state->window_size; i++) state->positions[i] = -1;
+}
+
+void coli_v4_dspark_attention_destroy(ColiV4DSparkAttentionState *state) {
+ if (!state) return;
+ free(state->positions); free(state->kv); free(state);
+}
+
+int coli_v4_dspark_attention_context_count(
+ const ColiV4DSparkAttentionState *state) { return state ? state->valid : 0; }
+
+int coli_v4_dspark_attention_precompute_context(
+ ColiV4DSparkAttentionState *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const float *main_x, int start_position, int batch,
+ char *error, size_t error_size) {
+ if (!state || !weights || !config || !main_x || start_position < 0 ||
+ batch < 1 || batch > 64 || state->head_dim != config->head_dim)
+ return ds_attn_error(error, error_size, "invalid DSpark context KV input");
+ ColiTensorView wkv;
+ if (ds_fp8_view(&wkv, weights, "attn.wkv"))
+ return ds_attn_error(error, error_size, "missing DSpark wkv");
+ int d = config->head_dim, rope = config->qk_rope_head_dim;
+ float *kv = malloc((size_t)batch * d * sizeof(*kv));
+ float *norm = malloc((size_t)d * sizeof(*norm));
+ int end = start_position + batch;
+ size_t pairs = (size_t)rope / 2;
+ float *cosines = malloc((size_t)end * pairs * sizeof(*cosines));
+ float *sines = malloc((size_t)end * pairs * sizeof(*sines));
+ const uint16_t *raw_norm = ds_layer_data(weights, "attn.kv_norm.weight", NULL);
+ if (!kv || !norm || !cosines || !sines || !raw_norm) {
+ free(sines); free(cosines); free(norm); free(kv);
+ return ds_attn_error(error, error_size, "out of memory in DSpark context KV");
+ }
+ for (int i = 0; i < d; i++) norm[i] = coli_bf16_decode(raw_norm[i]);
+ int result = coli_fp8_matmul_batch_ref(kv, &wkv, main_x, batch);
+ if (!result) coli_bf16_round_array(kv, (size_t)batch * d);
+ if (!result) result = coli_v4_rope_precompute(
+ cosines, sines, rope, end, 0, config->rope_theta,
+ config->rope_factor, config->rope_beta_fast, config->rope_beta_slow);
+ for (int item = 0; !result && item < batch; item++) {
+ float *item_kv = kv + (size_t)item * d;
+ result = coli_v4_rmsnorm(item_kv, item_kv, norm, d, config->rms_norm_eps);
+ if (result) break;
+ coli_bf16_round_array(item_kv, (size_t)d);
+ int position = start_position + item;
+ coli_v4_rope_apply(item_kv + d - rope, 1, rope,
+ cosines + (size_t)position * pairs,
+ sines + (size_t)position * pairs, 0);
+ coli_bf16_round_array(item_kv + d - rope, (size_t)rope);
+ size_t nope = (size_t)(d - rope);
+ float *qdq = malloc(nope * sizeof(*qdq));
+ uint8_t *scales = malloc((nope + 63) / 64);
+ if (!qdq || !scales || coli_fp8_activation_qdq_ref(
+ qdq, scales, item_kv, nope, 64)) result = -1;
+ if (!result) {
+ memcpy(item_kv, qdq, nope * sizeof(*item_kv));
+ coli_bf16_round_array(item_kv, nope);
+ int slot = coli_v4_dspark_position_window_put(
+ state->positions, state->window_size, &state->valid, position);
+ if (slot < 0)
+ result = ds_attn_error(
+ error, error_size,
+ "invalid DSpark context window update position=%d", position);
+ else
+ memcpy(state->kv + (size_t)slot * d, item_kv,
+ (size_t)d * sizeof(*item_kv));
+ }
+ free(scales); free(qdq);
+ }
+ free(sines); free(cosines); free(norm); free(kv);
+ return result ? ds_attn_error(error, error_size,
+ "DSpark context KV precompute failed") : 0;
+}
+/* ---- end inlined deepseek_v4_dspark_attention.c ---- */
+
+
+#include "deepseek_v4_dspark.h"
+#include "deepseek_v4_internal.h"
+
+int coli_v4_dspark_attention_block(
+ float *outputs, ColiV4DSparkAttentionState *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, const float *inputs,
+ int query_start_position, int batch, char *error, size_t error_size) {
+ if (!outputs || !state || !weights || !config || !inputs ||
+ query_start_position < 0 || batch < 1 || batch > 64)
+ return ds_attn_error(error, error_size, "invalid DSpark query block");
+ int hidden = config->hidden_size, heads = config->num_attention_heads;
+ int d = config->head_dim, rope = config->qk_rope_head_dim;
+ int qr = config->q_lora_rank, groups = config->o_groups;
+ int orank = config->o_lora_rank;
+ size_t qwidth = (size_t)heads * d;
+ size_t oawidth = (size_t)groups * orank;
+ ColiTensorView wqa, wqb, wkv, woa, wob;
+ if (ds_fp8_view(&wqa, weights, "attn.wq_a") ||
+ ds_fp8_view(&wqb, weights, "attn.wq_b") ||
+ ds_fp8_view(&wkv, weights, "attn.wkv") ||
+ ds_fp8_view(&woa, weights, "attn.wo_a") ||
+ ds_fp8_view(&wob, weights, "attn.wo_b"))
+ return ds_attn_error(error, error_size, "missing DSpark attention matrix");
+ float *qa = calloc((size_t)batch * qr, sizeof(float));
+ float *q = calloc((size_t)batch * qwidth, sizeof(float));
+ float *kv = calloc((size_t)batch * d, sizeof(float));
+ float *attended = calloc((size_t)batch * qwidth, sizeof(float));
+ float *oa = calloc((size_t)batch * oawidth, sizeof(float));
+ float *norm = malloc((size_t)(qr > d ? qr : d) * sizeof(float));
+ int end = query_start_position + batch; size_t pairs = (size_t)rope / 2;
+ float *cosines = malloc((size_t)end * pairs * sizeof(float));
+ float *sines = malloc((size_t)end * pairs * sizeof(float));
+ int context = state->valid, kv_count = context + batch;
+ float *all_kv = malloc((size_t)kv_count * d * sizeof(float));
+ int *indices = malloc((size_t)kv_count * sizeof(int));
+ if (!qa || !q || !kv || !attended || !oa || !norm || !cosines ||
+ !sines || !all_kv || !indices) {
+ free(indices); free(all_kv); free(sines); free(cosines); free(norm);
+ free(oa); free(attended); free(kv); free(q); free(qa);
+ return ds_attn_error(error, error_size, "out of memory in DSpark block");
+ }
+ int result = coli_fp8_matmul_batch_ref(qa, &wqa, inputs, batch);
+ if (result) ds_attn_error(error, error_size,
+ "DSpark attention q_a matmul failed");
+ if (!result) coli_bf16_round_array(qa, (size_t)batch * qr);
+ const uint16_t *qnorm = ds_layer_data(weights, "attn.q_norm.weight", NULL);
+ if (!qnorm) result = ds_attn_error(
+ error, error_size, "missing DSpark attention q_norm");
+ for (int i = 0; !result && i < qr; i++) norm[i] = coli_bf16_decode(qnorm[i]);
+ for (int item = 0; !result && item < batch; item++) {
+ float *row = qa + (size_t)item * qr;
+ result = coli_v4_rmsnorm(row, row, norm, qr, config->rms_norm_eps);
+ if (!result) coli_bf16_round_array(row, (size_t)qr);
+ }
+ if (!result) {
+ result = coli_fp8_matmul_batch_ref(q, &wqb, qa, batch);
+ if (result) ds_attn_error(error, error_size,
+ "DSpark attention q_b matmul failed");
+ }
+ if (!result) coli_bf16_round_array(q, (size_t)batch * qwidth);
+ for (int item = 0; !result && item < batch; item++)
+ for (int head = 0; head < heads; head++) {
+ float *row = q + (size_t)item * qwidth + (size_t)head * d;
+ float squares = 0.0f;
+ for (int i = 0; i < d; i++) squares += row[i] * row[i];
+ float scale = 1.0f / sqrtf(squares / d + config->rms_norm_eps);
+ for (int i = 0; i < d; i++) row[i] = coli_bf16_round(row[i] * scale);
+ }
+ if (!result) {
+ result = coli_fp8_matmul_batch_ref(kv, &wkv, inputs, batch);
+ if (result) ds_attn_error(error, error_size,
+ "DSpark attention query KV matmul failed");
+ }
+ if (!result) coli_bf16_round_array(kv, (size_t)batch * d);
+ const uint16_t *knorm = ds_layer_data(weights, "attn.kv_norm.weight", NULL);
+ if (!knorm) result = ds_attn_error(
+ error, error_size, "missing DSpark attention kv_norm");
+ for (int i = 0; !result && i < d; i++) norm[i] = coli_bf16_decode(knorm[i]);
+ if (!result) {
+ result = coli_v4_rope_precompute(
+ cosines, sines, rope, end, 0, config->rope_theta,
+ config->rope_factor, config->rope_beta_fast, config->rope_beta_slow);
+ if (result) ds_attn_error(error, error_size,
+ "DSpark attention RoPE precompute failed");
+ }
+ for (int item = 0; !result && item < batch; item++) {
+ int position = query_start_position + item;
+ float *item_kv = kv + (size_t)item * d;
+ result = coli_v4_rmsnorm(item_kv, item_kv, norm, d, config->rms_norm_eps);
+ if (result) break;
+ coli_bf16_round_array(item_kv, (size_t)d);
+ for (int head = 0; head < heads; head++) {
+ float *r = q + (size_t)item * qwidth + (size_t)head * d + d - rope;
+ coli_v4_rope_apply(r, 1, rope,
+ cosines + (size_t)position * pairs,
+ sines + (size_t)position * pairs, 0);
+ coli_bf16_round_array(r, (size_t)rope);
+ }
+ coli_v4_rope_apply(item_kv + d - rope, 1, rope,
+ cosines + (size_t)position * pairs,
+ sines + (size_t)position * pairs, 0);
+ coli_bf16_round_array(item_kv + d - rope, (size_t)rope);
+ size_t nope = (size_t)(d - rope);
+ float *qdq = malloc(nope * sizeof(float));
+ uint8_t *scales = malloc((nope + 63) / 64);
+ if (!qdq || !scales || coli_fp8_activation_qdq_ref(
+ qdq, scales, item_kv, nope, 64))
+ result = ds_attn_error(
+ error, error_size,
+ "DSpark attention query KV quantization failed item=%d", item);
+ if (!result) {
+ memcpy(item_kv, qdq, nope * sizeof(float));
+ coli_bf16_round_array(item_kv, nope);
+ }
+ free(scales); free(qdq);
+ }
+ int copied = 0;
+ for (int slot = 0; !result && slot < state->window_size; slot++)
+ if (state->positions[slot] >= 0) {
+ memcpy(all_kv + (size_t)copied * d,
+ state->kv + (size_t)slot * d, (size_t)d * sizeof(float));
+ copied++;
+ }
+ if (copied != context) result = ds_attn_error(
+ error, error_size,
+ "DSpark attention context state mismatch valid=%d copied=%d",
+ context, copied);
+ if (!result) memcpy(all_kv + (size_t)context * d, kv,
+ (size_t)batch * d * sizeof(float));
+ for (int i = 0; i < kv_count; i++) indices[i] = i;
+ const float *sinks = ds_layer_data(weights, "attn.attn_sink", NULL);
+ for (int item = 0; !result && item < batch; item++) {
+ result = coli_v4_sparse_attention_ref(
+ attended + (size_t)item * qwidth,
+ q + (size_t)item * qwidth, all_kv, sinks, indices,
+ heads, d, kv_count, kv_count, 1.0f / sqrtf((float)d));
+ if (result) ds_attn_error(
+ error, error_size,
+ "DSpark sparse attention failed item=%d position=%d context=%d",
+ item, query_start_position + item, context);
+ int position = query_start_position + item;
+ for (int head = 0; !result && head < heads; head++) {
+ float *r = attended + (size_t)item * qwidth +
+ (size_t)head * d + d - rope;
+ coli_v4_rope_apply(r, 1, rope,
+ cosines + (size_t)position * pairs,
+ sines + (size_t)position * pairs, 1);
+ coli_bf16_round_array(r, (size_t)rope);
+ }
+ }
+ int heads_per_group = heads / groups;
+ int group_width = heads_per_group * d;
+ int scale_columns = (hidden + 127) / 128;
+ int scale_rows = (orank + 127) / 128;
+ float *group_inputs = malloc((size_t)batch * group_width * sizeof(float));
+ float *group_outputs = malloc((size_t)batch * orank * sizeof(float));
+ if (!group_inputs || !group_outputs) result = ds_attn_error(
+ error, error_size, "out of memory in DSpark grouped output");
+ for (int group = 0; !result && group < groups; group++) {
+ for (int item = 0; item < batch; item++)
+ memcpy(group_inputs + (size_t)item * group_width,
+ attended + (size_t)item * qwidth +
+ (size_t)group * group_width,
+ (size_t)group_width * sizeof(float));
+ ColiTensorView view = woa;
+ view.rows = orank; view.columns = group_width;
+ view.data = (const uint8_t *)woa.data +
+ (size_t)group * orank * group_width;
+ view.scales = (const uint8_t *)woa.scales +
+ (size_t)group * scale_rows * scale_columns;
+ view.data_bytes = (size_t)orank * group_width;
+ view.scale_bytes = (size_t)scale_rows * scale_columns;
+ result = coli_fp8_matmul_batch_ref(group_outputs, &view,
+ group_inputs, batch);
+ if (result) ds_attn_error(
+ error, error_size,
+ "DSpark attention o_a matmul failed group=%d rows=%lld columns=%lld "
+ "data_bytes=%zu scale_bytes=%zu",
+ group, (long long)view.rows, (long long)view.columns,
+ view.data_bytes, view.scale_bytes);
+ for (int item = 0; !result && item < batch; item++)
+ memcpy(oa + (size_t)item * oawidth + (size_t)group * orank,
+ group_outputs + (size_t)item * orank,
+ (size_t)orank * sizeof(float));
+ }
+ if (!result) coli_bf16_round_array(oa, (size_t)batch * oawidth);
+ if (!result) {
+ result = coli_fp8_matmul_batch_ref(outputs, &wob, oa, batch);
+ if (result) ds_attn_error(error, error_size,
+ "DSpark attention o_b matmul failed");
+ }
+ if (!result) coli_bf16_round_array(outputs, (size_t)batch * hidden);
+ free(group_outputs); free(group_inputs); free(indices); free(all_kv);
+ free(sines); free(cosines); free(norm); free(oa); free(attended);
+ free(kv); free(q); free(qa);
+ if (!result) return 0;
+ if (error && error_size && error[0]) return -1;
+ return ds_attn_error(error, error_size,
+ "DSpark non-causal attention failed");
+}
+#endif /* COLI_V4_UNIT_DSPARK_ATTENTION */
+
+#ifdef COLI_V4_UNIT_DSPARK_BLOCK
+/* ######## deepseek_v4_dspark_block.c ######## */
+#define coli_v4_block_window_batch_ref coli_v4_dspark_causal_block_unused
+/* ---- begin include deepseek_v4_block_batch.c ---- */
+#define coli_v4_block_token_ref coli_v4_block_token_batch_serial_ref
+#define coli_v4_block_window_token_ref coli_v4_block_window_token_batch_serial_ref
+/* ---- begin include deepseek_v4_block.c ---- */
+#include "deepseek_v4_internal.h"
+
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "native_quant.h"
+
+static int set_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list arguments;
+ va_start(arguments, format);
+ vsnprintf(error, size, format, arguments);
+ va_end(arguments);
+ }
+ return -1;
+}
+
+static const void *value(const ColiDeepSeekV4LayerWeights *weights,
+ const char *suffix,
+ const ColiDeepSeekV4TensorSpec **spec) {
+ char name[COLI_V4_MAX_TENSOR_NAME];
+ snprintf(name, sizeof(name), "layers.%d.%s", weights->plan.layer, suffix);
+ return coli_v4_layer_data(weights, name, spec);
+}
+
+static int fp8_view(ColiTensorView *view,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const char *prefix) {
+ char name[128];
+ const ColiDeepSeekV4TensorSpec *ws = NULL, *ss = NULL;
+ snprintf(name, sizeof(name), "%s.weight", prefix);
+ const void *data = value(weights, name, &ws);
+ snprintf(name, sizeof(name), "%s.scale", prefix);
+ const void *scales = value(weights, name, &ss);
+ if (!data || !scales || !ws || !ss || ws->rank != 2) return -1;
+ *view = (ColiTensorView){
+ COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_UE8M0, data, scales,
+ (size_t)(ws->shape[0] * ws->shape[1]),
+ (size_t)(ss->shape[0] * ss->shape[1]),
+ ws->shape[0], ws->shape[1], 128, 128
+ };
+ return 0;
+}
+
+static void decode_bf16(float *output, const uint16_t *input, size_t count) {
+ for (size_t i = 0; i < count; i++) output[i] = coli_bf16_decode(input[i]);
+}
+
+static int normalized_hc_pre(float *reduced, float *post, float *comb,
+ float *normalized, const float *input_hc,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const char *branch, const char *norm_name) {
+ char name[64];
+ snprintf(name, sizeof(name), "hc_%s_fn", branch);
+ const float *function = value(weights, name, NULL);
+ snprintf(name, sizeof(name), "hc_%s_scale", branch);
+ const float *scale = value(weights, name, NULL);
+ snprintf(name, sizeof(name), "hc_%s_base", branch);
+ const float *base = value(weights, name, NULL);
+ const uint16_t *raw_norm = value(weights, norm_name, NULL);
+ int d = config->hidden_size;
+ float *norm = malloc((size_t)d * sizeof(*norm));
+ if (!function || !scale || !base || !raw_norm || !norm) {
+ free(norm);
+ return -1;
+ }
+ decode_bf16(norm, raw_norm, (size_t)d);
+ int result = coli_v4_hc_pre(reduced, post, comb, input_hc, function,
+ scale, base, config->hc_mult, d,
+ config->hc_sinkhorn_iters,
+ config->rms_norm_eps, config->hc_eps);
+ if (!result) {
+ coli_bf16_round_array(reduced, (size_t)d);
+ result = coli_v4_rmsnorm(normalized, reduced, norm, d,
+ config->rms_norm_eps);
+ coli_bf16_round_array(normalized, (size_t)d);
+ }
+ free(norm);
+ return result;
+}
+
+static int moe_token(float *output,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ ColiExpertStore *store, const float *input, int token) {
+ int d = config->hidden_size;
+ int n = config->n_routed_experts;
+ int topk = config->num_experts_per_tok;
+ size_t gate_count = (size_t)n * d;
+ float *gate = malloc(gate_count * sizeof(*gate));
+ float *route_weights = malloc((size_t)topk * sizeof(*route_weights));
+ int *indices = malloc((size_t)topk * sizeof(*indices));
+ float *expert_output = malloc((size_t)d * sizeof(*expert_output));
+ float *shared_output = malloc((size_t)d * sizeof(*shared_output));
+ if (!gate || !route_weights || !indices || !expert_output || !shared_output) {
+ free(shared_output); free(expert_output); free(indices);
+ free(route_weights); free(gate);
+ return -1;
+ }
+ decode_bf16(gate, value(weights, "ffn.gate.weight", NULL), gate_count);
+ const int64_t *table = value(weights, "ffn.gate.tid2eid", NULL);
+ const float *bias = value(weights, "ffn.gate.bias", NULL);
+ int result = token < 0 || token >= config->vocab_size;
+ if (!result && weights->plan.uses_hash_router) {
+ if (!table) result = -1;
+ }
+ if (!result && weights->plan.uses_hash_router) {
+ for (int i = 0; i < topk; i++)
+ indices[i] = (int)table[(size_t)token * topk + i];
+ }
+ if (!result) result = coli_v4_route(
+ route_weights, indices, input, gate, bias,
+ weights->plan.uses_hash_router ? indices : NULL,
+ n, d, topk, config->routed_scaling_factor);
+
+ ColiTensorView w1, w2, w3;
+ if (!result && (fp8_view(&w1, weights, "ffn.shared_experts.w1") ||
+ fp8_view(&w2, weights, "ffn.shared_experts.w2") ||
+ fp8_view(&w3, weights, "ffn.shared_experts.w3"))) result = -1;
+ if (!result) result = coli_v4_shared_expert_forward_ref(
+ shared_output, &w1, &w2, &w3, input, config->swiglu_limit);
+ if (!result) memset(output, 0, (size_t)d * sizeof(*output));
+ for (int expert_id = 0; !result && expert_id < n; expert_id++) {
+ int rank = -1;
+ for (int candidate = 0; candidate < topk; candidate++)
+ if (indices[candidate] == expert_id) rank = candidate;
+ if (rank < 0) continue;
+ ColiExpertView expert;
+ if (coli_expert_lookup(store,
+ (ColiExpertKey){weights->plan.layer, expert_id},
+ &expert)) {
+ result = -1;
+ break;
+ }
+ result = coli_v4_expert_forward_ref(expert_output, &expert, input,
+ route_weights[rank],
+ config->swiglu_limit);
+ coli_expert_release(store, &expert);
+ if (!result)
+ for (int i = 0; i < d; i++) output[i] += expert_output[i];
+ }
+ if (!result)
+ for (int i = 0; i < d; i++)
+ output[i] = coli_bf16_round(output[i] + shared_output[i]);
+ free(shared_output); free(expert_output); free(indices);
+ free(route_weights); free(gate);
+ return result;
+}
+
+static int block_token_impl(float *output_hc,
+ ColiDeepSeekV4WindowAttentionState *attention,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ ColiExpertStore *experts,
+ const float *input_hc, int token, int position,
+ char *error, size_t error_size) {
+ if (!output_hc || !weights || !config || !experts || !input_hc)
+ return set_error(error, error_size, "invalid block arguments");
+ int d = config->hidden_size, hc = config->hc_mult;
+ size_t hd = (size_t)hc * d;
+ float *residual = malloc(hd * sizeof(*residual));
+ float *state = malloc(hd * sizeof(*state));
+ float *reduced = malloc((size_t)d * sizeof(*reduced));
+ float *normalized = malloc((size_t)d * sizeof(*normalized));
+ float *branch = malloc((size_t)d * sizeof(*branch));
+ float *post = malloc((size_t)hc * sizeof(*post));
+ float *comb = malloc((size_t)hc * hc * sizeof(*comb));
+ if (!residual || !state || !reduced || !normalized || !branch || !post || !comb) {
+ free(comb); free(post); free(branch); free(normalized);
+ free(reduced); free(state); free(residual);
+ return set_error(error, error_size, "out of memory in block");
+ }
+ memcpy(residual, input_hc, hd * sizeof(*residual));
+ int result = normalized_hc_pre(reduced, post, comb, normalized, input_hc,
+ weights, config, "attn", "attn_norm.weight");
+ if (!result) result = attention
+ ? coli_v4_attention_window_token_ref(branch, attention, weights, config,
+ normalized, position, error, error_size)
+ : coli_v4_attention_token_ref(branch, weights, config, normalized,
+ position, error, error_size);
+ if (!result) result = coli_v4_hc_post(state, branch, residual, post, comb, hc, d);
+ if (!result) coli_bf16_round_array(state, hd);
+
+ if (!result) memcpy(residual, state, hd * sizeof(*residual));
+ if (!result) result = normalized_hc_pre(reduced, post, comb, normalized, state,
+ weights, config, "ffn", "ffn_norm.weight");
+ if (!result) result = moe_token(branch, weights, config, experts, normalized, token);
+ if (!result) result = coli_v4_hc_post(output_hc, branch, residual, post, comb, hc, d);
+ if (!result) coli_bf16_round_array(output_hc, hd);
+
+ free(comb); free(post); free(branch); free(normalized);
+ free(reduced); free(state); free(residual);
+ return result ? set_error(error, error_size, "block computation failed") : 0;
+}
+
+int coli_v4_block_token_ref(float *output_hc,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ ColiExpertStore *experts,
+ const float *input_hc, int token, int position,
+ char *error, size_t error_size) {
+ return block_token_impl(output_hc, NULL, weights, config, experts, input_hc,
+ token, position, error, error_size);
+}
+
+int coli_v4_block_window_token_ref(
+ float *output_hc, ColiDeepSeekV4WindowAttentionState *attention,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, ColiExpertStore *experts,
+ const float *input_hc, int token, int position,
+ char *error, size_t error_size) {
+ return block_token_impl(output_hc, attention, weights, config, experts,
+ input_hc, token, position, error, error_size);
+}
+/* ---- end include deepseek_v4_block.c ---- */
+
+#undef coli_v4_block_token_ref
+#undef coli_v4_block_window_token_ref
+
+#include "deepseek_v4_internal.h"
+#include "native_quant_batch.h"
+
+static int shared_expert_batch(float *outputs, const ColiTensorView *w1,
+ const ColiTensorView *w2,
+ const ColiTensorView *w3,
+ const float *inputs, int batch,
+ float swiglu_limit) {
+ int intermediate = (int)w1->rows, d = (int)w2->rows;
+ float *gate = malloc((size_t)batch * intermediate * sizeof(*gate));
+ float *up = malloc((size_t)batch * intermediate * sizeof(*up));
+ float *activated = malloc((size_t)batch * intermediate * sizeof(*activated));
+ if (!gate || !up || !activated) {
+ free(activated); free(up); free(gate); return -1;
+ }
+ int result = coli_fp8_matmul_batch_ref(gate, w1, inputs, batch) ||
+ coli_fp8_matmul_batch_ref(up, w3, inputs, batch);
+ for (int item = 0; !result && item < batch; item++) {
+ float *item_gate = gate + (size_t)item * intermediate;
+ float *item_up = up + (size_t)item * intermediate;
+ float *item_activated = activated + (size_t)item * intermediate;
+ coli_bf16_round_array(item_gate, (size_t)intermediate);
+ coli_bf16_round_array(item_up, (size_t)intermediate);
+ result = coli_v4_swiglu(item_activated, item_gate, item_up,
+ intermediate, swiglu_limit);
+ if (!result) coli_bf16_round_array(item_activated,
+ (size_t)intermediate);
+ }
+ if (!result) result = coli_fp8_matmul_batch_ref(
+ outputs, w2, activated, batch);
+ if (!result) coli_bf16_round_array(outputs, (size_t)batch * d);
+ free(activated); free(up); free(gate);
+ return result;
+}
+
+static int routed_expert_batch(float *outputs, const ColiExpertView *expert,
+ const float *inputs, const float *route_weights,
+ int batch, float swiglu_limit) {
+ int intermediate = (int)expert->gate.rows;
+ int d = (int)expert->down.rows;
+ float *gate = malloc((size_t)batch * intermediate * sizeof(*gate));
+ float *up = malloc((size_t)batch * intermediate * sizeof(*up));
+ float *activated = malloc((size_t)batch * intermediate * sizeof(*activated));
+ if (!gate || !up || !activated) {
+ free(activated); free(up); free(gate); return -1;
+ }
+ int result = coli_fp4_matmul_batch_ref(gate, &expert->gate, inputs, batch) ||
+ coli_fp4_matmul_batch_ref(up, &expert->up, inputs, batch);
+ for (int item = 0; !result && item < batch; item++) {
+ float *item_gate = gate + (size_t)item * intermediate;
+ float *item_up = up + (size_t)item * intermediate;
+ float *item_activated = activated + (size_t)item * intermediate;
+ coli_bf16_round_array(item_gate, (size_t)intermediate);
+ coli_bf16_round_array(item_up, (size_t)intermediate);
+ result = coli_v4_swiglu(item_activated, item_gate, item_up,
+ intermediate, swiglu_limit);
+ for (int i = 0; !result && i < intermediate; i++)
+ item_activated[i] = coli_bf16_round(
+ item_activated[i] * route_weights[item]);
+ }
+ if (!result) result = coli_fp4_matmul_batch_ref(
+ outputs, &expert->down, activated, batch);
+ if (!result) coli_bf16_round_array(outputs, (size_t)batch * d);
+ free(activated); free(up); free(gate);
+ return result;
+}
+
+static int moe_batch(float *outputs,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ ColiExpertStore *store, const float *inputs,
+ const int *tokens, int batch,
+ char *error, size_t error_size) {
+ int d = config->hidden_size, n = config->n_routed_experts;
+ int topk = config->num_experts_per_tok;
+ float *gate = malloc((size_t)n * d * sizeof(*gate));
+ float *route_weights = malloc((size_t)batch * topk * sizeof(*route_weights));
+ int *indices = malloc((size_t)batch * topk * sizeof(*indices));
+ float *shared = malloc((size_t)batch * d * sizeof(*shared));
+ float *compact_inputs = malloc((size_t)batch * d * sizeof(*compact_inputs));
+ float *compact_outputs = malloc((size_t)batch * d * sizeof(*compact_outputs));
+ float *compact_weights = malloc((size_t)batch * sizeof(*compact_weights));
+ int *compact_items = malloc((size_t)batch * sizeof(*compact_items));
+ if (!gate || !route_weights || !indices || !shared || !compact_inputs ||
+ !compact_outputs || !compact_weights || !compact_items) {
+ free(compact_items); free(compact_weights); free(compact_outputs);
+ free(compact_inputs); free(shared); free(indices); free(route_weights);
+ free(gate);
+ return set_error(error, error_size,
+ "out of memory in DSpark MoE scratch");
+ }
+ const uint16_t *raw_gate = value(weights, "ffn.gate.weight", NULL);
+ if (!raw_gate) {
+ free(compact_items); free(compact_weights); free(compact_outputs);
+ free(compact_inputs); free(shared); free(indices); free(route_weights);
+ free(gate);
+ return set_error(error, error_size, "missing DSpark MoE gate weight");
+ }
+ decode_bf16(gate, raw_gate, (size_t)n * d);
+ const int64_t *table = value(weights, "ffn.gate.tid2eid", NULL);
+ const float *bias = value(weights, "ffn.gate.bias", NULL);
+ int result = 0;
+ for (int item = 0; !result && item < batch; item++) {
+ int *item_indices = indices + (size_t)item * topk;
+ float *item_weights = route_weights + (size_t)item * topk;
+ if (tokens[item] < 0 || tokens[item] >= config->vocab_size)
+ result = set_error(
+ error, error_size,
+ "DSpark MoE invalid token item=%d token=%d vocab=%d",
+ item, tokens[item], config->vocab_size);
+ if (!result && weights->plan.uses_hash_router) {
+ if (!table) result = set_error(
+ error, error_size, "missing DSpark MoE hash routing table");
+ else for (int i = 0; i < topk; i++)
+ item_indices[i] = (int)table[(size_t)tokens[item] * topk + i];
+ }
+ if (!result && coli_v4_route(
+ item_weights, item_indices, inputs + (size_t)item * d,
+ gate, bias, weights->plan.uses_hash_router ? item_indices : NULL,
+ n, d, topk, config->routed_scaling_factor))
+ result = set_error(error, error_size,
+ "DSpark MoE routing failed item=%d token=%d",
+ item, tokens[item]);
+ }
+ ColiTensorView w1, w2, w3;
+ if (!result && fp8_view(&w1, weights, "ffn.shared_experts.w1"))
+ result = set_error(error, error_size,
+ "invalid DSpark shared expert w1");
+ if (!result && fp8_view(&w2, weights, "ffn.shared_experts.w2"))
+ result = set_error(error, error_size,
+ "invalid DSpark shared expert w2");
+ if (!result && fp8_view(&w3, weights, "ffn.shared_experts.w3"))
+ result = set_error(error, error_size,
+ "invalid DSpark shared expert w3");
+ if (!result && shared_expert_batch(
+ shared, &w1, &w2, &w3, inputs, batch, config->swiglu_limit))
+ result = set_error(error, error_size,
+ "DSpark shared expert batch failed");
+ if (!result) memset(outputs, 0, (size_t)batch * d * sizeof(*outputs));
+
+ for (int expert_id = 0; !result && expert_id < n; expert_id++) {
+ int count = 0;
+ for (int item = 0; item < batch; item++) {
+ for (int rank = 0; rank < topk; rank++) {
+ if (indices[(size_t)item * topk + rank] == expert_id) {
+ compact_items[count] = item;
+ compact_weights[count] =
+ route_weights[(size_t)item * topk + rank];
+ memcpy(compact_inputs + (size_t)count * d,
+ inputs + (size_t)item * d, (size_t)d * sizeof(float));
+ count++;
+ }
+ }
+ }
+ if (!count) continue;
+ ColiExpertView expert;
+ if (coli_expert_lookup(store,
+ (ColiExpertKey){weights->plan.layer, expert_id},
+ &expert)) {
+ result = set_error(
+ error, error_size,
+ "DSpark expert lookup failed layer=%d expert=%d count=%d",
+ weights->plan.layer, expert_id, count);
+ break;
+ }
+ if (routed_expert_batch(compact_outputs, &expert, compact_inputs,
+ compact_weights, count,
+ config->swiglu_limit))
+ result = set_error(
+ error, error_size,
+ "DSpark routed expert batch failed layer=%d expert=%d count=%d",
+ weights->plan.layer, expert_id, count);
+ coli_expert_release(store, &expert);
+ for (int compact = 0; !result && compact < count; compact++) {
+ float *destination = outputs + (size_t)compact_items[compact] * d;
+ const float *source = compact_outputs + (size_t)compact * d;
+ for (int i = 0; i < d; i++) destination[i] += source[i];
+ }
+ }
+ for (int item = 0; !result && item < batch; item++)
+ for (int i = 0; i < d; i++)
+ outputs[(size_t)item * d + i] = coli_bf16_round(
+ outputs[(size_t)item * d + i] + shared[(size_t)item * d + i]);
+ free(compact_items); free(compact_weights); free(compact_outputs);
+ free(compact_inputs); free(shared); free(indices); free(route_weights);
+ free(gate); return result;
+}
+
+int coli_v4_block_window_batch_ref(
+ float *outputs_hc, ColiDeepSeekV4WindowAttentionState *attention,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, ColiExpertStore *experts,
+ const float *inputs_hc, const int *tokens, int start_position, int batch,
+ char *error, size_t error_size) {
+ if (!outputs_hc || !attention || !weights || !config || !experts ||
+ !inputs_hc || !tokens || batch < 1 || batch > 64) return -1;
+ int d = config->hidden_size, hc = config->hc_mult;
+ size_t hd = (size_t)hc * d;
+ float *states = malloc((size_t)batch * hd * sizeof(*states));
+ float *normalized_ffn = malloc((size_t)batch * d * sizeof(*normalized_ffn));
+ float *branches = malloc((size_t)batch * d * sizeof(*branches));
+ float *posts = malloc((size_t)batch * hc * sizeof(*posts));
+ float *combs = malloc((size_t)batch * hc * hc * sizeof(*combs));
+ float *residual = malloc(hd * sizeof(*residual));
+ float *reduced = malloc((size_t)d * sizeof(*reduced));
+ float *normalized = malloc((size_t)d * sizeof(*normalized));
+ float *branch = malloc((size_t)d * sizeof(*branch));
+ float *post = malloc((size_t)hc * sizeof(*post));
+ float *comb = malloc((size_t)hc * hc * sizeof(*comb));
+ if (!states || !normalized_ffn || !branches || !posts || !combs ||
+ !residual || !reduced || !normalized || !branch || !post || !comb) {
+ free(comb); free(post); free(branch); free(normalized); free(reduced);
+ free(residual); free(combs); free(posts); free(branches);
+ free(normalized_ffn); free(states); return -1;
+ }
+ int result = 0;
+ for (int item = 0; !result && item < batch; item++) {
+ const float *input = inputs_hc + (size_t)item * hd;
+ float *state = states + (size_t)item * hd;
+ memcpy(residual, input, hd * sizeof(float));
+ result = normalized_hc_pre(reduced, post, comb, normalized, input,
+ weights, config, "attn", "attn_norm.weight");
+ if (!result) result = coli_v4_attention_window_token_ref(
+ branch, attention, weights, config, normalized,
+ start_position + item, error, error_size);
+ if (!result) result = coli_v4_hc_post(state, branch, residual,
+ post, comb, hc, d);
+ if (!result) coli_bf16_round_array(state, hd);
+ if (!result) result = normalized_hc_pre(
+ reduced, posts + (size_t)item * hc,
+ combs + (size_t)item * hc * hc,
+ normalized_ffn + (size_t)item * d, state,
+ weights, config, "ffn", "ffn_norm.weight");
+ }
+ if (!result) result = moe_batch(branches, weights, config, experts,
+ normalized_ffn, tokens, batch,
+ error, error_size);
+ for (int item = 0; !result && item < batch; item++) {
+ result = coli_v4_hc_post(
+ outputs_hc + (size_t)item * hd,
+ branches + (size_t)item * d,
+ states + (size_t)item * hd,
+ posts + (size_t)item * hc,
+ combs + (size_t)item * hc * hc, hc, d);
+ if (!result) coli_bf16_round_array(
+ outputs_hc + (size_t)item * hd, hd);
+ }
+ free(comb); free(post); free(branch); free(normalized); free(reduced);
+ free(residual); free(combs); free(posts); free(branches);
+ free(normalized_ffn); free(states);
+ return result ? set_error(error, error_size, "batched block failed") : 0;
+}
+/* ---- end include deepseek_v4_block_batch.c ---- */
+
+#undef coli_v4_block_window_batch_ref
+
+#include "deepseek_v4_dspark.h"
+#include "deepseek_v4_dspark.h"
+
+int coli_v4_dspark_block(
+ float *outputs_hc, ColiV4DSparkAttentionState *attention,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, ColiExpertStore *experts,
+ const float *inputs_hc, const int *tokens,
+ int query_start_position, int batch,
+ char *error, size_t error_size) {
+ if (error && error_size) error[0] = '\0';
+ if (!outputs_hc || !attention || !weights || !config || !experts ||
+ !inputs_hc || !tokens || batch < 1 || batch > 64)
+ return set_error(error, error_size, "invalid DSpark block arguments");
+ int d = config->hidden_size, hc = config->hc_mult;
+ size_t hd = (size_t)hc * d;
+ float *states = malloc((size_t)batch * hd * sizeof(float));
+ float *attn_norm = malloc((size_t)batch * d * sizeof(float));
+ float *attn_branch = malloc((size_t)batch * d * sizeof(float));
+ float *attn_post = malloc((size_t)batch * hc * sizeof(float));
+ float *attn_comb = malloc((size_t)batch * hc * hc * sizeof(float));
+ float *ffn_norm = malloc((size_t)batch * d * sizeof(float));
+ float *ffn_branch = malloc((size_t)batch * d * sizeof(float));
+ float *ffn_post = malloc((size_t)batch * hc * sizeof(float));
+ float *ffn_comb = malloc((size_t)batch * hc * hc * sizeof(float));
+ float *reduced = malloc((size_t)d * sizeof(float));
+ if (!states || !attn_norm || !attn_branch || !attn_post || !attn_comb ||
+ !ffn_norm || !ffn_branch || !ffn_post || !ffn_comb || !reduced) {
+ free(reduced); free(ffn_comb); free(ffn_post); free(ffn_branch);
+ free(ffn_norm); free(attn_comb); free(attn_post); free(attn_branch);
+ free(attn_norm); free(states);
+ return set_error(error, error_size,
+ "out of memory in DSpark block scratch");
+ }
+ int result = 0;
+ const char *failed_step = NULL;
+ for (int item = 0; !result && item < batch; item++) {
+ result = normalized_hc_pre(
+ reduced, attn_post + (size_t)item * hc,
+ attn_comb + (size_t)item * hc * hc,
+ attn_norm + (size_t)item * d,
+ inputs_hc + (size_t)item * hd,
+ weights, config, "attn", "attn_norm.weight");
+ if (result) failed_step = "attention HC pre";
+ }
+ if (!result) {
+ result = coli_v4_dspark_attention_block(
+ attn_branch, attention, weights, config, attn_norm,
+ query_start_position, batch, error, error_size);
+ if (result) failed_step = "attention";
+ }
+ for (int item = 0; !result && item < batch; item++) {
+ result = coli_v4_hc_post(
+ states + (size_t)item * hd,
+ attn_branch + (size_t)item * d,
+ inputs_hc + (size_t)item * hd,
+ attn_post + (size_t)item * hc,
+ attn_comb + (size_t)item * hc * hc, hc, d);
+ if (result) failed_step = "attention HC post";
+ if (!result) coli_bf16_round_array(states + (size_t)item * hd, hd);
+ }
+ for (int item = 0; !result && item < batch; item++) {
+ result = normalized_hc_pre(
+ reduced, ffn_post + (size_t)item * hc,
+ ffn_comb + (size_t)item * hc * hc,
+ ffn_norm + (size_t)item * d,
+ states + (size_t)item * hd,
+ weights, config, "ffn", "ffn_norm.weight");
+ if (result) failed_step = "FFN HC pre";
+ }
+ if (!result) {
+ result = moe_batch(ffn_branch, weights, config, experts, ffn_norm,
+ tokens, batch, error, error_size);
+ if (result) failed_step = "MoE";
+ }
+ for (int item = 0; !result && item < batch; item++) {
+ result = coli_v4_hc_post(
+ outputs_hc + (size_t)item * hd,
+ ffn_branch + (size_t)item * d,
+ states + (size_t)item * hd,
+ ffn_post + (size_t)item * hc,
+ ffn_comb + (size_t)item * hc * hc, hc, d);
+ if (result) failed_step = "FFN HC post";
+ if (!result) coli_bf16_round_array(outputs_hc + (size_t)item * hd, hd);
+ }
+ free(reduced); free(ffn_comb); free(ffn_post); free(ffn_branch);
+ free(ffn_norm); free(attn_comb); free(attn_post); free(attn_branch);
+ free(attn_norm); free(states);
+ if (!result) return 0;
+ char detail[256] = {0};
+ if (error && error_size && error[0])
+ snprintf(detail, sizeof(detail), "%s", error);
+ return set_error(error, error_size,
+ "DSpark block failed layer=%d step=%s%s%s",
+ weights->plan.layer, failed_step ? failed_step : "unknown",
+ detail[0] ? ": " : "", detail);
+}
+#endif /* COLI_V4_UNIT_DSPARK_BLOCK */
+
+#ifdef COLI_V4_UNIT_DSPARK_FINAL
+/* ######## deepseek_v4_dspark_final.c ######## */
+#include "native_quant.h"
+/* ---- begin inlined deepseek_v4_dspark_final.c ---- */
+#include "deepseek_v4_dspark.h"
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "deepseek_v4_internal.h"
+#include "safetensors_index.h"
+#include "tensor_io.h"
+
+struct ColiV4DSparkFinal {
+ int hidden, hc;
+ float eps, hc_eps;
+ ColiFloatTensor function, base, scale, norm;
+};
+
+void coli_v4_dspark_final_close(ColiV4DSparkFinal *final) {
+ if (!final) return;
+ coli_float_tensor_free(&final->norm);
+ coli_float_tensor_free(&final->scale);
+ coli_float_tensor_free(&final->base);
+ coli_float_tensor_free(&final->function);
+ free(final);
+}
+
+int coli_v4_dspark_final_open(ColiV4DSparkFinal **output,
+ const char *model_dir,
+ const ColiDeepSeekV4Config *config,
+ const ColiDeepSeekV4DSparkManifest *manifest,
+ char *error, size_t error_size) {
+ if (!output || !model_dir || !config || !manifest ||
+ manifest->stage_count < 1) return -1;
+ *output = NULL; ColiSafetensorsIndex *index = NULL;
+ if (coli_st_index_open(&index, model_dir, error, error_size)) return -1;
+ ColiV4DSparkFinal *final = calloc(1, sizeof(*final));
+ if (!final) { coli_st_index_close(index); return -1; }
+ final->hidden = config->hidden_size; final->hc = config->hc_mult;
+ final->eps = config->rms_norm_eps; final->hc_eps = config->hc_eps;
+ char name[160]; int last = manifest->stage_count - 1, failed = 0;
+ snprintf(name, sizeof(name), "mtp.%d.hc_head_fn", last);
+ failed |= coli_tensor_load_f32(&final->function, index, name,
+ error, error_size);
+ snprintf(name, sizeof(name), "mtp.%d.hc_head_base", last);
+ failed |= coli_tensor_load_f32(&final->base, index, name, error, error_size);
+ snprintf(name, sizeof(name), "mtp.%d.hc_head_scale", last);
+ failed |= coli_tensor_load_f32(&final->scale, index, name, error, error_size);
+ snprintf(name, sizeof(name), "mtp.%d.norm.weight", last);
+ failed |= coli_tensor_load_f32(&final->norm, index, name, error, error_size);
+ coli_st_index_close(index);
+ if (failed) { coli_v4_dspark_final_close(final); return -1; }
+ *output = final; return 0;
+}
+
+int coli_v4_dspark_final_hidden(ColiV4DSparkFinal *final,
+ float *outputs, const float *states_hc,
+ int batch) {
+ if (!final || !outputs || !states_hc || batch < 1 || batch > 64) return -1;
+ int d = final->hidden, hc = final->hc, flattened = d * hc;
+ if (hc > 16) return -1;
+ for (int item = 0; item < batch; item++) {
+ const float *state = states_hc + (size_t)item * flattened;
+ float *output = outputs + (size_t)item * d;
+ float square = 0.0f, pre[16];
+ for (int i = 0; i < flattened; i++) square += state[i] * state[i];
+ float inverse = 1.0f / sqrtf(square / flattened + final->eps);
+ for (int copy = 0; copy < hc; copy++) {
+ float mix = 0.0f;
+ for (int i = 0; i < flattened; i++)
+ mix += final->function.data[(size_t)copy * flattened + i] *
+ state[i];
+ float z = mix * inverse * final->scale.data[0] +
+ final->base.data[copy];
+ float sigmoid = z >= 0.0f ? 1.0f / (1.0f + expf(-z))
+ : expf(z) / (1.0f + expf(z));
+ pre[copy] = sigmoid + final->hc_eps;
+ }
+ for (int i = 0; i < d; i++) {
+ float value = 0.0f;
+ for (int copy = 0; copy < hc; copy++)
+ value += pre[copy] * state[(size_t)copy * d + i];
+ output[i] = coli_bf16_round(value);
+ }
+ if (coli_v4_rmsnorm(output, output, final->norm.data, d, final->eps))
+ return -1;
+ coli_bf16_round_array(output, (size_t)d);
+ }
+ return 0;
+}
+/* ---- end inlined deepseek_v4_dspark_final.c ---- */
+#endif /* COLI_V4_UNIT_DSPARK_FINAL */
+
+#ifdef COLI_V4_UNIT_DSPARK_RUNNER
+/* ######## deepseek_v4_dspark_runner.c ######## */
+#include "native_quant.h"
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_dspark.h"
+
+#define coli_v4_dspark_runner_open coli_v4_dspark_runner_owned_open
+#define coli_v4_dspark_runner_close coli_v4_dspark_runner_owned_close
+/* ---- begin inlined deepseek_v4_dspark_runner.c ---- */
+#include "deepseek_v4_dspark.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "deepseek_v4_dspark.h"
+#include "deepseek_v4_dspark.h"
+#include "deepseek_v4_dspark.h"
+#include "deepseek_v4_dspark.h"
+#include "deepseek_v4_dspark.h"
+#include "deepseek_v4_dspark.h"
+#include "deepseek_v4_internal.h"
+#include "safetensors_index.h"
+
+struct ColiV4DSparkRunner {
+ ColiDeepSeekV4Config config;
+ ColiDeepSeekV4DSparkManifest manifest;
+ char *model_dir;
+ ColiSafetensorsIndex *dspark_index;
+ ColiSafetensorsIndex *target_index;
+ ColiV4DSparkHeads *heads;
+ ColiV4DSparkFinal *final;
+ ColiExpertStore *experts;
+ ColiV4DSparkAttentionState *attention[COLI_V4_DSPARK_MAX_STAGES];
+ uint64_t loaded_stage_peak;
+ ColiV4Engine *engine;
+};
+
+static int runner_error(char *error, size_t size, const char *message) {
+ if (error && size) snprintf(error, size, "%s", message);
+ return -1;
+}
+
+static int runner_embedding(ColiV4DSparkRunner *runner, float *state,
+ int token) {
+ const ColiSafetensorsTensor *embed =
+ coli_st_find(runner->dspark_index, "embed.weight");
+ int d = runner->config.hidden_size, hc = runner->config.hc_mult;
+ uint16_t *raw = malloc((size_t)d * sizeof(*raw));
+ if (!embed || embed->dtype != COLI_ST_BF16 || !raw || token < 0 ||
+ token >= runner->config.vocab_size ||
+ coli_st_read_at(runner->dspark_index, embed->shard,
+ embed->offset + (uint64_t)token * d * sizeof(*raw),
+ (size_t)d * sizeof(*raw), raw)) {
+ free(raw); return -1;
+ }
+ for (int copy = 0; copy < hc; copy++)
+ for (int i = 0; i < d; i++)
+ state[(size_t)copy * d + i] = coli_bf16_decode(raw[i]);
+ free(raw); return 0;
+}
+
+void coli_v4_dspark_runner_close(ColiV4DSparkRunner *runner) {
+ if (!runner) return;
+ for (int stage = 0; stage < runner->manifest.stage_count; stage++)
+ coli_v4_dspark_attention_destroy(runner->attention[stage]);
+ if (runner->experts) runner->experts->ops->destroy(runner->experts);
+ coli_v4_dspark_final_close(runner->final);
+ coli_v4_dspark_heads_close(runner->heads);
+ coli_st_index_close(runner->target_index);
+ coli_st_index_close(runner->dspark_index);
+ free(runner->model_dir); free(runner);
+}
+
+int coli_v4_dspark_runner_open(ColiV4DSparkRunner **output,
+ const char *dspark_model_dir,
+ const char *target_model_dir,
+ const ColiDeepSeekV4Config *config,
+ uint64_t expert_cache_bytes,
+ char *error, size_t error_size) {
+ if (!output || !dspark_model_dir || !target_model_dir || !config) return -1;
+ *output = NULL; ColiV4DSparkRunner *runner = calloc(1, sizeof(*runner));
+ if (!runner) return -1;
+ runner->config = *config;
+ runner->model_dir = malloc(strlen(dspark_model_dir) + 1);
+ if (!runner->model_dir) { coli_v4_dspark_runner_close(runner); return -1; }
+ strcpy(runner->model_dir, dspark_model_dir);
+ if (coli_v4_dspark_inspect(dspark_model_dir, config, &runner->manifest,
+ error, error_size) ||
+ coli_st_index_open(&runner->dspark_index, dspark_model_dir,
+ error, error_size) ||
+ coli_st_index_open(&runner->target_index, target_model_dir,
+ error, error_size) ||
+ coli_v4_dspark_heads_open(&runner->heads, dspark_model_dir, config,
+ &runner->manifest, error, error_size) ||
+ coli_v4_dspark_final_open(&runner->final, dspark_model_dir, config,
+ &runner->manifest, error, error_size)) {
+ coli_v4_dspark_runner_close(runner); return -1;
+ }
+ for (int stage = 0; stage < runner->manifest.stage_count; stage++) {
+ if (runner->manifest.common_stage_bytes[stage] > runner->loaded_stage_peak)
+ runner->loaded_stage_peak = runner->manifest.common_stage_bytes[stage];
+ if (coli_v4_dspark_attention_create(&runner->attention[stage], config)) {
+ coli_v4_dspark_runner_close(runner); return -1;
+ }
+ }
+ ColiDeepSeekV4ExpertStoreOptions options = {
+ dspark_model_dir, runner->manifest.stage_count,
+ config->n_routed_experts, expert_cache_bytes, -1, 0};
+ if (coli_deepseek_v4_dspark_expert_store_open(
+ &options, &runner->experts, error, error_size)) {
+ coli_v4_dspark_runner_close(runner); return -1;
+ }
+ *output = runner; return 0;
+}
+
+int coli_v4_dspark_runner_prefill(ColiV4DSparkRunner *runner,
+ const float *main_x,
+ int start_position, int batch,
+ char *error, size_t error_size) {
+ if (!runner || !main_x || start_position < 0 || batch < 1 || batch > 64)
+ return runner_error(error, error_size, "invalid DSpark prefill");
+ for (int stage = 0; stage < runner->manifest.stage_count; stage++) {
+ ColiDeepSeekV4LayerWeights weights;
+ if (coli_v4_dspark_layer_load(runner->engine, &weights, &runner->config,
+ runner->dspark_index, stage,
+ error, error_size)) return -1;
+ int result = coli_v4_dspark_attention_precompute_context(
+ runner->attention[stage], &weights, &runner->config,
+ main_x, start_position, batch, error, error_size);
+ coli_v4_dspark_layer_release(runner->engine, &weights);
+ if (result) return -1;
+ }
+ return 0;
+}
+
+int coli_v4_dspark_runner_draft(ColiV4DSparkRunner *runner,
+ const float *main_x, int anchor_token,
+ int position, int *draft_tokens,
+ float *draft_logits,
+ char *error, size_t error_size) {
+ if (!runner || !main_x || !draft_tokens || position < 0 ||
+ anchor_token < 0 || anchor_token >= runner->config.vocab_size)
+ return runner_error(error, error_size, "invalid DSpark draft");
+ int batch = runner->manifest.block_size;
+ int d = runner->config.hidden_size, hc = runner->config.hc_mult;
+ size_t hd = (size_t)hc * d;
+ float *state = calloc((size_t)batch * hd, sizeof(*state));
+ float *next = calloc((size_t)batch * hd, sizeof(*next));
+ float *hidden = malloc((size_t)batch * d * sizeof(*hidden));
+ int *input_tokens = malloc((size_t)batch * sizeof(*input_tokens));
+ if (!state || !next || !hidden || !input_tokens) {
+ free(input_tokens); free(hidden); free(next); free(state);
+ return runner_error(error, error_size, "out of memory in DSpark draft");
+ }
+ for (int item = 0; item < batch; item++) {
+ input_tokens[item] = item ? runner->manifest.noise_token_id : anchor_token;
+ if (runner_embedding(runner, state + (size_t)item * hd,
+ input_tokens[item])) {
+ free(input_tokens); free(hidden); free(next); free(state);
+ return runner_error(error, error_size, "DSpark embedding read failed");
+ }
+ }
+ for (int stage = 0; stage < runner->manifest.stage_count; stage++) {
+ ColiDeepSeekV4LayerWeights weights;
+ if (coli_v4_dspark_layer_load(runner->engine, &weights, &runner->config,
+ runner->dspark_index, stage,
+ error, error_size)) {
+ free(input_tokens); free(hidden); free(next); free(state); return -1;
+ }
+ int result = coli_v4_dspark_attention_precompute_context(
+ runner->attention[stage], &weights, &runner->config,
+ main_x, position, 1, error, error_size);
+ if (!result) result = coli_v4_dspark_block(
+ next, runner->attention[stage], &weights, &runner->config,
+ runner->experts, state, input_tokens, position + 1, batch,
+ error, error_size);
+ coli_v4_dspark_layer_release(runner->engine, &weights);
+ if (result) {
+ free(input_tokens); free(hidden); free(next); free(state); return -1;
+ }
+ float *swap = state; state = next; next = swap;
+ }
+ int result = coli_v4_dspark_final_hidden(runner->final, hidden, state, batch);
+ ColiV4Engine *engine = runner->engine;
+ int limit = batch;
+ if (engine) {
+ int requested = engine->runtime.verify_drafts;
+ if (requested >= 1 && requested < limit) limit = requested;
+ engine->dspark_verify.available = batch;
+ engine->dspark_verify.limit = limit;
+ engine->dspark_verify.head_slot = 0;
+ }
+ if (!result && limit > 0) {
+ int *tokens_out = engine ? engine->dspark_verify.tokens : draft_tokens;
+ float *logits_out = engine ? engine->dspark_verify.logits : draft_logits;
+ float local_logits[64];
+ if (!logits_out) logits_out = local_logits;
+ result = coli_v4_dspark_biased_argmax_batch(
+ engine, runner->heads, runner->target_index, hidden, anchor_token,
+ tokens_out, logits_out, limit);
+ for (int item = 0; !result && item < limit; item++) {
+ draft_tokens[item] = tokens_out[item];
+ if (draft_logits) draft_logits[item] = logits_out[item];
+ }
+ }
+ int previous = limit > 0 ? draft_tokens[limit - 1] : anchor_token;
+ for (int item = limit; !result && item < batch; item++) {
+ draft_tokens[item] = previous;
+ if (draft_logits) draft_logits[item] = 0.0f;
+ }
+ free(input_tokens); free(hidden); free(next); free(state);
+ return result ? runner_error(error, error_size, "DSpark draft head failed") : 0;
+}
+
+int coli_v4_dspark_runner_block_size(const ColiV4DSparkRunner *runner) {
+ return runner ? runner->manifest.block_size : 0;
+}
+
+int coli_v4_dspark_runner_verify_block_size(const ColiV4DSparkRunner *runner) {
+ if (!runner) return 0;
+ int available = runner->manifest.block_size;
+ ColiV4Engine *engine = runner->engine;
+ int requested = engine ? engine->runtime.verify_drafts : 0;
+ if (requested < 1) return available;
+ return requested < available ? requested : available;
+}
+
+uint64_t coli_v4_dspark_runner_loaded_stage_peak(
+ const ColiV4DSparkRunner *runner) {
+ return runner ? runner->loaded_stage_peak : 0;
+}
+/* ---- end inlined deepseek_v4_dspark_runner.c ---- */
+
+#undef coli_v4_dspark_runner_close
+#undef coli_v4_dspark_runner_open
+
+#include "deepseek_v4_dspark.h"
+
+int coli_v4_dspark_runner_open(ColiV4DSparkRunner **output,
+ ColiV4Engine *engine,
+ const char *dspark_model_dir,
+ const char *target_model_dir,
+ const ColiDeepSeekV4Config *config,
+ uint64_t expert_cache_bytes,
+ char *error, size_t error_size) {
+ uint64_t cache = (engine && engine->runtime.dspark_expert_cache_bytes)
+ ? engine->runtime.dspark_expert_cache_bytes : expert_cache_bytes;
+ int rc = coli_v4_dspark_runner_owned_open(
+ output, dspark_model_dir, target_model_dir, config,
+ cache, error, error_size);
+ if (!rc && output && *output) (*output)->engine = engine;
+ return rc;
+}
+
+int coli_v4_dspark_runner_use_shared_heads(ColiV4DSparkRunner *runner,
+ ColiV4DSparkHeads *heads) {
+ if (!runner || !heads) return -1;
+ coli_v4_dspark_heads_close(runner->heads);
+ runner->heads = heads;
+ return 0;
+}
+
+void coli_v4_dspark_runner_close(ColiV4DSparkRunner *runner) {
+ if (runner && runner->engine &&
+ runner->heads == runner->engine->dspark_capture.heads)
+ runner->heads = NULL;
+ coli_v4_dspark_runner_owned_close(runner);
+}
+#endif /* COLI_V4_UNIT_DSPARK_RUNNER */
+
+#ifdef COLI_V4_UNIT_TARGET_HEAD_BATCH
+/* ######## deepseek_v4_target_head_batch.c ######## */
+#include "deepseek_v4_dspark.h"
+
+#include <float.h>
+#include <math.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "native_quant.h"
+#include "tensor_io.h"
+
+int coli_v4_target_load_embeddings(float *states_hc,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config,
+ const int *tokens, int batch) {
+ const ColiSafetensorsTensor *embed = coli_st_find(index, "embed.weight");
+ if (!states_hc || !index || !config || !tokens || batch < 1 || batch > 64 ||
+ !embed || embed->dtype != COLI_ST_BF16) return -1;
+ int d = config->hidden_size, hc = config->hc_mult;
+ uint16_t *raw = malloc((size_t)d * sizeof(*raw));
+ if (!raw) return -1;
+ for (int item = 0; item < batch; item++) {
+ if (tokens[item] < 0 || tokens[item] >= config->vocab_size ||
+ coli_st_read_at(index, embed->shard,
+ embed->offset + (uint64_t)tokens[item] * d * 2,
+ (size_t)d * sizeof(*raw), raw)) {
+ free(raw); return -1;
+ }
+ float *state = states_hc + (size_t)item * hc * d;
+ for (int copy = 0; copy < hc; copy++)
+ for (int i = 0; i < d; i++)
+ state[(size_t)copy * d + i] = coli_bf16_decode(raw[i]);
+ }
+ free(raw); return 0;
+}
+
+int coli_v4_target_head_argmax_batch(
+ const float *states_hc, const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config, int batch,
+ int *tokens, float *logits, char *error, size_t error_size) {
+ if (!states_hc || !index || !config || batch < 1 || batch > 64 || !tokens)
+ return -1;
+ ColiFloatTensor function = {0}, base = {0}, scale = {0}, norm = {0};
+ if (coli_tensor_load_f32(&function, index, "hc_head_fn", error, error_size) ||
+ coli_tensor_load_f32(&base, index, "hc_head_base", error, error_size) ||
+ coli_tensor_load_f32(&scale, index, "hc_head_scale", error, error_size) ||
+ coli_tensor_load_f32(&norm, index, "norm.weight", error, error_size))
+ return -1;
+ int d = config->hidden_size, hc = config->hc_mult, flat = hc * d;
+ float *hidden = malloc((size_t)batch * d * sizeof(*hidden));
+ if (!hidden || hc > 16) { free(hidden); return -1; }
+ for (int item = 0; item < batch; item++) {
+ const float *state = states_hc + (size_t)item * flat;
+ float *out = hidden + (size_t)item * d;
+ float square = 0.0f, pre[16];
+ for (int i = 0; i < flat; i++) square += state[i] * state[i];
+ float inverse = 1.0f / sqrtf(square / flat + config->rms_norm_eps);
+ for (int copy = 0; copy < hc; copy++) {
+ float mix = 0.0f;
+ for (int i = 0; i < flat; i++)
+ mix += function.data[(size_t)copy * flat + i] * state[i];
+ float z = mix * inverse * scale.data[0] + base.data[copy];
+ float sigmoid = z >= 0.0f ? 1.0f / (1.0f + expf(-z))
+ : expf(z) / (1.0f + expf(z));
+ pre[copy] = sigmoid + config->hc_eps;
+ }
+ for (int i = 0; i < d; i++) {
+ float value = 0.0f;
+ for (int copy = 0; copy < hc; copy++)
+ value += pre[copy] * state[(size_t)copy * d + i];
+ out[i] = coli_bf16_round(value);
+ }
+ if (coli_v4_rmsnorm(out, out, norm.data, d, config->rms_norm_eps))
+ return -1;
+ coli_bf16_round_array(out, (size_t)d);
+ tokens[item] = -1; if (logits) logits[item] = -FLT_MAX;
+ }
+ const ColiSafetensorsTensor *head = coli_st_find(index, "head.weight");
+#ifndef COLI_V4_DISABLE_ZERO_COPY_HEAD
+ const uint16_t *resident = head ? coli_v4_head_cache_data(
+ NULL, head->shard, head->offset, (size_t)head->nbytes) : NULL;
+ if (resident) {
+ float *resident_scores = malloc(
+ (size_t)batch * config->vocab_size * sizeof(*resident_scores));
+ if (!resident_scores) return -1;
+ #pragma omp parallel for schedule(static)
+ for (int row = 0; row < config->vocab_size; row++) {
+ float sums[64] = {0};
+ const uint16_t *weight = resident + (size_t)row * d;
+ for (int i = 0; i < d; i++) {
+ float decoded = coli_bf16_decode(weight[i]);
+ for (int item = 0; item < batch; item++)
+ sums[item] += decoded * hidden[(size_t)item * d + i];
+ }
+ for (int item = 0; item < batch; item++)
+ resident_scores[(size_t)item * config->vocab_size + row] =
+ sums[item];
+ }
+ for (int item = 0; item < batch; item++)
+ for (int row = 0; row < config->vocab_size; row++) {
+ float score = resident_scores[
+ (size_t)item * config->vocab_size + row];
+ float current = logits ? logits[item] :
+ (tokens[item] < 0 ? -FLT_MAX : 0.0f);
+ if (tokens[item] < 0 || score > current) {
+ tokens[item] = row;
+ if (logits) logits[item] = score;
+ }
+ }
+ free(resident_scores);
+ goto head_complete;
+ }
+#endif
+ enum { ROWS = 32 };
+ uint16_t *raw = malloc((size_t)ROWS * d * sizeof(*raw));
+ float *scores = malloc((size_t)batch * ROWS * sizeof(*scores));
+ if (!head || head->dtype != COLI_ST_BF16 || !raw || !scores) return -1;
+ for (int start = 0; start < config->vocab_size; start += ROWS) {
+ int rows = config->vocab_size - start < ROWS
+ ? config->vocab_size - start : ROWS;
+ if (coli_st_read_at(index, head->shard,
+ head->offset + (uint64_t)start * d * 2,
+ (size_t)rows * d * sizeof(*raw), raw)) return -1;
+ #pragma omp parallel for collapse(2) schedule(static)
+ for (int item = 0; item < batch; item++) for (int row = 0; row < rows; row++) {
+ float sum = 0.0f; const uint16_t *weight = raw + (size_t)row * d;
+ const float *input = hidden + (size_t)item * d;
+ for (int i = 0; i < d; i++) sum += coli_bf16_decode(weight[i]) * input[i];
+ scores[(size_t)item * ROWS + row] = sum;
+ }
+ for (int item = 0; item < batch; item++) for (int row = 0; row < rows; row++) {
+ float score = scores[(size_t)item * ROWS + row];
+ float current = logits ? logits[item] :
+ (tokens[item] < 0 ? -FLT_MAX : 0.0f);
+ if (tokens[item] < 0 || score > current) {
+ tokens[item] = start + row;
+ if (logits) logits[item] = score;
+ }
+ }
+ }
+ free(scores); free(raw);
+#ifndef COLI_V4_DISABLE_ZERO_COPY_HEAD
+head_complete:
+#endif
+ free(hidden);
+ coli_float_tensor_free(&norm); coli_float_tensor_free(&scale);
+ coli_float_tensor_free(&base); coli_float_tensor_free(&function);
+ return 0;
+}
+#endif /* COLI_V4_UNIT_TARGET_HEAD_BATCH */
+
+#ifdef COLI_V4_UNIT_SPECULATIVE
+/* ######## deepseek_v4_speculative.c ######## */
+#include "deepseek_v4_dspark.h"
+
+#include <string.h>
+
+void coli_v4_speculative_controller_init(ColiV4SpeculativeController *state,
+ uint64_t minimum_proposals,
+ float disable_threshold) {
+ if (!state) return;
+ memset(state, 0, sizeof(*state));
+ state->minimum_proposals = minimum_proposals ? minimum_proposals : 32;
+ state->disable_threshold = disable_threshold > 0.0f &&
+ disable_threshold <= 1.0f
+ ? disable_threshold : 0.35f;
+ state->enabled = 1;
+}
+
+float coli_v4_speculative_acceptance(const ColiV4SpeculativeController *state) {
+ return state && state->proposed
+ ? (float)state->accepted / (float)state->proposed : 0.0f;
+}
+
+void coli_v4_speculative_record(ColiV4SpeculativeController *state,
+ int proposed, int accepted) {
+ if (!state || !state->enabled || proposed < 1 || accepted < 0 ||
+ accepted > proposed) return;
+ state->rounds++;
+ state->proposed += (uint64_t)proposed;
+ state->accepted += (uint64_t)accepted;
+ if (state->proposed >= state->minimum_proposals &&
+ coli_v4_speculative_acceptance(state) < state->disable_threshold)
+ state->enabled = 0;
+}
+
+int coli_v4_verify_greedy(ColiV4VerificationResult *result,
+ int *output_tokens, int output_capacity,
+ const int *draft_tokens, int draft_count,
+ const int *target_tokens, int target_count) {
+ if (!result || !output_tokens || !draft_tokens || !target_tokens ||
+ draft_count < 1 || target_count != draft_count + 1 ||
+ output_capacity < draft_count + 1) return -1;
+ int accepted = 0;
+ while (accepted < draft_count &&
+ draft_tokens[accepted] == target_tokens[accepted]) {
+ output_tokens[accepted] = draft_tokens[accepted];
+ accepted++;
+ }
+ output_tokens[accepted] = target_tokens[accepted];
+ result->accepted_draft_tokens = accepted;
+ result->output_count = accepted + 1;
+ result->mismatch_index = accepted < draft_count ? accepted : -1;
+ return 0;
+}
+#endif /* COLI_V4_UNIT_SPECULATIVE */
+
+#ifdef COLI_V4_UNIT_DSPARK_HEADS
+/* ######## deepseek_v4_dspark_heads.c ######## */
+/* ---- begin inlined deepseek_v4_dspark_heads_v2.c ---- */
+/* ---- begin inlined deepseek_v4_dspark_heads.c ---- */
+#include "deepseek_v4_dspark.h"
+
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "deepseek_v4_internal.h"
+#include "native_quant.h"
+#include "safetensors_index.h"
+
+struct ColiV4DSparkHeads {
+ int hidden, hc, targets, vocab, rank;
+ float rms_eps;
+ ColiTensorView main_proj;
+ unsigned char *main_weight;
+ unsigned char *main_scale;
+ float *main_norm;
+ uint16_t *markov_w1;
+ uint16_t *markov_w2;
+};
+
+static int heads_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list arguments; va_start(arguments, format);
+ vsnprintf(error, size, format, arguments); va_end(arguments);
+ }
+ return -1;
+}
+
+static void *read_named(ColiSafetensorsIndex *index, const char *name,
+ ColiSafetensorsDType dtype, char *error, size_t size) {
+ const ColiSafetensorsTensor *tensor = coli_st_find(index, name);
+ if (!tensor || tensor->dtype != dtype) {
+ heads_error(error, size, "invalid DSpark head tensor: %s", name);
+ return NULL;
+ }
+ void *data = malloc((size_t)tensor->nbytes);
+ if (!data || coli_st_read_tensor(index, tensor, data)) {
+ free(data); heads_error(error, size, "cannot read DSpark head: %s", name);
+ return NULL;
+ }
+ return data;
+}
+
+void coli_v4_dspark_heads_close(ColiV4DSparkHeads *heads) {
+ if (!heads) return;
+ free(heads->markov_w2); free(heads->markov_w1); free(heads->main_norm);
+ free(heads->main_scale); free(heads->main_weight); free(heads);
+}
+
+int coli_v4_dspark_heads_open(ColiV4DSparkHeads **output,
+ const char *model_dir,
+ const ColiDeepSeekV4Config *config,
+ const ColiDeepSeekV4DSparkManifest *manifest,
+ char *error, size_t error_size) {
+ if (!output || !model_dir || !config || !manifest ||
+ manifest->stage_count < 1) return -1;
+ *output = NULL; ColiSafetensorsIndex *index = NULL;
+ if (coli_st_index_open(&index, model_dir, error, error_size)) return -1;
+ ColiV4DSparkHeads *heads = calloc(1, sizeof(*heads));
+ if (!heads) { coli_st_index_close(index); return -1; }
+ heads->hidden = config->hidden_size; heads->hc = config->hc_mult;
+ heads->targets = manifest->target_count; heads->vocab = config->vocab_size;
+ heads->rank = manifest->markov_rank; heads->rms_eps = config->rms_norm_eps;
+ heads->main_weight = read_named(index, "mtp.0.main_proj.weight",
+ COLI_ST_F8_E4M3, error, error_size);
+ heads->main_scale = read_named(index, "mtp.0.main_proj.scale",
+ COLI_ST_F8_E8M0, error, error_size);
+ uint16_t *norm_bf16 = read_named(index, "mtp.0.main_norm.weight",
+ COLI_ST_BF16, error, error_size);
+ char name[160]; int last = manifest->stage_count - 1;
+ snprintf(name, sizeof(name), "mtp.%d.markov_head.markov_w1.weight", last);
+ heads->markov_w1 = read_named(index, name, COLI_ST_BF16, error, error_size);
+ snprintf(name, sizeof(name), "mtp.%d.markov_head.markov_w2.weight", last);
+ heads->markov_w2 = read_named(index, name, COLI_ST_BF16, error, error_size);
+ coli_st_index_close(index);
+ heads->main_norm = malloc((size_t)heads->hidden * sizeof(float));
+ if (!heads->main_weight || !heads->main_scale || !norm_bf16 ||
+ !heads->markov_w1 || !heads->markov_w2 || !heads->main_norm) {
+ free(norm_bf16); coli_v4_dspark_heads_close(heads); return -1;
+ }
+ for (int i = 0; i < heads->hidden; i++)
+ heads->main_norm[i] = coli_bf16_decode(norm_bf16[i]);
+ free(norm_bf16);
+ size_t columns = (size_t)heads->hidden * heads->targets;
+ heads->main_proj = (ColiTensorView){
+ COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_UE8M0,
+ heads->main_weight, heads->main_scale,
+ (size_t)heads->hidden * columns,
+ (size_t)((heads->hidden + 127) / 128) * ((columns + 127) / 128),
+ heads->hidden, (int64_t)columns, 128, 128
+ };
+ *output = heads; return 0;
+}
+
+int coli_v4_dspark_combine_hidden(ColiV4DSparkHeads *heads, float *output,
+ const float *target_hidden_hc) {
+ if (!heads || !output || !target_hidden_hc) return -1;
+ size_t columns = (size_t)heads->hidden * heads->targets;
+ float *combined = malloc(columns * sizeof(*combined));
+ float *projected = malloc((size_t)heads->hidden * sizeof(*projected));
+ if (!combined || !projected) { free(projected); free(combined); return -1; }
+ size_t hc_width = (size_t)heads->hc * heads->hidden;
+ for (int target = 0; target < heads->targets; target++)
+ for (int column = 0; column < heads->hidden; column++) {
+ float sum = 0.0f;
+ for (int copy = 0; copy < heads->hc; copy++)
+ sum += target_hidden_hc[(size_t)target * hc_width +
+ (size_t)copy * heads->hidden + column];
+ combined[(size_t)target * heads->hidden + column] =
+ sum / (float)heads->hc;
+ }
+ int result = coli_fp8_matvec_ref(projected, &heads->main_proj, combined);
+ if (!result) {
+ coli_bf16_round_array(projected, (size_t)heads->hidden);
+ result = coli_v4_rmsnorm(output, projected, heads->main_norm,
+ heads->hidden, heads->rms_eps);
+ if (!result) coli_bf16_round_array(output, (size_t)heads->hidden);
+ }
+ free(projected); free(combined); return result;
+}
+
+int coli_v4_dspark_markov_bias(ColiV4DSparkHeads *heads, float *bias,
+ int previous_token) {
+ if (!heads || !bias || previous_token < 0 || previous_token >= heads->vocab)
+ return -1;
+ float *embedding = malloc((size_t)heads->rank * sizeof(*embedding));
+ if (!embedding) return -1;
+ const uint16_t *row = heads->markov_w1 + (size_t)previous_token * heads->rank;
+ for (int i = 0; i < heads->rank; i++) embedding[i] = coli_bf16_decode(row[i]);
+ #pragma omp parallel for schedule(static)
+ for (int64_t token = 0; token < heads->vocab; token++) {
+ const uint16_t *weight = heads->markov_w2 + (size_t)token * heads->rank;
+ float sum = 0.0f;
+ for (int i = 0; i < heads->rank; i++)
+ sum += embedding[i] * coli_bf16_decode(weight[i]);
+ bias[token] = sum;
+ }
+ free(embedding); return 0;
+}
+/* ---- end inlined deepseek_v4_dspark_heads.c ---- */
+
+
+#include <float.h>
+
+
+
+int coli_v4_dspark_biased_argmax(
+ ColiV4Engine *engine, ColiV4DSparkHeads *heads,
+ const ColiSafetensorsIndex *target_index, const float *hidden,
+ int previous_token, int *best_token, float *best_logit) {
+ if (!heads || !target_index || !hidden || !best_token || !best_logit ||
+ previous_token < 0 || previous_token >= heads->vocab) return -1;
+ const ColiSafetensorsTensor *head = coli_st_find(target_index, "head.weight");
+ if (!head || head->dtype != COLI_ST_BF16 || head->rank != 2 ||
+ head->shape[0] != heads->vocab || head->shape[1] != heads->hidden)
+ return -1;
+ enum { ROWS = 32 };
+ uint16_t *raw = malloc((size_t)ROWS * heads->hidden * sizeof(*raw));
+ float *scores = malloc((size_t)ROWS * sizeof(*scores));
+ float *markov = malloc((size_t)heads->rank * sizeof(*markov));
+ if (!raw || !scores || !markov) {
+ free(markov); free(scores); free(raw); return -1;
+ }
+ const uint16_t *m1 = heads->markov_w1 +
+ (size_t)previous_token * heads->rank;
+ for (int i = 0; i < heads->rank; i++) markov[i] = coli_bf16_decode(m1[i]);
+ int winner = -1; float maximum = -FLT_MAX;
+ for (int start = 0; start < heads->vocab; start += ROWS) {
+ int rows = heads->vocab - start < ROWS ? heads->vocab - start : ROWS;
+ size_t bytes = (size_t)rows * heads->hidden * sizeof(*raw);
+ if (coli_st_read_at_engine(
+ engine, target_index, head->shard,
+ head->offset + (uint64_t)start * heads->hidden * 2,
+ bytes, raw)) {
+ free(markov); free(scores); free(raw); return -1;
+ }
+ #pragma omp parallel for schedule(static)
+ for (int row = 0; row < rows; row++) {
+ float sum = 0.0f;
+ const uint16_t *weight = raw + (size_t)row * heads->hidden;
+ const uint16_t *m2 = heads->markov_w2 +
+ (size_t)(start + row) * heads->rank;
+ for (int i = 0; i < heads->hidden; i++)
+ sum += coli_bf16_decode(weight[i]) * hidden[i];
+ for (int i = 0; i < heads->rank; i++)
+ sum += coli_bf16_decode(m2[i]) * markov[i];
+ scores[row] = sum;
+ }
+ for (int row = 0; row < rows; row++)
+ if (scores[row] > maximum) {
+ maximum = scores[row]; winner = start + row;
+ }
+ }
+ free(markov); free(scores); free(raw);
+ *best_token = winner; *best_logit = maximum;
+ return winner < 0 ? -1 : 0;
+}
+/* ---- end inlined deepseek_v4_dspark_heads_v2.c ---- */
+
+
+
+
+int coli_v4_dspark_biased_argmax_batch(
+ ColiV4Engine *engine, ColiV4DSparkHeads *heads,
+ const ColiSafetensorsIndex *target_index, const float *hidden_batch,
+ int previous_token, int *best_tokens, float *best_logits, int batch) {
+ if (!heads || !target_index || !hidden_batch || !best_tokens ||
+ !best_logits || batch < 1 || batch > 64 || previous_token < 0 ||
+ previous_token >= heads->vocab) return -1;
+ const ColiSafetensorsTensor *head = coli_st_find(target_index, "head.weight");
+ if (!head || head->dtype != COLI_ST_BF16 || head->rank != 2 ||
+ head->shape[0] != heads->vocab || head->shape[1] != heads->hidden)
+ return -1;
+
+ enum { ROWS = 32 };
+ uint16_t *raw = malloc((size_t)ROWS * heads->hidden * sizeof(*raw));
+ float *base = malloc((size_t)batch * heads->vocab * sizeof(*base));
+ float *markov = malloc((size_t)heads->rank * sizeof(*markov));
+ if (!raw || !base || !markov) {
+ free(markov); free(base); free(raw); return -1;
+ }
+
+ /* Each sum still visits columns in scalar order; BF16 decode is shared. */
+ for (int start = 0; start < heads->vocab; start += ROWS) {
+ int rows = heads->vocab - start < ROWS ? heads->vocab - start : ROWS;
+ size_t bytes = (size_t)rows * heads->hidden * sizeof(*raw);
+ if (coli_st_read_at_engine(
+ engine, target_index, head->shard,
+ head->offset + (uint64_t)start * heads->hidden * 2,
+ bytes, raw)) {
+ free(markov); free(base); free(raw); return -1;
+ }
+ #pragma omp parallel for schedule(static)
+ for (int row = 0; row < rows; row++) {
+ const uint16_t *weight = raw + (size_t)row * heads->hidden;
+ float sums[64] = {0};
+ for (int i = 0; i < heads->hidden; i++) {
+ float decoded = coli_bf16_decode(weight[i]);
+ for (int item = 0; item < batch; item++)
+ sums[item] += decoded *
+ hidden_batch[(size_t)item * heads->hidden + i];
+ }
+ for (int item = 0; item < batch; item++)
+ base[(size_t)item * heads->vocab + start + row] = sums[item];
+ }
+ }
+
+ for (int item = 0; item < batch; item++) {
+ const uint16_t *m1 = heads->markov_w1 +
+ (size_t)previous_token * heads->rank;
+ for (int i = 0; i < heads->rank; i++)
+ markov[i] = coli_bf16_decode(m1[i]);
+ float *scores = base + (size_t)item * heads->vocab;
+ #pragma omp parallel for schedule(static)
+ for (int token = 0; token < heads->vocab; token++) {
+ const uint16_t *m2 = heads->markov_w2 +
+ (size_t)token * heads->rank;
+ float sum = scores[token];
+ for (int i = 0; i < heads->rank; i++)
+ sum += coli_bf16_decode(m2[i]) * markov[i];
+ scores[token] = sum;
+ }
+ int winner = 0;
+ for (int token = 1; token < heads->vocab; token++)
+ if (scores[token] > scores[winner]) winner = token;
+ best_tokens[item] = winner;
+ best_logits[item] = scores[winner];
+ previous_token = winner;
+ }
+ free(markov); free(base); free(raw);
+ return 0;
+}
+#endif /* COLI_V4_UNIT_DSPARK_HEADS */
+
+#ifdef COLI_V4_UNIT_DSPARK_CAPTURE
+/* ######## deepseek_v4_dspark_capture.c ######## */
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_dspark.h"
+
+static int capture_init(ColiV4Engine *engine, const ColiDeepSeekV4Config *config) {
+ if (!engine) return -1;
+ if (engine->dspark_capture.heads) return 0;
+ ColiDeepSeekV4RuntimeOptions *runtime = &engine->runtime;
+ const char *model = runtime->dspark_model_dir;
+ char error[512];
+ if (!model ||
+ coli_v4_dspark_inspect(model, config, &engine->dspark_capture.manifest,
+ error, sizeof(error)) ||
+ coli_v4_dspark_heads_open(&engine->dspark_capture.heads, model, config,
+ &engine->dspark_capture.manifest, error,
+ sizeof(error))) {
+ fprintf(stderr, "dspark capture disabled: %s\n", model ? error :
+ "DSpark model path is unset");
+ return -1;
+ }
+ return 0;
+}
+
+static int target_ordinal(const ColiV4Engine *engine, int layer) {
+ for (int i = 0; i < engine->dspark_capture.manifest.target_count; i++)
+ if (engine->dspark_capture.manifest.target_layer_ids[i] == layer)
+ return i;
+ return -1;
+}
+
+static void capture_publish(ColiV4Engine *engine,
+ const ColiDeepSeekV4Config *config, int batch) {
+ size_t token_size = (size_t)config->hc_mult * config->hidden_size;
+ int targets = engine->dspark_capture.manifest.target_count;
+ float *joined = malloc((size_t)targets * token_size * sizeof(float));
+ float *main_x = malloc((size_t)config->hidden_size * sizeof(float));
+ if (!joined || !main_x) { free(main_x); free(joined); return; }
+ for (int item = 0; item < batch; item++) {
+ for (int target = 0; target < targets; target++)
+ memcpy(joined + (size_t)target * token_size,
+ engine->dspark_capture.states[target] +
+ (size_t)item * token_size,
+ token_size * sizeof(float));
+ if (coli_v4_dspark_combine_hidden(engine->dspark_capture.heads, main_x,
+ joined))
+ break;
+ double checksum = 0.0;
+ for (int i = 0; i < config->hidden_size; i++)
+ checksum += fabs((double)main_x[i]);
+ if (item == batch - 1)
+ fprintf(stderr, "dspark_main_x batch=%d first=%.9g l1=%.9g\n",
+ batch, main_x[0], checksum);
+ }
+ free(main_x); free(joined);
+}
+
+void coli_v4_dspark_capture_after_block(
+ ColiV4Engine *engine, const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, const float *outputs, int batch) {
+ if (!engine || !weights || !config || !outputs || batch < 1) return;
+ if (capture_init(engine, config)) return;
+ int ordinal = target_ordinal(engine, weights->plan.layer);
+ if (ordinal < 0) return;
+ size_t count = (size_t)batch * config->hc_mult * config->hidden_size;
+ if (count > engine->dspark_capture.capacity) {
+ for (int i = 0; i < engine->dspark_capture.manifest.target_count; i++) {
+ float *allocation = realloc(engine->dspark_capture.states[i],
+ count * sizeof(float));
+ if (!allocation) return;
+ engine->dspark_capture.states[i] = allocation;
+ }
+ engine->dspark_capture.capacity = count;
+ }
+ memcpy(engine->dspark_capture.states[ordinal], outputs,
+ count * sizeof(float));
+ if (ordinal == engine->dspark_capture.manifest.target_count - 1)
+ capture_publish(engine, config, batch);
+}
+
+static int capture_main_x_from_states(ColiV4Engine *engine, float *outputs,
+ int batch,
+ const ColiDeepSeekV4Config *config) {
+ if (!outputs || !config || batch < 1 || !engine ||
+ capture_init(engine, config) ||
+ engine->dspark_capture.capacity <
+ (size_t)batch * config->hc_mult * config->hidden_size)
+ return -1;
+ size_t token_size = (size_t)config->hc_mult * config->hidden_size;
+ int targets = engine->dspark_capture.manifest.target_count;
+ float *joined = malloc((size_t)targets * token_size * sizeof(*joined));
+ if (!joined) return -1;
+ int result = 0;
+ for (int item = 0; !result && item < batch; item++) {
+ for (int target = 0; target < targets; target++)
+ memcpy(joined + (size_t)target * token_size,
+ engine->dspark_capture.states[target] +
+ (size_t)item * token_size,
+ token_size * sizeof(*joined));
+ result = coli_v4_dspark_combine_hidden(
+ engine->dspark_capture.heads,
+ outputs + (size_t)item * config->hidden_size, joined);
+ }
+ free(joined); return result;
+}
+
+ColiV4DSparkHeads *coli_v4_dspark_capture_heads(ColiV4Engine *engine) {
+ return engine ? engine->dspark_capture.heads : NULL;
+}
+
+int coli_v4_dspark_capture_stage_main_x(ColiV4Engine *engine,
+ const float *values, int batch,
+ int hidden_size) {
+ if (!engine || !values || batch < 1 || batch > 64 || hidden_size < 1)
+ return -1;
+ size_t count = (size_t)batch * hidden_size;
+ float *copy = realloc(engine->dspark_capture.staged_main_x,
+ count * sizeof(*copy));
+ if (!copy) return -1;
+ engine->dspark_capture.staged_main_x = copy;
+ memcpy(engine->dspark_capture.staged_main_x, values,
+ count * sizeof(*copy));
+ engine->dspark_capture.staged_batch = batch;
+ engine->dspark_capture.staged_hidden = hidden_size;
+ return 0;
+}
+
+int coli_v4_dspark_capture_main_x(ColiV4Engine *engine, float *outputs,
+ int batch,
+ const ColiDeepSeekV4Config *config) {
+ if (engine && engine->dspark_capture.staged_batch) {
+ if (!outputs || !config || batch < 1 ||
+ batch > engine->dspark_capture.staged_batch ||
+ config->hidden_size != engine->dspark_capture.staged_hidden)
+ return -1;
+ memcpy(outputs, engine->dspark_capture.staged_main_x,
+ (size_t)batch * engine->dspark_capture.staged_hidden *
+ sizeof(*outputs));
+ engine->dspark_capture.staged_batch = 0;
+ return 0;
+ }
+ return capture_main_x_from_states(engine, outputs, batch, config);
+}
+#endif /* COLI_V4_UNIT_DSPARK_CAPTURE */
+
+
+#ifdef COLI_V4_UNIT_TARGET_VERIFY
+/* ######## deepseek_v4_target_verify.c ######## */
+#define coli_v4_target_verify_greedy_batch \
+ coli_v4_target_verify_greedy_batch_v2_fallback
+/* ---- begin inlined deepseek_v4_target_verify_v2.c ---- */
+#include "deepseek_v4_dspark.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_dspark.h"
+#include "deepseek_v4_dspark.h"
+
+static int run_target_block_record(
+ ColiV4Engine *engine, float **state_ptr, float **next_ptr,
+ ColiDeepSeekV4WindowAttentionState **attention,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config, ColiExpertStore *experts,
+ const int *tokens, int start, int batch, float *layer_inputs,
+ char *error, size_t error_size) {
+ float *state = *state_ptr, *next = *next_ptr;
+ size_t hd = (size_t)config->hc_mult * config->hidden_size;
+ for (int layer_id = 0; layer_id < config->num_hidden_layers; layer_id++) {
+ if (layer_inputs)
+ memcpy(layer_inputs + (size_t)layer_id * batch * hd, state,
+ (size_t)batch * hd * sizeof(float));
+ ColiDeepSeekV4LayerWeights layer;
+ if (coli_v4_layer_load(engine, &layer, config, index, layer_id,
+ error, error_size)) return -1;
+ int result = coli_v4_block_window_batch_ref(
+ next, attention[layer_id], &layer, config, experts,
+ state, tokens, start, batch, error, error_size);
+ if (!result)
+ coli_v4_dspark_capture_after_block(engine, &layer, config, next,
+ batch);
+ coli_v4_layer_free(engine, &layer);
+ if (result) return -1;
+ float *swap = state; state = next; next = swap;
+ }
+ *state_ptr = state; *next_ptr = next; return 0;
+}
+
+static int commit_target_attention(
+ ColiV4Engine *engine,
+ ColiDeepSeekV4WindowAttentionState **attention,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config,
+ const float *layer_inputs, int recorded_batch,
+ int start, int commit_batch, char *error, size_t error_size) {
+ size_t hd = (size_t)config->hc_mult * config->hidden_size;
+ for (int layer_id = 0; layer_id < config->num_hidden_layers; layer_id++) {
+ ColiDeepSeekV4LayerWeights layer;
+ if (coli_v4_layer_load(engine, &layer, config, index, layer_id,
+ error, error_size)) return -1;
+ int result = coli_v4_target_attention_commit_batch(
+ attention[layer_id], &layer, config,
+ layer_inputs + (size_t)layer_id * recorded_batch * hd,
+ start, commit_batch, error, error_size);
+ coli_v4_layer_free(engine, &layer);
+ if (result) return -1;
+ }
+ return 0;
+}
+
+int coli_v4_target_verify_greedy_batch(
+ ColiV4VerificationResult *verification,
+ int *output_tokens, int output_capacity,
+ ColiV4Engine *engine,
+ ColiDeepSeekV4WindowAttentionState **attention,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config, ColiExpertStore *experts,
+ int anchor_token, const int *draft_tokens, int draft_count,
+ int start_position, char *error, size_t error_size) {
+ if (!verification || !output_tokens || !attention || !index || !config ||
+ !experts || !draft_tokens || draft_count < 1 || draft_count > 63 ||
+ output_capacity < draft_count + 1) return -1;
+ int batch = draft_count + 1;
+ int *inputs = malloc((size_t)batch * sizeof(*inputs));
+ int *targets = malloc((size_t)batch * sizeof(*targets));
+ float *logits = malloc((size_t)batch * sizeof(*logits));
+ size_t hd = (size_t)config->hc_mult * config->hidden_size;
+ float *state = malloc((size_t)batch * hd * sizeof(*state));
+ float *next = malloc((size_t)batch * hd * sizeof(*next));
+ float *layer_inputs = malloc((size_t)config->num_hidden_layers *
+ batch * hd * sizeof(*layer_inputs));
+ ColiV4AttentionSnapshot **snapshots = calloc(
+ (size_t)config->num_hidden_layers, sizeof(*snapshots));
+ if (!inputs || !targets || !logits || !state || !next || !layer_inputs ||
+ !snapshots) return -1;
+ inputs[0] = anchor_token;
+ for (int i = 0; i < draft_count; i++) inputs[i + 1] = draft_tokens[i];
+ for (int layer = 0; layer < config->num_hidden_layers; layer++)
+ if (coli_v4_attention_snapshot_create(attention[layer],
+ &snapshots[layer])) return -1;
+ if (coli_v4_target_load_embeddings(state, index, config, inputs, batch) ||
+ run_target_block_record(engine, &state, &next, attention, index, config,
+ experts, inputs, start_position, batch,
+ layer_inputs, error, error_size) ||
+ coli_v4_target_head_argmax_batch(state, index, config, batch,
+ targets, logits, error, error_size) ||
+ coli_v4_verify_greedy(verification, output_tokens, output_capacity,
+ draft_tokens, draft_count, targets, batch))
+ return -1;
+ if (verification->accepted_draft_tokens < draft_count) {
+ for (int layer = 0; layer < config->num_hidden_layers; layer++)
+ if (coli_v4_attention_snapshot_restore(attention[layer],
+ snapshots[layer])) return -1;
+ int commit = verification->accepted_draft_tokens + 1;
+ if (commit_target_attention(engine, attention, index, config,
+ layer_inputs, batch, start_position, commit,
+ error, error_size)) return -1;
+ }
+ for (int layer = 0; layer < config->num_hidden_layers; layer++)
+ coli_v4_attention_snapshot_destroy(snapshots[layer]);
+ free(snapshots); free(layer_inputs); free(next); free(state);
+ free(logits); free(targets); free(inputs); return 0;
+}
+/* ---- end inlined deepseek_v4_target_verify_v2.c ---- */
+
+#undef coli_v4_target_verify_greedy_batch
+
+#include "deepseek_v4_dspark.h"
+
+
+static void destroy_snapshots(ColiV4AttentionSnapshot **snapshots, int layers) {
+ if (!snapshots) return;
+ for (int layer = 0; layer < layers; layer++)
+ coli_v4_attention_snapshot_destroy(snapshots[layer]);
+ free(snapshots);
+}
+
+static int create_snapshots(
+ ColiV4AttentionSnapshot ***output,
+ ColiDeepSeekV4WindowAttentionState **attention, int layers) {
+ ColiV4AttentionSnapshot **snapshots = calloc(
+ (size_t)layers, sizeof(*snapshots));
+ if (!snapshots) return -1;
+ for (int layer = 0; layer < layers; layer++)
+ if (coli_v4_attention_snapshot_create(attention[layer],
+ &snapshots[layer])) {
+ destroy_snapshots(snapshots, layers); return -1;
+ }
+ *output = snapshots; return 0;
+}
+
+static int restore_snapshots(
+ ColiDeepSeekV4WindowAttentionState **attention,
+ ColiV4AttentionSnapshot **snapshots, int layers) {
+ for (int layer = 0; layer < layers; layer++)
+ if (coli_v4_attention_snapshot_restore(attention[layer],
+ snapshots[layer])) return -1;
+ return 0;
+}
+
+int coli_v4_target_verify_greedy_batch(
+ ColiV4VerificationResult *verification,
+ int *output_tokens, int output_capacity,
+ ColiV4Engine *engine,
+ ColiDeepSeekV4WindowAttentionState **attention,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config, ColiExpertStore *experts,
+ int anchor_token, const int *draft_tokens, int draft_count,
+ int start_position, char *error, size_t error_size) {
+ if (draft_count != 4)
+ return coli_v4_target_verify_greedy_batch_v2_fallback(
+ verification, output_tokens, output_capacity, engine, attention,
+ index, config, experts, anchor_token, draft_tokens, draft_count,
+ start_position, error, error_size);
+ if (!verification || !output_tokens || output_capacity < 5 || !attention ||
+ !index || !config || !experts || !draft_tokens) return -1;
+
+ enum { FIRST_BATCH = 3, TAIL_BATCH = 2 };
+ int first_inputs[FIRST_BATCH] = {
+ anchor_token, draft_tokens[0], draft_tokens[1]
+ };
+ int first_targets[FIRST_BATCH]; float first_logits[FIRST_BATCH];
+ int d = config->hidden_size, hc = config->hc_mult;
+ size_t hd = (size_t)d * hc;
+ size_t layer_stride_first = (size_t)FIRST_BATCH * hd;
+ float *state = malloc((size_t)FIRST_BATCH * hd * sizeof(*state));
+ float *next = malloc((size_t)FIRST_BATCH * hd * sizeof(*next));
+ float *layer_inputs = malloc((size_t)config->num_hidden_layers *
+ layer_stride_first * sizeof(*layer_inputs));
+ float *combined_main_x = malloc((FIRST_BATCH + TAIL_BATCH) *
+ (size_t)d * sizeof(*combined_main_x));
+ ColiV4AttentionSnapshot **snapshots = NULL;
+ if (!state || !next || !layer_inputs || !combined_main_x ||
+ create_snapshots(&snapshots, attention, config->num_hidden_layers))
+ return -1;
+ int result = coli_v4_target_load_embeddings(
+ state, index, config, first_inputs, FIRST_BATCH) ||
+ run_target_block_record(
+ engine, &state, &next, attention, index, config, experts,
+ first_inputs, start_position, FIRST_BATCH, layer_inputs,
+ error, error_size) ||
+ coli_v4_target_head_argmax_batch(
+ state, index, config, FIRST_BATCH,
+ first_targets, first_logits, error, error_size) ||
+ coli_v4_dspark_capture_main_x(engine, combined_main_x, FIRST_BATCH, config);
+ if (result) goto cleanup;
+ int accepted = 0;
+ while (accepted < FIRST_BATCH &&
+ draft_tokens[accepted] == first_targets[accepted]) accepted++;
+ if (accepted < FIRST_BATCH) {
+ for (int i = 0; i < accepted; i++) output_tokens[i] = draft_tokens[i];
+ output_tokens[accepted] = first_targets[accepted];
+ verification->accepted_draft_tokens = accepted;
+ verification->output_count = accepted + 1;
+ verification->mismatch_index = accepted;
+ int commit = accepted + 1;
+ if (commit < FIRST_BATCH &&
+ (restore_snapshots(attention, snapshots,
+ config->num_hidden_layers) ||
+ commit_target_attention(
+ engine, attention, index, config, layer_inputs, FIRST_BATCH,
+ start_position, commit, error, error_size))) result = -1;
+ goto cleanup;
+ }
+ destroy_snapshots(snapshots, config->num_hidden_layers); snapshots = NULL;
+ free(layer_inputs); layer_inputs = NULL;
+ free(next); free(state); next = NULL; state = NULL;
+
+ int tail_inputs[TAIL_BATCH] = {draft_tokens[2], draft_tokens[3]};
+ int tail_targets[TAIL_BATCH]; float tail_logits[TAIL_BATCH];
+ state = malloc((size_t)TAIL_BATCH * hd * sizeof(*state));
+ next = malloc((size_t)TAIL_BATCH * hd * sizeof(*next));
+ layer_inputs = malloc((size_t)config->num_hidden_layers * TAIL_BATCH *
+ hd * sizeof(*layer_inputs));
+ if (!state || !next || !layer_inputs ||
+ create_snapshots(&snapshots, attention, config->num_hidden_layers)) {
+ result = -1; goto cleanup;
+ }
+ result = coli_v4_target_load_embeddings(
+ state, index, config, tail_inputs, TAIL_BATCH) ||
+ run_target_block_record(
+ engine, &state, &next, attention, index, config, experts,
+ tail_inputs, start_position + FIRST_BATCH, TAIL_BATCH,
+ layer_inputs, error, error_size) ||
+ coli_v4_target_head_argmax_batch(
+ state, index, config, TAIL_BATCH,
+ tail_targets, tail_logits, error, error_size) ||
+ coli_v4_dspark_capture_main_x(engine, combined_main_x + (size_t)FIRST_BATCH * d,
+ TAIL_BATCH, config);
+ if (result) goto cleanup;
+ for (int i = 0; i < FIRST_BATCH; i++) output_tokens[i] = draft_tokens[i];
+ if (draft_tokens[3] == tail_targets[0]) {
+ output_tokens[3] = draft_tokens[3];
+ output_tokens[4] = tail_targets[1];
+ verification->accepted_draft_tokens = 4;
+ verification->output_count = 5;
+ verification->mismatch_index = -1;
+ result = coli_v4_dspark_capture_stage_main_x(engine, combined_main_x, 5, d);
+ } else {
+ output_tokens[3] = tail_targets[0];
+ verification->accepted_draft_tokens = 3;
+ verification->output_count = 4;
+ verification->mismatch_index = 3;
+ if (restore_snapshots(attention, snapshots,
+ config->num_hidden_layers) ||
+ commit_target_attention(
+ engine, attention, index, config, layer_inputs, TAIL_BATCH,
+ start_position + FIRST_BATCH, 1, error, error_size) ||
+ coli_v4_dspark_capture_stage_main_x(engine, combined_main_x, 4, d))
+ result = -1;
+ }
+
+cleanup:
+ destroy_snapshots(snapshots, config->num_hidden_layers);
+ free(combined_main_x); free(layer_inputs); free(next); free(state);
+ return result;
+}
+#endif /* COLI_V4_UNIT_TARGET_VERIFY */
+
+#ifdef COLI_V4_UNIT_TARGET_ATTENTION_COMMIT
+/* ######## deepseek_v4_target_attention_commit.c ######## */
+#include "deepseek_v4_internal.h"
+#define coli_v4_block_token_batch_serial_ref \
+ coli_v4_commit_block_token_serial_ref
+#define coli_v4_block_window_token_batch_serial_ref \
+ coli_v4_commit_block_window_token_serial_ref
+/* ---- begin inlined deepseek_v4_target_attention_commit.c ---- */
+#define coli_v4_block_window_batch_ref coli_v4_commit_unused_full_block
+/* ---- begin include deepseek_v4_block_batch.c ---- */
+#define coli_v4_block_token_ref coli_v4_block_token_batch_serial_ref
+#define coli_v4_block_window_token_ref coli_v4_block_window_token_batch_serial_ref
+/* ---- begin include deepseek_v4_block.c ---- */
+#include "deepseek_v4_internal.h"
+
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "native_quant.h"
+
+static int set_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list arguments;
+ va_start(arguments, format);
+ vsnprintf(error, size, format, arguments);
+ va_end(arguments);
+ }
+ return -1;
+}
+
+static const void *value(const ColiDeepSeekV4LayerWeights *weights,
+ const char *suffix,
+ const ColiDeepSeekV4TensorSpec **spec) {
+ char name[COLI_V4_MAX_TENSOR_NAME];
+ snprintf(name, sizeof(name), "layers.%d.%s", weights->plan.layer, suffix);
+ return coli_v4_layer_data(weights, name, spec);
+}
+
+static int fp8_view(ColiTensorView *view,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const char *prefix) {
+ char name[128];
+ const ColiDeepSeekV4TensorSpec *ws = NULL, *ss = NULL;
+ snprintf(name, sizeof(name), "%s.weight", prefix);
+ const void *data = value(weights, name, &ws);
+ snprintf(name, sizeof(name), "%s.scale", prefix);
+ const void *scales = value(weights, name, &ss);
+ if (!data || !scales || !ws || !ss || ws->rank != 2) return -1;
+ *view = (ColiTensorView){
+ COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_UE8M0, data, scales,
+ (size_t)(ws->shape[0] * ws->shape[1]),
+ (size_t)(ss->shape[0] * ss->shape[1]),
+ ws->shape[0], ws->shape[1], 128, 128
+ };
+ return 0;
+}
+
+static void decode_bf16(float *output, const uint16_t *input, size_t count) {
+ for (size_t i = 0; i < count; i++) output[i] = coli_bf16_decode(input[i]);
+}
+
+static int normalized_hc_pre(float *reduced, float *post, float *comb,
+ float *normalized, const float *input_hc,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const char *branch, const char *norm_name) {
+ char name[64];
+ snprintf(name, sizeof(name), "hc_%s_fn", branch);
+ const float *function = value(weights, name, NULL);
+ snprintf(name, sizeof(name), "hc_%s_scale", branch);
+ const float *scale = value(weights, name, NULL);
+ snprintf(name, sizeof(name), "hc_%s_base", branch);
+ const float *base = value(weights, name, NULL);
+ const uint16_t *raw_norm = value(weights, norm_name, NULL);
+ int d = config->hidden_size;
+ float *norm = malloc((size_t)d * sizeof(*norm));
+ if (!function || !scale || !base || !raw_norm || !norm) {
+ free(norm);
+ return -1;
+ }
+ decode_bf16(norm, raw_norm, (size_t)d);
+ int result = coli_v4_hc_pre(reduced, post, comb, input_hc, function,
+ scale, base, config->hc_mult, d,
+ config->hc_sinkhorn_iters,
+ config->rms_norm_eps, config->hc_eps);
+ if (!result) {
+ coli_bf16_round_array(reduced, (size_t)d);
+ result = coli_v4_rmsnorm(normalized, reduced, norm, d,
+ config->rms_norm_eps);
+ coli_bf16_round_array(normalized, (size_t)d);
+ }
+ free(norm);
+ return result;
+}
+
+static int moe_token(float *output,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ ColiExpertStore *store, const float *input, int token) {
+ int d = config->hidden_size;
+ int n = config->n_routed_experts;
+ int topk = config->num_experts_per_tok;
+ size_t gate_count = (size_t)n * d;
+ float *gate = malloc(gate_count * sizeof(*gate));
+ float *route_weights = malloc((size_t)topk * sizeof(*route_weights));
+ int *indices = malloc((size_t)topk * sizeof(*indices));
+ float *expert_output = malloc((size_t)d * sizeof(*expert_output));
+ float *shared_output = malloc((size_t)d * sizeof(*shared_output));
+ if (!gate || !route_weights || !indices || !expert_output || !shared_output) {
+ free(shared_output); free(expert_output); free(indices);
+ free(route_weights); free(gate);
+ return -1;
+ }
+ decode_bf16(gate, value(weights, "ffn.gate.weight", NULL), gate_count);
+ const int64_t *table = value(weights, "ffn.gate.tid2eid", NULL);
+ const float *bias = value(weights, "ffn.gate.bias", NULL);
+ int result = token < 0 || token >= config->vocab_size;
+ if (!result && weights->plan.uses_hash_router) {
+ if (!table) result = -1;
+ }
+ if (!result && weights->plan.uses_hash_router) {
+ for (int i = 0; i < topk; i++)
+ indices[i] = (int)table[(size_t)token * topk + i];
+ }
+ if (!result) result = coli_v4_route(
+ route_weights, indices, input, gate, bias,
+ weights->plan.uses_hash_router ? indices : NULL,
+ n, d, topk, config->routed_scaling_factor);
+
+ ColiTensorView w1, w2, w3;
+ if (!result && (fp8_view(&w1, weights, "ffn.shared_experts.w1") ||
+ fp8_view(&w2, weights, "ffn.shared_experts.w2") ||
+ fp8_view(&w3, weights, "ffn.shared_experts.w3"))) result = -1;
+ if (!result) result = coli_v4_shared_expert_forward_ref(
+ shared_output, &w1, &w2, &w3, input, config->swiglu_limit);
+ if (!result) memset(output, 0, (size_t)d * sizeof(*output));
+ for (int expert_id = 0; !result && expert_id < n; expert_id++) {
+ int rank = -1;
+ for (int candidate = 0; candidate < topk; candidate++)
+ if (indices[candidate] == expert_id) rank = candidate;
+ if (rank < 0) continue;
+ ColiExpertView expert;
+ if (coli_expert_lookup(store,
+ (ColiExpertKey){weights->plan.layer, expert_id},
+ &expert)) {
+ result = -1;
+ break;
+ }
+ result = coli_v4_expert_forward_ref(expert_output, &expert, input,
+ route_weights[rank],
+ config->swiglu_limit);
+ coli_expert_release(store, &expert);
+ if (!result)
+ for (int i = 0; i < d; i++) output[i] += expert_output[i];
+ }
+ if (!result)
+ for (int i = 0; i < d; i++)
+ output[i] = coli_bf16_round(output[i] + shared_output[i]);
+ free(shared_output); free(expert_output); free(indices);
+ free(route_weights); free(gate);
+ return result;
+}
+
+static int block_token_impl(float *output_hc,
+ ColiDeepSeekV4WindowAttentionState *attention,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ ColiExpertStore *experts,
+ const float *input_hc, int token, int position,
+ char *error, size_t error_size) {
+ if (!output_hc || !weights || !config || !experts || !input_hc)
+ return set_error(error, error_size, "invalid block arguments");
+ int d = config->hidden_size, hc = config->hc_mult;
+ size_t hd = (size_t)hc * d;
+ float *residual = malloc(hd * sizeof(*residual));
+ float *state = malloc(hd * sizeof(*state));
+ float *reduced = malloc((size_t)d * sizeof(*reduced));
+ float *normalized = malloc((size_t)d * sizeof(*normalized));
+ float *branch = malloc((size_t)d * sizeof(*branch));
+ float *post = malloc((size_t)hc * sizeof(*post));
+ float *comb = malloc((size_t)hc * hc * sizeof(*comb));
+ if (!residual || !state || !reduced || !normalized || !branch || !post || !comb) {
+ free(comb); free(post); free(branch); free(normalized);
+ free(reduced); free(state); free(residual);
+ return set_error(error, error_size, "out of memory in block");
+ }
+ memcpy(residual, input_hc, hd * sizeof(*residual));
+ int result = normalized_hc_pre(reduced, post, comb, normalized, input_hc,
+ weights, config, "attn", "attn_norm.weight");
+ if (!result) result = attention
+ ? coli_v4_attention_window_token_ref(branch, attention, weights, config,
+ normalized, position, error, error_size)
+ : coli_v4_attention_token_ref(branch, weights, config, normalized,
+ position, error, error_size);
+ if (!result) result = coli_v4_hc_post(state, branch, residual, post, comb, hc, d);
+ if (!result) coli_bf16_round_array(state, hd);
+
+ if (!result) memcpy(residual, state, hd * sizeof(*residual));
+ if (!result) result = normalized_hc_pre(reduced, post, comb, normalized, state,
+ weights, config, "ffn", "ffn_norm.weight");
+ if (!result) result = moe_token(branch, weights, config, experts, normalized, token);
+ if (!result) result = coli_v4_hc_post(output_hc, branch, residual, post, comb, hc, d);
+ if (!result) coli_bf16_round_array(output_hc, hd);
+
+ free(comb); free(post); free(branch); free(normalized);
+ free(reduced); free(state); free(residual);
+ return result ? set_error(error, error_size, "block computation failed") : 0;
+}
+
+int coli_v4_block_token_ref(float *output_hc,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ ColiExpertStore *experts,
+ const float *input_hc, int token, int position,
+ char *error, size_t error_size) {
+ return block_token_impl(output_hc, NULL, weights, config, experts, input_hc,
+ token, position, error, error_size);
+}
+
+int coli_v4_block_window_token_ref(
+ float *output_hc, ColiDeepSeekV4WindowAttentionState *attention,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, ColiExpertStore *experts,
+ const float *input_hc, int token, int position,
+ char *error, size_t error_size) {
+ return block_token_impl(output_hc, attention, weights, config, experts,
+ input_hc, token, position, error, error_size);
+}
+/* ---- end include deepseek_v4_block.c ---- */
+
+#undef coli_v4_block_token_ref
+#undef coli_v4_block_window_token_ref
+
+#include "deepseek_v4_internal.h"
+#include "native_quant_batch.h"
+
+static int shared_expert_batch(float *outputs, const ColiTensorView *w1,
+ const ColiTensorView *w2,
+ const ColiTensorView *w3,
+ const float *inputs, int batch,
+ float swiglu_limit) {
+ int intermediate = (int)w1->rows, d = (int)w2->rows;
+ float *gate = malloc((size_t)batch * intermediate * sizeof(*gate));
+ float *up = malloc((size_t)batch * intermediate * sizeof(*up));
+ float *activated = malloc((size_t)batch * intermediate * sizeof(*activated));
+ if (!gate || !up || !activated) {
+ free(activated); free(up); free(gate); return -1;
+ }
+ int result = coli_fp8_matmul_batch_ref(gate, w1, inputs, batch) ||
+ coli_fp8_matmul_batch_ref(up, w3, inputs, batch);
+ for (int item = 0; !result && item < batch; item++) {
+ float *item_gate = gate + (size_t)item * intermediate;
+ float *item_up = up + (size_t)item * intermediate;
+ float *item_activated = activated + (size_t)item * intermediate;
+ coli_bf16_round_array(item_gate, (size_t)intermediate);
+ coli_bf16_round_array(item_up, (size_t)intermediate);
+ result = coli_v4_swiglu(item_activated, item_gate, item_up,
+ intermediate, swiglu_limit);
+ if (!result) coli_bf16_round_array(item_activated,
+ (size_t)intermediate);
+ }
+ if (!result) result = coli_fp8_matmul_batch_ref(
+ outputs, w2, activated, batch);
+ if (!result) coli_bf16_round_array(outputs, (size_t)batch * d);
+ free(activated); free(up); free(gate);
+ return result;
+}
+
+static int routed_expert_batch(float *outputs, const ColiExpertView *expert,
+ const float *inputs, const float *route_weights,
+ int batch, float swiglu_limit) {
+ int intermediate = (int)expert->gate.rows;
+ int d = (int)expert->down.rows;
+ float *gate = malloc((size_t)batch * intermediate * sizeof(*gate));
+ float *up = malloc((size_t)batch * intermediate * sizeof(*up));
+ float *activated = malloc((size_t)batch * intermediate * sizeof(*activated));
+ if (!gate || !up || !activated) {
+ free(activated); free(up); free(gate); return -1;
+ }
+ int result = coli_fp4_matmul_batch_ref(gate, &expert->gate, inputs, batch) ||
+ coli_fp4_matmul_batch_ref(up, &expert->up, inputs, batch);
+ for (int item = 0; !result && item < batch; item++) {
+ float *item_gate = gate + (size_t)item * intermediate;
+ float *item_up = up + (size_t)item * intermediate;
+ float *item_activated = activated + (size_t)item * intermediate;
+ coli_bf16_round_array(item_gate, (size_t)intermediate);
+ coli_bf16_round_array(item_up, (size_t)intermediate);
+ result = coli_v4_swiglu(item_activated, item_gate, item_up,
+ intermediate, swiglu_limit);
+ for (int i = 0; !result && i < intermediate; i++)
+ item_activated[i] = coli_bf16_round(
+ item_activated[i] * route_weights[item]);
+ }
+ if (!result) result = coli_fp4_matmul_batch_ref(
+ outputs, &expert->down, activated, batch);
+ if (!result) coli_bf16_round_array(outputs, (size_t)batch * d);
+ free(activated); free(up); free(gate);
+ return result;
+}
+
+static int moe_batch(float *outputs,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ ColiExpertStore *store, const float *inputs,
+ const int *tokens, int batch) {
+ int d = config->hidden_size, n = config->n_routed_experts;
+ int topk = config->num_experts_per_tok;
+ float *gate = malloc((size_t)n * d * sizeof(*gate));
+ float *route_weights = malloc((size_t)batch * topk * sizeof(*route_weights));
+ int *indices = malloc((size_t)batch * topk * sizeof(*indices));
+ float *shared = malloc((size_t)batch * d * sizeof(*shared));
+ float *compact_inputs = malloc((size_t)batch * d * sizeof(*compact_inputs));
+ float *compact_outputs = malloc((size_t)batch * d * sizeof(*compact_outputs));
+ float *compact_weights = malloc((size_t)batch * sizeof(*compact_weights));
+ int *compact_items = malloc((size_t)batch * sizeof(*compact_items));
+ if (!gate || !route_weights || !indices || !shared || !compact_inputs ||
+ !compact_outputs || !compact_weights || !compact_items) {
+ free(compact_items); free(compact_weights); free(compact_outputs);
+ free(compact_inputs); free(shared); free(indices); free(route_weights);
+ free(gate); return -1;
+ }
+ decode_bf16(gate, value(weights, "ffn.gate.weight", NULL), (size_t)n * d);
+ const int64_t *table = value(weights, "ffn.gate.tid2eid", NULL);
+ const float *bias = value(weights, "ffn.gate.bias", NULL);
+ int result = 0;
+ for (int item = 0; !result && item < batch; item++) {
+ int *item_indices = indices + (size_t)item * topk;
+ float *item_weights = route_weights + (size_t)item * topk;
+ if (tokens[item] < 0 || tokens[item] >= config->vocab_size) result = -1;
+ if (!result && weights->plan.uses_hash_router) {
+ if (!table) result = -1;
+ else for (int i = 0; i < topk; i++)
+ item_indices[i] = (int)table[(size_t)tokens[item] * topk + i];
+ }
+ if (!result) result = coli_v4_route(
+ item_weights, item_indices, inputs + (size_t)item * d,
+ gate, bias, weights->plan.uses_hash_router ? item_indices : NULL,
+ n, d, topk, config->routed_scaling_factor);
+ }
+ ColiTensorView w1, w2, w3;
+ if (!result && (fp8_view(&w1, weights, "ffn.shared_experts.w1") ||
+ fp8_view(&w2, weights, "ffn.shared_experts.w2") ||
+ fp8_view(&w3, weights, "ffn.shared_experts.w3"))) result = -1;
+ if (!result) result = shared_expert_batch(
+ shared, &w1, &w2, &w3, inputs, batch, config->swiglu_limit);
+ if (!result) memset(outputs, 0, (size_t)batch * d * sizeof(*outputs));
+
+ for (int expert_id = 0; !result && expert_id < n; expert_id++) {
+ int count = 0;
+ for (int item = 0; item < batch; item++) {
+ for (int rank = 0; rank < topk; rank++) {
+ if (indices[(size_t)item * topk + rank] == expert_id) {
+ compact_items[count] = item;
+ compact_weights[count] =
+ route_weights[(size_t)item * topk + rank];
+ memcpy(compact_inputs + (size_t)count * d,
+ inputs + (size_t)item * d, (size_t)d * sizeof(float));
+ count++;
+ }
+ }
+ }
+ if (!count) continue;
+ ColiExpertView expert;
+ if (coli_expert_lookup(store,
+ (ColiExpertKey){weights->plan.layer, expert_id},
+ &expert)) { result = -1; break; }
+ result = routed_expert_batch(compact_outputs, &expert, compact_inputs,
+ compact_weights, count,
+ config->swiglu_limit);
+ coli_expert_release(store, &expert);
+ for (int compact = 0; !result && compact < count; compact++) {
+ float *destination = outputs + (size_t)compact_items[compact] * d;
+ const float *source = compact_outputs + (size_t)compact * d;
+ for (int i = 0; i < d; i++) destination[i] += source[i];
+ }
+ }
+ for (int item = 0; !result && item < batch; item++)
+ for (int i = 0; i < d; i++)
+ outputs[(size_t)item * d + i] = coli_bf16_round(
+ outputs[(size_t)item * d + i] + shared[(size_t)item * d + i]);
+ free(compact_items); free(compact_weights); free(compact_outputs);
+ free(compact_inputs); free(shared); free(indices); free(route_weights);
+ free(gate); return result;
+}
+
+int coli_v4_block_window_batch_ref(
+ float *outputs_hc, ColiDeepSeekV4WindowAttentionState *attention,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, ColiExpertStore *experts,
+ const float *inputs_hc, const int *tokens, int start_position, int batch,
+ char *error, size_t error_size) {
+ if (!outputs_hc || !attention || !weights || !config || !experts ||
+ !inputs_hc || !tokens || batch < 1 || batch > 64) return -1;
+ int d = config->hidden_size, hc = config->hc_mult;
+ size_t hd = (size_t)hc * d;
+ float *states = malloc((size_t)batch * hd * sizeof(*states));
+ float *normalized_ffn = malloc((size_t)batch * d * sizeof(*normalized_ffn));
+ float *branches = malloc((size_t)batch * d * sizeof(*branches));
+ float *posts = malloc((size_t)batch * hc * sizeof(*posts));
+ float *combs = malloc((size_t)batch * hc * hc * sizeof(*combs));
+ float *residual = malloc(hd * sizeof(*residual));
+ float *reduced = malloc((size_t)d * sizeof(*reduced));
+ float *normalized = malloc((size_t)d * sizeof(*normalized));
+ float *branch = malloc((size_t)d * sizeof(*branch));
+ float *post = malloc((size_t)hc * sizeof(*post));
+ float *comb = malloc((size_t)hc * hc * sizeof(*comb));
+ if (!states || !normalized_ffn || !branches || !posts || !combs ||
+ !residual || !reduced || !normalized || !branch || !post || !comb) {
+ free(comb); free(post); free(branch); free(normalized); free(reduced);
+ free(residual); free(combs); free(posts); free(branches);
+ free(normalized_ffn); free(states); return -1;
+ }
+ int result = 0;
+ for (int item = 0; !result && item < batch; item++) {
+ const float *input = inputs_hc + (size_t)item * hd;
+ float *state = states + (size_t)item * hd;
+ memcpy(residual, input, hd * sizeof(float));
+ result = normalized_hc_pre(reduced, post, comb, normalized, input,
+ weights, config, "attn", "attn_norm.weight");
+ if (!result) result = coli_v4_attention_window_token_ref(
+ branch, attention, weights, config, normalized,
+ start_position + item, error, error_size);
+ if (!result) result = coli_v4_hc_post(state, branch, residual,
+ post, comb, hc, d);
+ if (!result) coli_bf16_round_array(state, hd);
+ if (!result) result = normalized_hc_pre(
+ reduced, posts + (size_t)item * hc,
+ combs + (size_t)item * hc * hc,
+ normalized_ffn + (size_t)item * d, state,
+ weights, config, "ffn", "ffn_norm.weight");
+ }
+ if (!result) result = moe_batch(branches, weights, config, experts,
+ normalized_ffn, tokens, batch);
+ for (int item = 0; !result && item < batch; item++) {
+ result = coli_v4_hc_post(
+ outputs_hc + (size_t)item * hd,
+ branches + (size_t)item * d,
+ states + (size_t)item * hd,
+ posts + (size_t)item * hc,
+ combs + (size_t)item * hc * hc, hc, d);
+ if (!result) coli_bf16_round_array(
+ outputs_hc + (size_t)item * hd, hd);
+ }
+ free(comb); free(post); free(branch); free(normalized); free(reduced);
+ free(residual); free(combs); free(posts); free(branches);
+ free(normalized_ffn); free(states);
+ return result ? set_error(error, error_size, "batched block failed") : 0;
+}
+/* ---- end include deepseek_v4_block_batch.c ---- */
+
+#undef coli_v4_block_window_batch_ref
+
+#include "deepseek_v4_dspark.h"
+
+int coli_v4_target_attention_commit_batch(
+ ColiDeepSeekV4WindowAttentionState *attention,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const float *inputs_hc, int start_position, int batch,
+ char *error, size_t error_size) {
+ if (!attention || !weights || !config || !inputs_hc ||
+ start_position < 0 || batch < 1 || batch > 64) return -1;
+ int d = config->hidden_size, hc = config->hc_mult;
+ size_t hd = (size_t)hc * d;
+ float *normalized = malloc((size_t)batch * d * sizeof(*normalized));
+ float *discarded = malloc((size_t)batch * d * sizeof(*discarded));
+ float *reduced = malloc((size_t)d * sizeof(*reduced));
+ float *post = malloc((size_t)hc * sizeof(*post));
+ float *comb = malloc((size_t)hc * hc * sizeof(*comb));
+ if (!normalized || !discarded || !reduced || !post || !comb) {
+ free(comb); free(post); free(reduced); free(discarded); free(normalized);
+ return -1;
+ }
+ int result = 0;
+ for (int item = 0; !result && item < batch; item++)
+ result = normalized_hc_pre(
+ reduced, post, comb, normalized + (size_t)item * d,
+ inputs_hc + (size_t)item * hd, weights, config,
+ "attn", "attn_norm.weight");
+ if (!result) result = coli_v4_attention_window_batch_ref(
+ discarded, attention, weights, config, normalized,
+ start_position, batch, error, error_size);
+ free(comb); free(post); free(reduced); free(discarded); free(normalized);
+ return result;
+}
+/* ---- end inlined deepseek_v4_target_attention_commit.c ---- */
+
+#undef coli_v4_block_window_token_batch_serial_ref
+#undef coli_v4_block_token_batch_serial_ref
+#endif /* COLI_V4_UNIT_TARGET_ATTENTION_COMMIT */
+
+#ifdef COLI_V4_UNIT_TARGET_VERIFY_PREFIX
+/* ######## deepseek_v4_target_verify_prefix.c ######## */
+#include "deepseek_v4_dspark.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_dspark.h"
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_dspark.h"
+#include "deepseek_v4_dspark.h"
+
+static void prefix_destroy_snapshots(ColiV4AttentionSnapshot **snapshots,
+ int layers) {
+ if (!snapshots) return;
+ for (int layer = 0; layer < layers; layer++)
+ coli_v4_attention_snapshot_destroy(snapshots[layer]);
+ free(snapshots);
+}
+
+static int prefix_create_snapshots(
+ ColiV4AttentionSnapshot ***output,
+ ColiDeepSeekV4WindowAttentionState **attention, int layers) {
+ ColiV4AttentionSnapshot **snapshots = calloc(
+ (size_t)layers, sizeof(*snapshots));
+ if (!snapshots) return -1;
+ for (int layer = 0; layer < layers; layer++)
+ if (coli_v4_attention_snapshot_create(attention[layer],
+ &snapshots[layer])) {
+ prefix_destroy_snapshots(snapshots, layers); return -1;
+ }
+ *output = snapshots;
+ return 0;
+}
+
+static int prefix_restore_snapshots(
+ ColiDeepSeekV4WindowAttentionState **attention,
+ ColiV4AttentionSnapshot **snapshots, int layers) {
+ for (int layer = 0; layer < layers; layer++)
+ if (coli_v4_attention_snapshot_restore(attention[layer],
+ snapshots[layer])) return -1;
+ return 0;
+}
+
+static int prefix_run_target_batch(
+ ColiV4Engine *engine, float **state_ptr, float **next_ptr,
+ ColiDeepSeekV4WindowAttentionState **attention,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config, ColiExpertStore *experts,
+ const int *tokens, int start, int batch, float *layer_inputs,
+ char *error, size_t error_size) {
+ float *state = *state_ptr, *next = *next_ptr;
+ size_t hd = (size_t)config->hc_mult * config->hidden_size;
+ for (int layer_id = 0; layer_id < config->num_hidden_layers; layer_id++) {
+ memcpy(layer_inputs + (size_t)layer_id * batch * hd, state,
+ (size_t)batch * hd * sizeof(*state));
+ ColiDeepSeekV4LayerWeights layer;
+ if (coli_v4_layer_load(engine, &layer, config, index, layer_id,
+ error, error_size)) return -1;
+ int result = coli_v4_block_window_batch_ref(
+ next, attention[layer_id], &layer, config, experts,
+ state, tokens, start, batch, error, error_size);
+ if (!result)
+ coli_v4_dspark_capture_after_block(engine, &layer, config, next,
+ batch);
+ coli_v4_layer_free(engine, &layer);
+ if (result) return -1;
+ float *swap = state; state = next; next = swap;
+ }
+ *state_ptr = state;
+ *next_ptr = next;
+ return 0;
+}
+
+static int prefix_commit_attention(
+ ColiV4Engine *engine,
+ ColiDeepSeekV4WindowAttentionState **attention,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config,
+ const float *layer_inputs, int recorded_batch,
+ int start, int commit_batch, char *error, size_t error_size) {
+ size_t hd = (size_t)config->hc_mult * config->hidden_size;
+ for (int layer_id = 0; layer_id < config->num_hidden_layers; layer_id++) {
+ ColiDeepSeekV4LayerWeights layer;
+ if (coli_v4_layer_load(engine, &layer, config, index, layer_id,
+ error, error_size)) return -1;
+ int result = coli_v4_target_attention_commit_batch(
+ attention[layer_id], &layer, config,
+ layer_inputs + (size_t)layer_id * recorded_batch * hd,
+ start, commit_batch, error, error_size);
+ coli_v4_layer_free(engine, &layer);
+ if (result) return -1;
+ }
+ return 0;
+}
+
+int coli_v4_target_verify_after_prefix_v69(
+ ColiV4VerificationResult *verification,
+ int *output_tokens, int output_capacity,
+ ColiV4Engine *engine,
+ ColiDeepSeekV4WindowAttentionState **attention,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config, ColiExpertStore *experts,
+ const int *draft_tokens, int draft_count, int prefix_position,
+ const float *prefix_main_x, char *error, size_t error_size) {
+ enum { FIRST_BATCH = 2, TAIL_BATCH = 2, TOTAL_ROWS = 5 };
+ if (!verification || !output_tokens || output_capacity < TOTAL_ROWS ||
+ !attention || !index || !config || !experts || !draft_tokens ||
+ draft_count != 4 || prefix_position < 0 || !prefix_main_x) return -1;
+
+ int result = 0;
+ int d = config->hidden_size, hc = config->hc_mult;
+ size_t hd = (size_t)d * hc;
+ float *combined_main_x = malloc((size_t)TOTAL_ROWS * d * sizeof(float));
+ float *state = malloc((size_t)FIRST_BATCH * hd * sizeof(float));
+ float *next = malloc((size_t)FIRST_BATCH * hd * sizeof(float));
+ float *layer_inputs = malloc((size_t)config->num_hidden_layers *
+ FIRST_BATCH * hd * sizeof(float));
+ ColiV4AttentionSnapshot **snapshots = NULL;
+ if (!combined_main_x || !state || !next || !layer_inputs ||
+ prefix_create_snapshots(&snapshots, attention,
+ config->num_hidden_layers)) {
+ result = -1; goto cleanup;
+ }
+ memcpy(combined_main_x, prefix_main_x, (size_t)d * sizeof(float));
+
+ int first_inputs[FIRST_BATCH] = {draft_tokens[0], draft_tokens[1]};
+ int first_targets[FIRST_BATCH];
+ float first_logits[FIRST_BATCH];
+ result = coli_v4_target_load_embeddings(
+ state, index, config, first_inputs, FIRST_BATCH) ||
+ prefix_run_target_batch(
+ engine, &state, &next, attention, index, config, experts,
+ first_inputs, prefix_position + 1, FIRST_BATCH,
+ layer_inputs, error, error_size) ||
+ coli_v4_target_head_argmax_batch(
+ state, index, config, FIRST_BATCH,
+ first_targets, first_logits, error, error_size) ||
+ coli_v4_dspark_capture_main_x(engine, combined_main_x + d, FIRST_BATCH, config);
+ if (result) goto cleanup;
+
+ output_tokens[0] = draft_tokens[0];
+ int accepted = 0;
+ while (accepted < FIRST_BATCH &&
+ draft_tokens[accepted + 1] == first_targets[accepted]) {
+ output_tokens[accepted + 1] = draft_tokens[accepted + 1];
+ accepted++;
+ }
+ if (accepted < FIRST_BATCH) {
+ int commit = accepted + 1;
+ output_tokens[accepted + 1] = first_targets[accepted];
+ verification->accepted_draft_tokens = accepted + 1;
+ verification->output_count = accepted + 2;
+ verification->mismatch_index = accepted + 1;
+ if (prefix_restore_snapshots(attention, snapshots,
+ config->num_hidden_layers) ||
+ prefix_commit_attention(
+ engine, attention, index, config, layer_inputs, FIRST_BATCH,
+ prefix_position + 1, commit, error, error_size) ||
+ coli_v4_dspark_capture_stage_main_x(engine, combined_main_x, commit + 1, d)) result = -1;
+ goto cleanup;
+ }
+
+ prefix_destroy_snapshots(snapshots, config->num_hidden_layers);
+ snapshots = NULL;
+ free(layer_inputs); free(next); free(state);
+ layer_inputs = NULL; next = NULL; state = NULL;
+
+ state = malloc((size_t)TAIL_BATCH * hd * sizeof(float));
+ next = malloc((size_t)TAIL_BATCH * hd * sizeof(float));
+ layer_inputs = malloc((size_t)config->num_hidden_layers *
+ TAIL_BATCH * hd * sizeof(float));
+ if (!state || !next || !layer_inputs ||
+ prefix_create_snapshots(&snapshots, attention,
+ config->num_hidden_layers)) {
+ result = -1; goto cleanup;
+ }
+ int tail_inputs[TAIL_BATCH] = {draft_tokens[2], draft_tokens[3]};
+ int tail_targets[TAIL_BATCH];
+ float tail_logits[TAIL_BATCH];
+ result = coli_v4_target_load_embeddings(
+ state, index, config, tail_inputs, TAIL_BATCH) ||
+ prefix_run_target_batch(
+ engine, &state, &next, attention, index, config, experts,
+ tail_inputs, prefix_position + 3, TAIL_BATCH,
+ layer_inputs, error, error_size) ||
+ coli_v4_target_head_argmax_batch(
+ state, index, config, TAIL_BATCH,
+ tail_targets, tail_logits, error, error_size) ||
+ coli_v4_dspark_capture_main_x(engine, combined_main_x + (size_t)3 * d, TAIL_BATCH, config);
+ if (result) goto cleanup;
+
+ output_tokens[1] = draft_tokens[1];
+ output_tokens[2] = draft_tokens[2];
+ if (draft_tokens[3] == tail_targets[0]) {
+ output_tokens[3] = draft_tokens[3];
+ output_tokens[4] = tail_targets[1];
+ verification->accepted_draft_tokens = 4;
+ verification->output_count = 5;
+ verification->mismatch_index = -1;
+ result = coli_v4_dspark_capture_stage_main_x(engine, combined_main_x, TOTAL_ROWS, d);
+ } else {
+ output_tokens[3] = tail_targets[0];
+ verification->accepted_draft_tokens = 3;
+ verification->output_count = 4;
+ verification->mismatch_index = 3;
+ if (prefix_restore_snapshots(attention, snapshots,
+ config->num_hidden_layers) ||
+ prefix_commit_attention(
+ engine, attention, index, config, layer_inputs, TAIL_BATCH,
+ prefix_position + 3, 1, error, error_size) ||
+ coli_v4_dspark_capture_stage_main_x(engine, combined_main_x, 4, d)) result = -1;
+ }
+
+cleanup:
+ prefix_destroy_snapshots(snapshots, config->num_hidden_layers);
+ free(layer_inputs); free(next); free(state); free(combined_main_x);
+ return result ? -1 : 0;
+}
+#endif /* COLI_V4_UNIT_TARGET_VERIFY_PREFIX */
+
+#ifdef COLI_V4_UNIT_DSPARK_VERIFY_WINDOW
+/* ######## deepseek_v4_dspark_verify_window.c ######## */
+/* Verify-window batching now lives in coli_v4_dspark_runner_draft /
+ * coli_v4_dspark_runner_verify_block_size (no ld --wrap). */
+#endif /* COLI_V4_UNIT_DSPARK_VERIFY_WINDOW */