Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

v4: forward-port the DeepSeek V4 target engine

DrewZca6b8bbb04eb
diff --git a/c/Makefile.deepseek-v4 b/c/Makefile.deepseek-v4
new file mode 100644
index 00000000..095212bc
--- /dev/null
+++ b/c/Makefile.deepseek-v4
@@ -0,0 +1,87 @@
+# DeepSeek V4 amalgamation build — x86-64 Linux and Windows/MSYS2 only.
+# Uses GNU ld -Wl,--wrap and gcc -march; do not invoke from macOS/PowerPC
+# `make check` (the parent Makefile gates COLI_V4_SUPPORTED).
+
+ifeq ($(OS),Windows_NT)
+IS_WIN := 1
+UNAME_S :=
+UNAME_M :=
+else
+IS_WIN :=
+UNAME_S := $(shell uname -s 2>/dev/null)
+UNAME_M := $(shell uname -m 2>/dev/null)
+endif
+
+COLI_V4_OK :=
+ifeq ($(IS_WIN),1)
+COLI_V4_OK := 1
+else ifeq ($(UNAME_S),Linux)
+ifneq (,$(filter x86_64 amd64,$(UNAME_M)))
+COLI_V4_OK := 1
+endif
+endif
+
+ifneq ($(COLI_V4_OK),1)
+$(error Makefile.deepseek-v4 supports only x86-64 Linux and Windows/MSYS2)
+endif
+
+ifneq ($(IS_WIN),)
+CC = gcc
+ARCH ?= x86-64-v3
+CFLAGS = -D_FILE_OFFSET_BITS=64 -O3 -march=$(ARCH) -fopenmp \
+ -include pthread.h -Wall -Wextra -Wno-unused-parameter \
+ -Wno-misleading-indentation -Wno-unused-function \
+ -DCOLI_V4_EXPERIMENTAL_PARALLEL_PREFIX_VERIFY \
+ -DCOLI_V4_MAX_PIN_SLOTS_PER_LAYER=16 \
+ -DCOLI_V4_PIN_RAMP_REQUESTS=24
+LDFLAGS = -lm -fopenmp -static -pthread
+V4_BINARY := deepseek_v4.exe
+else
+CC = gcc
+ARCH ?= native
+CFLAGS = -O3 -march=$(ARCH) -fopenmp -pthread -include pthread.h \
+ -Wall -Wextra -Wno-unused-parameter \
+ -Wno-misleading-indentation -Wno-unused-function \
+ -DCOLI_V4_EXPERIMENTAL_PARALLEL_PREFIX_VERIFY \
+ -DCOLI_V4_MAX_PIN_SLOTS_PER_LAYER=16 \
+ -DCOLI_V4_PIN_RAMP_REQUESTS=24
+LDFLAGS = -lm -fopenmp -pthread
+V4_BINARY := deepseek_v4
+endif
+
+include Makefile.deepseek-v4.units
+
+# native_quant.c / native_quant_batch.c are pulled in by the parallel/avx512 TUs.
+SHARED_SOURCES = safetensors_index.c native_quant_parallel.c \
+ native_quant_dual.c native_quant_batch_avx512.c \
+ native_quant_fp4_rows16.c tensor_io.c
+
+SHARED_OBJS = $(SHARED_SOURCES:.c=.o)
+TARGET_OBJS = $(addsuffix .o,$(V4_TARGET_UNITS))
+DSPARK_OBJS = $(addsuffix .o,$(V4_DSPARK_UNITS))
+TEST_UNIT_OBJS = $(addsuffix .o,$(V4_TEST_UNITS))
+V4_OBJS = $(SHARED_OBJS) $(TARGET_OBJS) $(DSPARK_OBJS)
+
+.PHONY: deepseek-v4 deepseek-v4-clean deepseek-v4-test-objs
+deepseek-v4: $(V4_BINARY)
+
+$(V4_BINARY): $(V4_OBJS)
+ $(CC) $(V4_OBJS) -o $@ $(LDFLAGS)
+
+$(SHARED_OBJS): %.o: %.c
+ $(CC) $(CFLAGS) -c $< -o $@
+
+# These translation units include native_quant.c directly.
+native_quant_parallel.o native_quant_dual.o native_quant_batch_avx512.o: \
+ native_quant.c native_quant.h
+
+$(TARGET_OBJS) $(TEST_UNIT_OBJS): %.o: deepseek_v4.c deepseek_v4.h deepseek_v4_dspark.h
+ $(CC) $(CFLAGS) -D$* -c deepseek_v4.c -o $@
+
+$(DSPARK_OBJS): %.o: deepseek_v4_dspark.c deepseek_v4.h deepseek_v4_dspark.h
+ $(CC) $(CFLAGS) -D$* -c deepseek_v4_dspark.c -o $@
+
+deepseek-v4-test-objs: $(TEST_UNIT_OBJS)
+
+deepseek-v4-clean:
+ rm -f $(V4_BINARY) $(V4_OBJS) $(TEST_UNIT_OBJS)
diff --git a/c/Makefile.deepseek-v4.units b/c/Makefile.deepseek-v4.units
new file mode 100644
index 00000000..31e1e0e0
--- /dev/null
+++ b/c/Makefile.deepseek-v4.units
@@ -0,0 +1,47 @@
+# Generated by _amalgamate_v4.py — object units for amalgamated sources.
+V4_TARGET_UNITS = \
+ COLI_V4_UNIT_LAYER_RESIDENT \
+ COLI_V4_UNIT_RESOURCE_PLAN \
+ COLI_V4_UNIT_HEAD_CACHE \
+ COLI_V4_UNIT_EXPERT_STORE_AUTO \
+ COLI_V4_UNIT_MATH \
+ COLI_V4_UNIT_ATTENTION \
+ COLI_V4_UNIT_ATTENTION_BATCH \
+ COLI_V4_UNIT_COMPRESSOR \
+ COLI_V4_UNIT_INDEXER \
+ COLI_V4_UNIT_SPARSE_ATTENTION \
+ COLI_V4_UNIT_BLOCK_HYBRID \
+ COLI_V4_UNIT_COMPRESSOR_SNAPSHOT \
+ COLI_V4_UNIT_INDEXER_SNAPSHOT \
+ COLI_V4_UNIT_ATTENTION_TRANSACTION \
+ COLI_V4_UNIT_EXPERT_STORE_HOT_ROWS16 \
+ COLI_V4_UNIT_EXPERT_ROWS16 \
+ COLI_V4_UNIT_ROUTE_BF16 \
+ COLI_V4_UNIT_RUNTIME \
+ COLI_V4_UNIT_PROMPT \
+ COLI_V4_UNIT_GENERATE_STATS
+V4_DSPARK_UNITS = \
+ COLI_V4_UNIT_CONFIG_DSPARK_COMPAT \
+ COLI_V4_UNIT_DSPARK \
+ COLI_V4_UNIT_DSPARK_MEMORY \
+ COLI_V4_UNIT_DSPARK_RUNTIME_RESIDENT \
+ COLI_V4_UNIT_DSPARK_EXPERT_STORE \
+ COLI_V4_UNIT_DSPARK_ATTENTION \
+ COLI_V4_UNIT_DSPARK_BLOCK \
+ COLI_V4_UNIT_DSPARK_FINAL \
+ COLI_V4_UNIT_DSPARK_RUNNER \
+ COLI_V4_UNIT_TARGET_HEAD_BATCH \
+ COLI_V4_UNIT_SPECULATIVE \
+ COLI_V4_UNIT_DSPARK_HEADS \
+ COLI_V4_UNIT_DSPARK_CAPTURE \
+ COLI_V4_UNIT_TARGET_VERIFY \
+ COLI_V4_UNIT_TARGET_ATTENTION_COMMIT \
+ COLI_V4_UNIT_TARGET_VERIFY_PREFIX \
+ COLI_V4_UNIT_DSPARK_VERIFY_WINDOW
+V4_TEST_UNITS = \
+ COLI_V4_UNIT_KV_CACHE \
+ COLI_V4_UNIT_ATTENTION_CACHE \
+ COLI_V4_UNIT_EXPERT \
+ COLI_V4_UNIT_EXPERT_STORE \
+ COLI_V4_UNIT_LAYER \
+ COLI_V4_UNIT_CONFIG
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.c
new file mode 100644
index 00000000..99735422
--- /dev/null
+++ b/c/deepseek_v4.c
@@ -0,0 +1,8717 @@
+/* Amalgamated deepseek_v4.c — GLM-style source; compile with -DCOLI_V4_UNIT_* per object */
+/* Umbrella API: deepseek_v4.h + deepseek_v4_dspark.h (included by units) */
+
+#ifdef COLI_V4_UNIT_LAYER_RESIDENT
+/* ######## deepseek_v4_layer_resident.c ######## */
+#define coli_v4_layer_load coli_v4_layer_resident_reference_load
+#define coli_v4_layer_free coli_v4_layer_resident_reference_free
+/* ---- begin include deepseek_v4_layer.c ---- */
+#include "deepseek_v4_internal.h"
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static int set_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list arguments;
+ va_start(arguments, format);
+ vsnprintf(error, size, format, arguments);
+ va_end(arguments);
+ }
+ return -1;
+}
+
+static int add_spec(ColiDeepSeekV4LayerPlan *plan, ColiSafetensorsDType dtype,
+ int rank, const int64_t *shape, const char *suffix,
+ char *error, size_t error_size) {
+ if (plan->tensor_count >= COLI_V4_MAX_LAYER_TENSORS)
+ return set_error(error, error_size, "too many tensors in layer %d", plan->layer);
+ ColiDeepSeekV4TensorSpec *spec = &plan->tensors[plan->tensor_count++];
+ int written = snprintf(spec->name, sizeof(spec->name), "layers.%d.%s",
+ plan->layer, suffix);
+ if (written < 0 || (size_t)written >= sizeof(spec->name))
+ return set_error(error, error_size, "tensor name is too long: %s", suffix);
+ spec->dtype = dtype;
+ spec->rank = rank;
+ memcpy(spec->shape, shape, (size_t)rank * sizeof(*shape));
+ return 0;
+}
+
+static int add_1d(ColiDeepSeekV4LayerPlan *plan, ColiSafetensorsDType dtype,
+ int64_t d0, const char *name, char *error, size_t size) {
+ int64_t shape[] = {d0};
+ return add_spec(plan, dtype, 1, shape, name, error, size);
+}
+
+static int add_2d(ColiDeepSeekV4LayerPlan *plan, ColiSafetensorsDType dtype,
+ int64_t d0, int64_t d1, const char *name,
+ char *error, size_t size) {
+ int64_t shape[] = {d0, d1};
+ return add_spec(plan, dtype, 2, shape, name, error, size);
+}
+
+static int add_fp8(ColiDeepSeekV4LayerPlan *plan, int64_t rows, int64_t columns,
+ const char *prefix, char *error, size_t size) {
+ char name[128];
+ snprintf(name, sizeof(name), "%s.weight", prefix);
+ if (add_2d(plan, COLI_ST_F8_E4M3, rows, columns, name, error, size) != 0) return -1;
+ snprintf(name, sizeof(name), "%s.scale", prefix);
+ return add_2d(plan, COLI_ST_F8_E8M0, (rows + 127) / 128,
+ (columns + 127) / 128, name, error, size);
+}
+
+#define ADD(call) do { if ((call) != 0) return -1; } while (0)
+
+int coli_v4_layer_plan(ColiDeepSeekV4LayerPlan *plan,
+ const ColiDeepSeekV4Config *config, int layer,
+ char *error, size_t error_size) {
+ if (!plan || !config || layer < 0 || layer >= config->num_hidden_layers ||
+ layer >= config->compress_ratio_count)
+ return set_error(error, error_size, "invalid DeepSeek-V4 layer plan arguments");
+ memset(plan, 0, sizeof(*plan));
+ plan->layer = layer;
+ plan->compression_ratio = config->compress_ratios[layer];
+ plan->uses_hash_router = layer < config->num_hash_layers;
+ plan->has_compressor = plan->compression_ratio != 0;
+ plan->has_indexer = plan->compression_ratio == 4;
+
+ const int64_t hidden = config->hidden_size;
+ const int64_t heads = config->num_attention_heads;
+ const int64_t head_dim = config->head_dim;
+ const int64_t q_rank = config->q_lora_rank;
+ const int64_t o_width = (int64_t)config->o_groups * config->o_lora_rank;
+ const int64_t experts = config->n_routed_experts;
+ const int64_t moe = config->moe_intermediate_size;
+ const int64_t hc = config->hc_mult;
+ const int64_t hc_params = (2 + hc) * hc;
+
+ ADD(add_1d(plan, COLI_ST_F32, heads, "attn.attn_sink", error, error_size));
+ ADD(add_1d(plan, COLI_ST_BF16, head_dim, "attn.kv_norm.weight", error, error_size));
+ ADD(add_1d(plan, COLI_ST_BF16, q_rank, "attn.q_norm.weight", error, error_size));
+ ADD(add_fp8(plan, head_dim, hidden, "attn.wkv", error, error_size));
+ ADD(add_fp8(plan, o_width, hidden, "attn.wo_a", error, error_size));
+ ADD(add_fp8(plan, hidden, o_width, "attn.wo_b", error, error_size));
+ ADD(add_fp8(plan, q_rank, hidden, "attn.wq_a", error, error_size));
+ ADD(add_fp8(plan, heads * head_dim, q_rank, "attn.wq_b", error, error_size));
+ ADD(add_1d(plan, COLI_ST_BF16, hidden, "attn_norm.weight", error, error_size));
+
+ if (plan->has_compressor) {
+ int64_t ratio = plan->compression_ratio;
+ int64_t coff = ratio == 4 ? 2 : 1;
+ ADD(add_2d(plan, COLI_ST_F32, ratio, coff * head_dim,
+ "attn.compressor.ape", error, error_size));
+ ADD(add_1d(plan, COLI_ST_BF16, head_dim,
+ "attn.compressor.norm.weight", error, error_size));
+ ADD(add_2d(plan, COLI_ST_BF16, coff * head_dim, hidden,
+ "attn.compressor.wgate.weight", error, error_size));
+ ADD(add_2d(plan, COLI_ST_BF16, coff * head_dim, hidden,
+ "attn.compressor.wkv.weight", error, error_size));
+ }
+ if (plan->has_indexer) {
+ int64_t ih = config->index_head_dim;
+ int64_t in = config->index_n_heads;
+ ADD(add_2d(plan, COLI_ST_F32, 4, 2 * ih,
+ "attn.indexer.compressor.ape", error, error_size));
+ ADD(add_1d(plan, COLI_ST_BF16, ih,
+ "attn.indexer.compressor.norm.weight", error, error_size));
+ ADD(add_2d(plan, COLI_ST_BF16, 2 * ih, hidden,
+ "attn.indexer.compressor.wgate.weight", error, error_size));
+ ADD(add_2d(plan, COLI_ST_BF16, 2 * ih, hidden,
+ "attn.indexer.compressor.wkv.weight", error, error_size));
+ ADD(add_2d(plan, COLI_ST_BF16, in, hidden,
+ "attn.indexer.weights_proj.weight", error, error_size));
+ ADD(add_fp8(plan, in * ih, q_rank, "attn.indexer.wq_b", error, error_size));
+ }
+
+ ADD(add_2d(plan, COLI_ST_BF16, experts, hidden,
+ "ffn.gate.weight", error, error_size));
+ if (plan->uses_hash_router)
+ ADD(add_2d(plan, COLI_ST_I64, config->vocab_size,
+ config->num_experts_per_tok, "ffn.gate.tid2eid", error, error_size));
+ else
+ ADD(add_1d(plan, COLI_ST_F32, experts, "ffn.gate.bias", error, error_size));
+ ADD(add_fp8(plan, moe, hidden, "ffn.shared_experts.w1", error, error_size));
+ ADD(add_fp8(plan, hidden, moe, "ffn.shared_experts.w2", error, error_size));
+ ADD(add_fp8(plan, moe, hidden, "ffn.shared_experts.w3", error, error_size));
+ ADD(add_1d(plan, COLI_ST_BF16, hidden, "ffn_norm.weight", error, error_size));
+
+ ADD(add_1d(plan, COLI_ST_F32, hc_params, "hc_attn_base", error, error_size));
+ ADD(add_2d(plan, COLI_ST_F32, hc_params, hc * hidden,
+ "hc_attn_fn", error, error_size));
+ ADD(add_1d(plan, COLI_ST_F32, 3, "hc_attn_scale", error, error_size));
+ ADD(add_1d(plan, COLI_ST_F32, hc_params, "hc_ffn_base", error, error_size));
+ ADD(add_2d(plan, COLI_ST_F32, hc_params, hc * hidden,
+ "hc_ffn_fn", error, error_size));
+ ADD(add_1d(plan, COLI_ST_F32, 3, "hc_ffn_scale", error, error_size));
+ return 0;
+}
+
+int coli_v4_layer_validate(const ColiDeepSeekV4LayerPlan *plan,
+ const ColiSafetensorsIndex *index,
+ ColiDeepSeekV4LayerStats *stats,
+ char *error, size_t error_size) {
+ if (!plan || !index)
+ return set_error(error, error_size, "invalid DeepSeek-V4 layer validation arguments");
+ ColiDeepSeekV4LayerStats local = {0};
+ for (size_t i = 0; i < plan->tensor_count; i++) {
+ const ColiDeepSeekV4TensorSpec *spec = &plan->tensors[i];
+ const ColiSafetensorsTensor *tensor = coli_st_find(index, spec->name);
+ if (!tensor)
+ return set_error(error, error_size, "missing tensor: %s", spec->name);
+ if (tensor->dtype != spec->dtype || tensor->rank != spec->rank)
+ return set_error(error, error_size, "dtype/rank mismatch: %s", spec->name);
+ for (int dimension = 0; dimension < spec->rank; dimension++)
+ if (tensor->shape[dimension] != spec->shape[dimension])
+ return set_error(error, error_size, "shape mismatch: %s", spec->name);
+ local.tensor_count++;
+ local.total_bytes += tensor->nbytes;
+ switch (tensor->dtype) {
+ case COLI_ST_BF16: local.bf16_bytes += tensor->nbytes; break;
+ case COLI_ST_F32: local.f32_bytes += tensor->nbytes; break;
+ case COLI_ST_F8_E4M3: local.fp8_weight_bytes += tensor->nbytes; break;
+ case COLI_ST_F8_E8M0: local.fp8_scale_bytes += tensor->nbytes; break;
+ case COLI_ST_I64: local.i64_bytes += tensor->nbytes; break;
+ default: break;
+ }
+ }
+ if (stats) *stats = local;
+ return 0;
+}
+
+void coli_v4_layer_free(ColiV4Engine *engine,
+ ColiDeepSeekV4LayerWeights *weights) {
+ (void)engine;
+ if (!weights) return;
+ for (size_t i = 0; i < weights->plan.tensor_count; i++) free(weights->data[i]);
+ memset(weights, 0, sizeof(*weights));
+}
+
+int coli_v4_layer_load(ColiV4Engine *engine,
+ ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const ColiSafetensorsIndex *index, int layer,
+ char *error, size_t error_size) {
+ (void)engine;
+ if (!weights) return set_error(error, error_size, "missing layer weights output");
+ memset(weights, 0, sizeof(*weights));
+ if (coli_v4_layer_plan(&weights->plan, config, layer, error, error_size) != 0 ||
+ coli_v4_layer_validate(&weights->plan, index, &weights->stats,
+ error, error_size) != 0)
+ return -1;
+ for (size_t i = 0; i < weights->plan.tensor_count; i++) {
+ const ColiDeepSeekV4TensorSpec *spec = &weights->plan.tensors[i];
+ const ColiSafetensorsTensor *tensor = coli_st_find(index, spec->name);
+ weights->data[i] = malloc((size_t)tensor->nbytes);
+ if (!weights->data[i]) {
+ coli_v4_layer_free(NULL, weights);
+ return set_error(error, error_size, "out of memory loading: %s", spec->name);
+ }
+ if (coli_st_read_tensor(index, tensor, weights->data[i]) != 0) {
+ coli_v4_layer_free(NULL, weights);
+ return set_error(error, error_size, "cannot read tensor: %s", spec->name);
+ }
+ }
+ return 0;
+}
+
+const void *coli_v4_layer_data(const ColiDeepSeekV4LayerWeights *weights,
+ const char *name,
+ const ColiDeepSeekV4TensorSpec **spec) {
+ if (spec) *spec = NULL;
+ if (!weights || !name) return NULL;
+ for (size_t i = 0; i < weights->plan.tensor_count; i++) {
+ if (strcmp(weights->plan.tensors[i].name, name) == 0) {
+ if (spec) *spec = &weights->plan.tensors[i];
+ return weights->data[i];
+ }
+ }
+ return NULL;
+}
+/* ---- end include deepseek_v4_layer.c ---- */
+
+#undef coli_v4_layer_free
+#undef coli_v4_layer_load
+
+#include "deepseek_v4_internal.h"
+
+enum { COLI_V4_RESIDENT_MAX_LAYERS_V2 = COLI_V4_RESIDENT_MAX_LAYERS };
+
+static int resident_enabled_v2(ColiV4Engine *engine) {
+ return engine && engine->runtime.dense_resident;
+}
+
+int coli_v4_layer_load(ColiV4Engine *engine,
+ ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const ColiSafetensorsIndex *index, int layer,
+ char *error, size_t error_size) {
+ const ColiDeepSeekV4Config *effective_config =
+ engine ? &engine->config : config;
+ if (!weights || !effective_config || !index || layer < 0 ||
+ layer >= effective_config->num_hidden_layers ||
+ layer >= COLI_V4_RESIDENT_MAX_LAYERS_V2) return -1;
+ if (!resident_enabled_v2(engine))
+ return coli_v4_layer_resident_reference_load(
+ NULL, weights, effective_config, index, layer, error, error_size);
+ if (engine->dense_resident.index && engine->dense_resident.index != index) {
+ if (error && error_size)
+ snprintf(error, error_size,
+ "resident V4 dense cache cannot switch model instances");
+ return -1;
+ }
+ engine->dense_resident.index = index;
+ if (!engine->dense_resident.ready[layer]) {
+ if (coli_v4_layer_resident_reference_load(
+ NULL, &engine->dense_resident.layers[layer], effective_config, index,
+ layer, error, error_size)) return -1;
+ engine->dense_resident.ready[layer] = 1;
+ engine->dense_resident.total_bytes +=
+ engine->dense_resident.layers[layer].stats.total_bytes;
+ if (layer == effective_config->num_hidden_layers - 1)
+ fprintf(stderr, "v4_dense_resident layers=%d bytes=%.3fGiB\n",
+ effective_config->num_hidden_layers,
+ engine->dense_resident.total_bytes / 1073741824.0);
+ }
+ *weights = engine->dense_resident.layers[layer]; return 0;
+}
+
+void coli_v4_layer_free(ColiV4Engine *engine,
+ ColiDeepSeekV4LayerWeights *weights) {
+ if (!weights) return;
+ int layer = weights->plan.layer;
+ if (engine && layer >= 0 && layer < COLI_V4_RESIDENT_MAX_LAYERS_V2 &&
+ engine->dense_resident.ready[layer] &&
+ weights->plan.tensor_count ==
+ engine->dense_resident.layers[layer].plan.tensor_count &&
+ weights->data[0] == engine->dense_resident.layers[layer].data[0]) {
+ memset(weights, 0, sizeof(*weights)); return;
+ }
+ coli_v4_layer_resident_reference_free(NULL, weights);
+}
+#endif /* COLI_V4_UNIT_LAYER_RESIDENT */
+
+#ifdef COLI_V4_UNIT_RESOURCE_PLAN
+/* ######## deepseek_v4_resource_plan.c ######## */
+#include "deepseek_v4_internal.h"
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <string.h>
+
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <unistd.h>
+#endif
+
+#define MIB UINT64_C(1048576)
+
+static int plan_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list arguments;
+ va_start(arguments, format);
+ vsnprintf(error, size, format, arguments);
+ va_end(arguments);
+ }
+ return -1;
+}
+
+static int add_u64(uint64_t a, uint64_t b, uint64_t *output) {
+ if (UINT64_MAX - a < b) return -1;
+ *output = a + b;
+ return 0;
+}
+
+static int multiply_u64(uint64_t a, uint64_t b, uint64_t *output) {
+ if (a && b > UINT64_MAX / a) return -1;
+ *output = a * b;
+ return 0;
+}
+
+uint64_t coli_v4_os_available_memory(void) {
+#ifdef _WIN32
+ MEMORYSTATUSEX status;
+ memset(&status, 0, sizeof(status));
+ status.dwLength = sizeof(status);
+ return GlobalMemoryStatusEx(&status) ? (uint64_t)status.ullAvailPhys : 0;
+#else
+ FILE *stream = fopen("/proc/meminfo", "r");
+ if (stream) {
+ char line[256];
+ unsigned long long kib = 0;
+ while (fgets(line, sizeof(line), stream))
+ if (sscanf(line, "MemAvailable: %llu kB", &kib) == 1) break;
+ fclose(stream);
+ if (kib) return (uint64_t)kib * 1024;
+ }
+ long pages = sysconf(_SC_AVPHYS_PAGES), page_size = sysconf(_SC_PAGESIZE);
+ if (pages <= 0 || page_size <= 0) return 0;
+ return (uint64_t)pages * (uint64_t)page_size;
+#endif
+}
+
+int coli_v4_resource_plan_compute(
+ ColiDeepSeekV4ResourcePlan *plan,
+ const ColiDeepSeekV4ResourceInputs *inputs,
+ char *error, size_t error_size) {
+ if (!plan || !inputs || !inputs->available_bytes ||
+ !inputs->maximum_layer_bytes || !inputs->expert_record_bytes ||
+ inputs->sparse_layers < 1 || inputs->routed_topk < 1 ||
+ inputs->experts_per_layer < inputs->routed_topk)
+ return plan_error(error, error_size, "invalid V4 resource-plan inputs");
+ memset(plan, 0, sizeof(*plan));
+ plan->os_available_bytes = inputs->available_bytes;
+ uint64_t available = inputs->available_bytes;
+ int explicit_process_limit = inputs->user_limit_bytes &&
+ inputs->user_limit_bytes < available;
+ if (explicit_process_limit)
+ available = inputs->user_limit_bytes;
+ plan->planner_available_bytes = available;
+
+ /* A process limit leaves all RAM outside the limit to the OS. Automatic
+ * mode starts from MemAvailable and therefore reserves that share here. */
+ uint64_t system = explicit_process_limit ? 0 : available / 8;
+ if (!explicit_process_limit && system < 512 * MIB) system = 512 * MIB;
+ if (system > 4096 * MIB) system = 4096 * MIB;
+ plan->system_reserve_bytes = system;
+
+ uint64_t layers_twice;
+ if (multiply_u64(inputs->maximum_layer_bytes, 2, &layers_twice) ||
+ add_u64(layers_twice, inputs->runtime_other_bytes,
+ &plan->runtime_reserve_bytes))
+ return plan_error(error, error_size, "V4 runtime reserve overflow");
+
+ uint64_t per_slot;
+ if (multiply_u64((uint64_t)inputs->sparse_layers,
+ inputs->expert_record_bytes, &per_slot) ||
+ multiply_u64(per_slot, (uint64_t)inputs->routed_topk,
+ &plan->minimum_expert_bytes))
+ return plan_error(error, error_size, "V4 expert-cache size overflow");
+
+ uint64_t fixed;
+ if (add_u64(system, plan->runtime_reserve_bytes, &fixed) || fixed >= available)
+ return plan_error(error, error_size,
+ "available RAM cannot hold V4 runtime reserves");
+ uint64_t usable = available - fixed;
+ if (usable < plan->minimum_expert_bytes)
+ return plan_error(error, error_size,
+ "V4 minimum expert cache needs %.2f GiB but only %.2f GiB remains",
+ plan->minimum_expert_bytes / 1073741824.0,
+ usable / 1073741824.0);
+
+ uint64_t slots = usable / per_slot;
+ if (slots > (uint64_t)inputs->experts_per_layer)
+ slots = (uint64_t)inputs->experts_per_layer;
+ if (slots < (uint64_t)inputs->routed_topk)
+ slots = (uint64_t)inputs->routed_topk;
+ plan->slots_per_layer = (int)slots;
+ if (multiply_u64(per_slot, slots, &plan->expert_cache_bytes) ||
+ add_u64(fixed, plan->expert_cache_bytes, &plan->projected_bytes))
+ return plan_error(error, error_size, "V4 projected memory overflow");
+ if (plan->projected_bytes > available)
+ return plan_error(error, error_size, "V4 plan exceeds available RAM");
+ return 0;
+}
+#endif /* COLI_V4_UNIT_RESOURCE_PLAN */
+
+#ifdef COLI_V4_UNIT_HEAD_CACHE
+/* ######## deepseek_v4_head_cache.c ######## */
+#include "deepseek_v4_internal.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "safetensors_index.h"
+
+static int find_head(const char *model_dir, ColiSafetensorsIndex **index,
+ const ColiSafetensorsTensor **head,
+ char *error, size_t error_size) {
+ *index = NULL; *head = NULL;
+ if (coli_st_index_open(index, model_dir, error, error_size)) return -1;
+ *head = coli_st_find(*index, "head.weight");
+ if (!*head || (*head)->dtype != COLI_ST_BF16 || (*head)->rank != 2) {
+ snprintf(error, error_size, "missing or invalid BF16 head.weight");
+ coli_st_index_close(*index); *index = NULL; return -1;
+ }
+ return 0;
+}
+
+int coli_v4_head_cache_probe(const char *model_dir, uint64_t *bytes,
+ char *error, size_t error_size) {
+ ColiSafetensorsIndex *index;
+ const ColiSafetensorsTensor *head;
+ if (!bytes || find_head(model_dir, &index, &head, error, error_size)) return -1;
+ *bytes = head->nbytes;
+ coli_st_index_close(index); return 0;
+}
+
+int coli_v4_head_cache_load(ColiV4Engine *engine, const char *model_dir,
+ char *error, size_t error_size) {
+ if (!engine) {
+ snprintf(error, error_size, "head cache requires a V4 engine");
+ return -1;
+ }
+ ColiSafetensorsIndex *index;
+ const ColiSafetensorsTensor *head;
+ if (find_head(model_dir, &index, &head, error, error_size)) return -1;
+ unsigned char *data = malloc((size_t)head->nbytes);
+ if (!data || coli_st_read_at(index, head->shard, head->offset,
+ (size_t)head->nbytes, data)) {
+ free(data); coli_st_index_close(index);
+ snprintf(error, error_size, "cannot load resident BF16 head.weight");
+ return -1;
+ }
+ free(engine->head_cache.data);
+ engine->head_cache.data = data;
+ engine->head_cache.bytes = head->nbytes;
+ engine->head_cache.offset = head->offset;
+ engine->head_cache.shard = head->shard;
+ coli_st_index_close(index); return 0;
+}
+
+uint64_t coli_v4_head_cache_bytes(const ColiV4Engine *engine) {
+ return engine ? engine->head_cache.bytes : 0;
+}
+
+const void *coli_v4_head_cache_data(const ColiV4Engine *engine,
+ int shard, uint64_t offset, size_t length) {
+ if (!engine || !engine->head_cache.data || shard != engine->head_cache.shard ||
+ offset < engine->head_cache.offset ||
+ offset - engine->head_cache.offset > engine->head_cache.bytes ||
+ length > engine->head_cache.bytes - (offset - engine->head_cache.offset))
+ return NULL;
+ return engine->head_cache.data +
+ (size_t)(offset - engine->head_cache.offset);
+}
+
+int coli_st_read_at_engine(ColiV4Engine *engine,
+ const ColiSafetensorsIndex *index, int shard,
+ uint64_t offset, size_t length, void *destination) {
+ if (engine && engine->head_cache.data && destination &&
+ shard == engine->head_cache.shard &&
+ offset >= engine->head_cache.offset &&
+ offset - engine->head_cache.offset <= engine->head_cache.bytes &&
+ length <= engine->head_cache.bytes -
+ (offset - engine->head_cache.offset)) {
+ memcpy(destination,
+ engine->head_cache.data +
+ (size_t)(offset - engine->head_cache.offset),
+ length);
+ return 0;
+ }
+ return coli_st_read_at(index, shard, offset, length, destination);
+}
+#endif /* COLI_V4_UNIT_HEAD_CACHE */
+
+#ifdef COLI_V4_UNIT_EXPERT_STORE_AUTO
+/* ######## deepseek_v4_expert_store_auto.c ######## */
+#include "deepseek_v4_dspark.h"
+/* ---- begin inlined deepseek_v4_expert_store_auto_v5.c ---- */
+#include "deepseek_v4_internal.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+#include "deepseek_v4_dspark.h"
+#include "safetensors_index.h"
+
+#define MIB UINT64_C(1048576)
+#define GIB UINT64_C(1073741824)
+
+static uint64_t expert_record_bytes(const ColiSafetensorsIndex *index) {
+ static const char *parts[] = {
+ "layers.0.ffn.experts.0.w1.weight", "layers.0.ffn.experts.0.w1.scale",
+ "layers.0.ffn.experts.0.w2.weight", "layers.0.ffn.experts.0.w2.scale",
+ "layers.0.ffn.experts.0.w3.weight", "layers.0.ffn.experts.0.w3.scale",
+ };
+ uint64_t total = 0;
+ for (size_t i = 0; i < sizeof(parts) / sizeof(parts[0]); i++) {
+ const ColiSafetensorsTensor *tensor = coli_st_find(index, parts[i]);
+ if (!tensor || UINT64_MAX - total < tensor->nbytes) return 0;
+ total += tensor->nbytes;
+ }
+ return total;
+}
+
+static uint64_t context_bytes(const ColiDeepSeekV4Config *config, int context) {
+ uint64_t total = (uint64_t)config->num_hidden_layers *
+ config->sliding_window * config->head_dim * sizeof(float);
+ for (int layer = 0; layer < config->num_hidden_layers; layer++) {
+ int ratio = config->compress_ratios[layer];
+ if (!ratio) continue;
+ uint64_t compressed = ((uint64_t)context + (uint64_t)ratio - 1) /
+ (uint64_t)ratio;
+ total += compressed * config->head_dim * sizeof(float);
+ if (ratio == 4)
+ total += compressed * config->index_head_dim * sizeof(float);
+ }
+ return total;
+}
+
+static int build_runtime_plan(ColiV4Engine *engine,
+ const ColiDeepSeekV4ExpertStoreOptions *options,
+ ColiDeepSeekV4ResourcePlan *plan,
+ char *error, size_t error_size) {
+ ColiDeepSeekV4Config config;
+ ColiSafetensorsIndex *index = NULL;
+ if (!engine) {
+ snprintf(error, error_size, "V4 runtime requires an engine");
+ return -1;
+ }
+ if (coli_v4_config_load(&config, options->model_dir, error, error_size) ||
+ coli_st_index_open(&index, options->model_dir, error, error_size))
+ return -1;
+ uint64_t maximum_layer = 0;
+ for (int layer = 0; layer < config.num_hidden_layers; layer++) {
+ ColiDeepSeekV4LayerPlan layer_plan;
+ ColiDeepSeekV4LayerStats stats;
+ if (coli_v4_layer_plan(&layer_plan, &config, layer,
+ error, error_size) ||
+ coli_v4_layer_validate(&layer_plan, index, &stats,
+ error, error_size)) {
+ coli_st_index_close(index); return -1;
+ }
+ if (stats.total_bytes > maximum_layer) maximum_layer = stats.total_bytes;
+ }
+ uint64_t record = expert_record_bytes(index);
+ coli_st_index_close(index);
+ if (!record) {
+ snprintf(error, error_size, "cannot determine V4 expert record size");
+ return -1;
+ }
+ ColiDeepSeekV4RuntimeOptions *runtime = &engine->runtime;
+ int context = runtime->context_tokens;
+ if (context > config.max_position_embeddings)
+ context = config.max_position_embeddings;
+ uint64_t hidden = (uint64_t)64 * config.hc_mult * config.hidden_size *
+ sizeof(float) * 2;
+ uint64_t scratch = 512 * MIB;
+ uint64_t runtime_other = context_bytes(&config, context) + hidden + scratch;
+ uint64_t available = coli_v4_os_available_memory();
+ if (!available) {
+ snprintf(error, error_size, "cannot determine OS available memory");
+ return -1;
+ }
+ ColiDeepSeekV4ResourceInputs inputs = {
+ available, runtime->memory_limit_bytes, maximum_layer,
+ runtime_other, record, config.num_hidden_layers,
+ config.num_experts_per_tok, config.n_routed_experts,
+ };
+ return coli_v4_resource_plan_compute(plan, &inputs, error, error_size);
+}
+
+static int v5_dense_inventory(const char *model_dir,
+ uint64_t *bytes,
+ char *error, size_t error_size) {
+ ColiDeepSeekV4Config config; ColiSafetensorsIndex *index = NULL;
+ if (coli_v4_config_load(&config, model_dir, error, error_size) ||
+ coli_st_index_open(&index, model_dir, error, error_size)) return -1;
+ uint64_t total = 0;
+ for (int layer = 0; layer < config.num_hidden_layers; layer++) {
+ ColiDeepSeekV4LayerPlan layer_plan; ColiDeepSeekV4LayerStats stats;
+ if (coli_v4_layer_plan(&layer_plan, &config, layer,
+ error, error_size) ||
+ coli_v4_layer_validate(&layer_plan, index, &stats,
+ error, error_size)) {
+ coli_st_index_close(index); return -1;
+ }
+ total += stats.total_bytes;
+ }
+ coli_st_index_close(index); *bytes = total; return 0;
+}
+
+int coli_v4_expert_store_open_planned(
+ ColiV4Engine *engine,
+ const ColiDeepSeekV4ExpertStoreOptions *options, ColiExpertStore **output,
+ char *error, size_t error_size) {
+ if (!options || !engine) return -1;
+ ColiDeepSeekV4ResourcePlan plan;
+ ColiDeepSeekV4RuntimeOptions *runtime = &engine->runtime;
+ if (build_runtime_plan(engine, options, &plan, error, error_size)) return -1;
+ uint64_t per_slot = plan.expert_cache_bytes /
+ (uint64_t)plan.slots_per_layer;
+ uint64_t head_bytes = 0, dense_bytes = 0;
+ if (coli_v4_head_cache_probe(options->model_dir, &head_bytes,
+ error, error_size) ||
+ v5_dense_inventory(options->model_dir, &dense_bytes,
+ error, error_size)) return -1;
+
+ uint64_t dspark_bytes = 0, dspark_full_bytes = 0;
+ uint64_t dspark_full_experts = 0;
+ const char *dspark_model = runtime->dspark_model_dir;
+ ColiV4DSparkMemoryPlan dspark = {0};
+ if (dspark_model && *dspark_model) {
+ ColiDeepSeekV4Config ds_config;
+ ColiDeepSeekV4DSparkManifest manifest;
+ if (coli_v4_config_load(&ds_config, dspark_model, error, error_size) ||
+ coli_v4_dspark_memory_plan(dspark_model, &ds_config, &dspark,
+ error, error_size) ||
+ coli_v4_dspark_inspect(dspark_model, &ds_config, &manifest,
+ error, error_size)) return -1;
+ uint64_t all_stages = 0;
+ for (int stage = 0; stage < manifest.stage_count; stage++)
+ all_stages += manifest.common_stage_bytes[stage];
+ dspark_full_experts = dspark.expert_record_bytes *
+ (uint64_t)manifest.stage_count * ds_config.n_routed_experts;
+ dspark_full_bytes = dspark.resident_heads_bytes + all_stages +
+ dspark_full_experts + dspark.working_bytes;
+ dspark_bytes = dspark.incremental_reserve_bytes;
+ }
+
+ uint64_t fixed = plan.system_reserve_bytes + plan.runtime_reserve_bytes;
+ uint64_t full_required = fixed + head_bytes + dense_bytes +
+ dspark_full_bytes + plan.minimum_expert_bytes;
+ int full_resident = dspark_full_bytes &&
+ full_required <= plan.planner_available_bytes;
+ if (full_resident) {
+ dspark_bytes = dspark_full_bytes;
+ runtime->dense_resident = 1;
+ runtime->dspark_resident = 1;
+ runtime->dspark_expert_cache_bytes = dspark_full_experts;
+ } else {
+ dense_bytes = 0;
+ runtime->dense_resident = 0;
+ runtime->dspark_resident = 0;
+ runtime->dspark_expert_cache_bytes = 0;
+ }
+ if (dspark_bytes + dense_bytes > plan.planner_available_bytes - fixed) {
+ snprintf(error, error_size, "resident V4 tiers exceed available RAM");
+ return -1;
+ }
+ uint64_t safe_payload = plan.planner_available_bytes - fixed -
+ dspark_bytes - dense_bytes;
+ int requested_head = -1;
+ int resident_head = safe_payload >= plan.minimum_expert_bytes +
+ head_bytes + 256 * MIB;
+ if (requested_head == 0) resident_head = 0;
+ if (requested_head == 1 && !resident_head) {
+ snprintf(error, error_size, "resident BF16 head does not fit RAM plan");
+ return -1;
+ }
+ uint64_t cache_limit = safe_payload - (resident_head ? head_bytes : 0);
+
+ if (cache_limit < plan.minimum_expert_bytes) {
+ snprintf(error, error_size, "resident tiers leave too little target cache");
+ return -1;
+ }
+ int slots = (int)(cache_limit / per_slot);
+ if (slots > plan.slots_per_layer) slots = plan.slots_per_layer;
+ if (slots < options->experts_per_layer && slots < 6) slots = 6;
+ plan.expert_cache_bytes = (uint64_t)slots * per_slot;
+ plan.projected_bytes = fixed + dense_bytes + dspark_bytes +
+ plan.expert_cache_bytes + (resident_head ? head_bytes : 0);
+ if (resident_head && coli_v4_head_cache_load(
+ engine, options->model_dir, error, error_size)) return -1;
+ fprintf(stderr,
+ "ram_tiers available=%.2fGiB dense=%s(%.2fGiB) "
+ "dspark=%s(%.2fGiB) dspark_experts=%.2fGiB "
+ "target_slots=%d target_cache=%.2fGiB head=%s projected=%.2fGiB\n",
+ plan.planner_available_bytes / (double)GIB,
+ full_resident ? "resident" : "streamed", dense_bytes / (double)GIB,
+ full_resident ? "resident" : "streamed", dspark_bytes / (double)GIB,
+ dspark_full_experts / (double)GIB, slots,
+ plan.expert_cache_bytes / (double)GIB,
+ resident_head ? "resident-bf16" : "streamed-bf16",
+ plan.projected_bytes / (double)GIB);
+ ColiDeepSeekV4ExpertStoreOptions automatic = *options;
+ automatic.cache_bytes = plan.expert_cache_bytes;
+ automatic.pin_slots_per_layer = runtime->pin_slots_per_layer;
+ automatic.repin_interval = runtime->repin_interval;
+ return coli_deepseek_v4_expert_store_open(
+ &automatic, output, error, error_size);
+}
+/* ---- end inlined deepseek_v4_expert_store_auto_v5.c ---- */
+#endif /* COLI_V4_UNIT_EXPERT_STORE_AUTO */
+
+#ifdef COLI_V4_UNIT_MATH
+/* ######## deepseek_v4_math.c ######## */
+#include "deepseek_v4_internal.h"
+
+#include <math.h>
+#include <stdlib.h>
+
+static float sigmoidf_stable(float value) {
+ if (value >= 0.0f) {
+ float decay = expf(-value);
+ return 1.0f / (1.0f + decay);
+ }
+ float growth = expf(value);
+ return growth / (1.0f + growth);
+}
+
+int coli_v4_hc_split_sinkhorn(float *pre, float *post, float *comb,
+ const float *mixes, const float scale[3],
+ const float *base, int hc, int iterations,
+ float eps) {
+ if (!pre || !post || !comb || !mixes || !scale || !base ||
+ hc < 1 || iterations < 1 || eps < 0.0f)
+ return -1;
+ for (int index = 0; index < hc; index++) {
+ pre[index] = sigmoidf_stable(
+ mixes[index] * scale[0] + base[index]) + eps;
+ post[index] = 2.0f * sigmoidf_stable(
+ mixes[hc + index] * scale[1] + base[hc + index]);
+ }
+ int matrix_offset = 2 * hc;
+ for (int row = 0; row < hc; row++) {
+ float maximum = -INFINITY;
+ for (int column = 0; column < hc; column++) {
+ int index = matrix_offset + row * hc + column;
+ float value = mixes[index] * scale[2] + base[index];
+ comb[row * hc + column] = value;
+ if (value > maximum) maximum = value;
+ }
+ float sum = 0.0f;
+ for (int column = 0; column < hc; column++) {
+ float value = expf(comb[row * hc + column] - maximum);
+ comb[row * hc + column] = value;
+ sum += value;
+ }
+ for (int column = 0; column < hc; column++)
+ comb[row * hc + column] = comb[row * hc + column] / sum + eps;
+ }
+ float *sums = malloc((size_t)hc * sizeof(*sums));
+ if (!sums) return -1;
+ for (int column = 0; column < hc; column++) {
+ float sum = 0.0f;
+ for (int row = 0; row < hc; row++)
+ sum += comb[row * hc + column];
+ sums[column] = sum;
+ }
+ for (int row = 0; row < hc; row++)
+ for (int column = 0; column < hc; column++)
+ comb[row * hc + column] /= sums[column] + eps;
+
+ for (int iteration = 1; iteration < iterations; iteration++) {
+ for (int row = 0; row < hc; row++) {
+ float sum = 0.0f;
+ for (int column = 0; column < hc; column++)
+ sum += comb[row * hc + column];
+ sums[row] = sum;
+ }
+ for (int row = 0; row < hc; row++)
+ for (int column = 0; column < hc; column++)
+ comb[row * hc + column] /= sums[row] + eps;
+ for (int column = 0; column < hc; column++) {
+ float sum = 0.0f;
+ for (int row = 0; row < hc; row++)
+ sum += comb[row * hc + column];
+ sums[column] = sum;
+ }
+ for (int row = 0; row < hc; row++)
+ for (int column = 0; column < hc; column++)
+ comb[row * hc + column] /= sums[column] + eps;
+ }
+ free(sums);
+ return 0;
+}
+
+int coli_v4_hc_pre(float *output, float *post, float *comb,
+ const float *input, const float *hc_fn,
+ const float scale[3], const float *base,
+ int hc, int dimension, int iterations,
+ float norm_eps, float hc_eps) {
+ if (!output || !post || !comb || !input || !hc_fn || !scale || !base ||
+ hc < 1 || dimension < 1 || norm_eps < 0.0f)
+ return -1;
+ int flattened = hc * dimension;
+ int mix_count = (2 + hc) * hc;
+ float mean_square = 0.0f;
+ for (int index = 0; index < flattened; index++)
+ mean_square += input[index] * input[index];
+ float inverse_rms = 1.0f / sqrtf(mean_square / flattened + norm_eps);
+ float *mixes = malloc((size_t)mix_count * sizeof(*mixes));
+ float *pre = malloc((size_t)hc * sizeof(*pre));
+ if (!mixes || !pre) {
+ free(mixes);
+ free(pre);
+ return -1;
+ }
+ for (int row = 0; row < mix_count; row++) {
+ float sum = 0.0f;
+ for (int column = 0; column < flattened; column++)
+ sum += hc_fn[(size_t)row * flattened + column] * input[column];
+ mixes[row] = sum * inverse_rms;
+ }
+ if (coli_v4_hc_split_sinkhorn(pre, post, comb, mixes, scale, base,
+ hc, iterations, hc_eps) != 0) {
+ free(pre);
+ free(mixes);
+ return -1;
+ }
+ for (int column = 0; column < dimension; column++) {
+ float sum = 0.0f;
+ for (int copy = 0; copy < hc; copy++)
+ sum += pre[copy] * input[copy * dimension + column];
+ output[column] = sum;
+ }
+ free(pre);
+ free(mixes);
+ return 0;
+}
+
+int coli_v4_hc_post(float *output, const float *branch,
+ const float *residual, const float *post,
+ const float *comb, int hc, int dimension) {
+ if (!output || !branch || !residual || !post || !comb ||
+ hc < 1 || dimension < 1)
+ return -1;
+ for (int destination = 0; destination < hc; destination++) {
+ for (int column = 0; column < dimension; column++) {
+ float value = 0.0f;
+ for (int source = 0; source < hc; source++)
+ value += comb[source * hc + destination] *
+ residual[source * dimension + column];
+ value += post[destination] * branch[column];
+ output[destination * dimension + column] = value;
+ }
+ }
+ return 0;
+}
+
+int coli_v4_rmsnorm(float *output, const float *input, const float *weight,
+ int dimension, float eps) {
+ if (!output || !input || !weight || dimension < 1 || eps < 0.0f)
+ return -1;
+ float mean_square = 0.0f;
+ for (int index = 0; index < dimension; index++)
+ mean_square += input[index] * input[index];
+ float inverse_rms = 1.0f / sqrtf(mean_square / dimension + eps);
+ for (int index = 0; index < dimension; index++)
+ output[index] = input[index] * inverse_rms * weight[index];
+ return 0;
+}
+
+int coli_v4_rope_precompute(float *cosines, float *sines,
+ int dimension, int sequence_length,
+ int original_sequence_length, float base,
+ float factor, int beta_fast, int beta_slow) {
+ if (!cosines || !sines || dimension < 2 || (dimension & 1) ||
+ sequence_length < 1 || !(base > 1.0f) || !(factor > 0.0f))
+ return -1;
+ int pairs = dimension / 2;
+ int low = 0, high = -1;
+ if (original_sequence_length > 0) {
+ const float two_pi = 6.2831853071795864769f;
+ float denominator = 2.0f * logf(base);
+ float low_value = dimension * logf(
+ original_sequence_length / (beta_fast * two_pi)) / denominator;
+ float high_value = dimension * logf(
+ original_sequence_length / (beta_slow * two_pi)) / denominator;
+ low = (int)floorf(low_value);
+ high = (int)ceilf(high_value);
+ if (low < 0) low = 0;
+ if (high > dimension - 1) high = dimension - 1;
+ }
+ for (int pair = 0; pair < pairs; pair++) {
+ float frequency = 1.0f / powf(base, (float)(2 * pair) / dimension);
+ if (original_sequence_length > 0) {
+ float width = high == low ? 0.001f : (float)(high - low);
+ float ramp = (pair - low) / width;
+ if (ramp < 0.0f) ramp = 0.0f;
+ if (ramp > 1.0f) ramp = 1.0f;
+ float smooth = 1.0f - ramp;
+ frequency = frequency / factor * (1.0f - smooth) + frequency * smooth;
+ }
+ for (int position = 0; position < sequence_length; position++) {
+ size_t index = (size_t)position * pairs + pair;
+ float angle = position * frequency;
+ cosines[index] = cosf(angle);
+ sines[index] = sinf(angle);
+ }
+ }
+ return 0;
+}
+
+int coli_v4_rope_apply(float *vectors, int vector_count, int dimension,
+ const float *cosines, const float *sines, int inverse) {
+ if (!vectors || !cosines || !sines || vector_count < 1 ||
+ dimension < 2 || (dimension & 1))
+ return -1;
+ int pairs = dimension / 2;
+ float direction = inverse ? -1.0f : 1.0f;
+ for (int vector = 0; vector < vector_count; vector++) {
+ for (int pair = 0; pair < pairs; pair++) {
+ size_t value_index = (size_t)vector * dimension + 2 * pair;
+ size_t frequency_index = (size_t)vector * pairs + pair;
+ float real = vectors[value_index];
+ float imaginary = vectors[value_index + 1];
+ float cosine = cosines[frequency_index];
+ float sine = sines[frequency_index] * direction;
+ vectors[value_index] = real * cosine - imaginary * sine;
+ vectors[value_index + 1] = real * sine + imaginary * cosine;
+ }
+ }
+ return 0;
+}
+
+static float softplusf_stable(float value) {
+ return fmaxf(value, 0.0f) + log1pf(expf(-fabsf(value)));
+}
+
+int coli_v4_route(float *weights, int *indices, const float *hidden,
+ const float *gate, const float *bias,
+ const int *forced_indices, int experts, int dimension,
+ int topk, float route_scale) {
+ if (!weights || !indices || !hidden || !gate || experts < 1 ||
+ dimension < 1 || topk < 1 || topk > experts)
+ return -1;
+ float *scores = malloc((size_t)experts * sizeof(*scores));
+ float *selection = malloc((size_t)experts * sizeof(*selection));
+ unsigned char *selected = calloc((size_t)experts, 1);
+ if (!scores || !selection || !selected) {
+ free(scores);
+ free(selection);
+ free(selected);
+ return -1;
+ }
+ for (int expert = 0; expert < experts; expert++) {
+ float sum = 0.0f;
+ for (int column = 0; column < dimension; column++)
+ sum += gate[(size_t)expert * dimension + column] * hidden[column];
+ scores[expert] = sqrtf(softplusf_stable(sum));
+ selection[expert] = scores[expert] + (bias ? bias[expert] : 0.0f);
+ }
+ if (forced_indices) {
+ for (int rank = 0; rank < topk; rank++) {
+ if (forced_indices[rank] < 0 || forced_indices[rank] >= experts) {
+ free(selected);
+ free(selection);
+ free(scores);
+ return -1;
+ }
+ indices[rank] = forced_indices[rank];
+ }
+ } else {
+ for (int rank = 0; rank < topk; rank++) {
+ int best = -1;
+ for (int expert = 0; expert < experts; expert++) {
+ if (!selected[expert] &&
+ (best < 0 || selection[expert] > selection[best]))
+ best = expert;
+ }
+ indices[rank] = best;
+ selected[best] = 1;
+ }
+ }
+ float total = 0.0f;
+ for (int rank = 0; rank < topk; rank++)
+ total += scores[indices[rank]];
+ if (!(total > 0.0f)) {
+ free(selected);
+ free(selection);
+ free(scores);
+ return -1;
+ }
+ for (int rank = 0; rank < topk; rank++)
+ weights[rank] = scores[indices[rank]] / total * route_scale;
+ free(selected);
+ free(selection);
+ free(scores);
+ return 0;
+}
+
+int coli_v4_swiglu(float *output, const float *gate, const float *up,
+ int dimension, float limit) {
+ if (!output || !gate || !up || dimension < 1 || limit < 0.0f)
+ return -1;
+ for (int index = 0; index < dimension; index++) {
+ float gate_value = gate[index];
+ float up_value = up[index];
+ if (limit > 0.0f) {
+ gate_value = fminf(gate_value, limit);
+ up_value = fmaxf(-limit, fminf(up_value, limit));
+ }
+ output[index] = gate_value * sigmoidf_stable(gate_value) * up_value;
+ }
+ return 0;
+}
+#endif /* COLI_V4_UNIT_MATH */
+
+#ifdef COLI_V4_UNIT_ATTENTION
+/* ######## deepseek_v4_attention.c ######## */
+#include "deepseek_v4_internal.h"
+
+#include <math.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "native_quant.h"
+
+static int set_error(char *error, size_t size, const char *format, ...);
+
+struct ColiDeepSeekV4WindowAttentionState {
+ int window_size;
+ int head_dim;
+ int layer;
+ int ratio;
+ float *kv;
+ ColiDeepSeekV4CompressorState *compressor;
+ ColiDeepSeekV4Indexer *indexer;
+ float *compressed;
+ int compressed_count;
+ int compressed_capacity;
+};
+
+int coli_v4_window_attention_create(ColiDeepSeekV4WindowAttentionState **output,
+ const ColiDeepSeekV4Config *config) {
+ if (!output || !config || config->sliding_window < 1 || config->head_dim < 1)
+ return -1;
+ *output = calloc(1, sizeof(**output));
+ if (!*output) return -1;
+ (*output)->window_size = config->sliding_window;
+ (*output)->head_dim = config->head_dim;
+ (*output)->layer = -1;
+ (*output)->kv = calloc((size_t)config->sliding_window * config->head_dim,
+ sizeof(*(*output)->kv));
+ if (!(*output)->kv) {
+ free(*output);
+ *output = NULL;
+ return -1;
+ }
+ return 0;
+}
+
+void coli_v4_window_attention_reset(ColiDeepSeekV4WindowAttentionState *state) {
+ if (!state) return;
+ memset(state->kv, 0,
+ (size_t)state->window_size * state->head_dim * sizeof(*state->kv));
+ state->compressed_count = 0;
+ if (state->compressor) coli_v4_compressor_reset(state->compressor);
+ if (state->indexer) coli_v4_indexer_reset(state->indexer);
+}
+
+void coli_v4_window_attention_destroy(ColiDeepSeekV4WindowAttentionState *state) {
+ if (!state) return;
+ coli_v4_indexer_destroy(state->indexer);
+ coli_v4_compressor_destroy(state->compressor);
+ free(state->compressed);
+ free(state->kv);
+ free(state);
+}
+
+static int prepare_compressed_state(
+ ColiDeepSeekV4WindowAttentionState *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, char *error, size_t error_size) {
+ int ratio = weights->plan.compression_ratio;
+ if (!ratio) return 0;
+ if (state->layer < 0) {
+ state->layer = weights->plan.layer;
+ state->ratio = ratio;
+ state->compressed_capacity = 16;
+ state->compressed = calloc((size_t)state->compressed_capacity * state->head_dim,
+ sizeof(*state->compressed));
+ if (!state->compressed || coli_v4_compressor_create(
+ &state->compressor, weights, config, error, error_size)) return -1;
+ if (ratio == 4 && coli_v4_indexer_create(
+ &state->indexer, weights, config, config->max_position_embeddings,
+ error, error_size)) return -1;
+ } else if (state->layer != weights->plan.layer || state->ratio != ratio) {
+ return set_error(error, error_size, "attention state belongs to another layer");
+ }
+ if (coli_v4_compressor_bind_weights(state->compressor, weights,
+ error, error_size)) return -1;
+ if (state->indexer && coli_v4_indexer_bind_weights(
+ state->indexer, weights, error, error_size)) return -1;
+ return 0;
+}
+
+static int grow_compressed_state(ColiDeepSeekV4WindowAttentionState *state,
+ char *error, size_t error_size) {
+ if (state->compressed_count < state->compressed_capacity) return 0;
+ int capacity = state->compressed_capacity * 2;
+ float *grown = realloc(state->compressed,
+ (size_t)capacity * state->head_dim * sizeof(*grown));
+ if (!grown) return set_error(error, error_size, "cannot grow compressed KV cache");
+ state->compressed = grown;
+ state->compressed_capacity = capacity;
+ return 0;
+}
+
+static int set_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list arguments;
+ va_start(arguments, format);
+ vsnprintf(error, size, format, arguments);
+ va_end(arguments);
+ }
+ return -1;
+}
+
+static const void *layer_data(const ColiDeepSeekV4LayerWeights *weights,
+ const char *suffix,
+ const ColiDeepSeekV4TensorSpec **spec) {
+ char name[COLI_V4_MAX_TENSOR_NAME];
+ snprintf(name, sizeof(name), "layers.%d.%s", weights->plan.layer, suffix);
+ return coli_v4_layer_data(weights, name, spec);
+}
+
+static int fp8_view(ColiTensorView *view,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const char *prefix) {
+ char suffix[128];
+ const ColiDeepSeekV4TensorSpec *weight_spec = NULL, *scale_spec = NULL;
+ snprintf(suffix, sizeof(suffix), "%s.weight", prefix);
+ const void *data = layer_data(weights, suffix, &weight_spec);
+ snprintf(suffix, sizeof(suffix), "%s.scale", prefix);
+ const void *scales = layer_data(weights, suffix, &scale_spec);
+ if (!data || !scales || !weight_spec || !scale_spec ||
+ weight_spec->dtype != COLI_ST_F8_E4M3 ||
+ scale_spec->dtype != COLI_ST_F8_E8M0 || weight_spec->rank != 2)
+ return -1;
+ *view = (ColiTensorView){
+ COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_UE8M0, data, scales,
+ (size_t)(weight_spec->shape[0] * weight_spec->shape[1]),
+ (size_t)(scale_spec->shape[0] * scale_spec->shape[1]),
+ weight_spec->shape[0], weight_spec->shape[1], 128, 128
+ };
+ return 0;
+}
+
+static int decode_bf16(float *output, const void *data, size_t count) {
+ if (!output || !data) return -1;
+ const uint16_t *values = data;
+ for (size_t i = 0; i < count; i++) output[i] = coli_bf16_decode(values[i]);
+ return 0;
+}
+
+static int attention_token_impl(float *output,
+ ColiDeepSeekV4WindowAttentionState *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const float *input, int position,
+ char *error, size_t error_size) {
+ if (!output || !weights || !config || !input || position < 0 ||
+ (!state && weights->plan.compression_ratio != 0 && position != 0))
+ return set_error(error, error_size, "invalid uncompressed attention arguments");
+ int hidden = config->hidden_size;
+ int heads = config->num_attention_heads;
+ int head_dim = config->head_dim;
+ int rope_dim = config->qk_rope_head_dim;
+ int q_rank = config->q_lora_rank;
+ int groups = config->o_groups;
+ int o_rank = config->o_lora_rank;
+ if (hidden < 1 || heads < 1 || head_dim < 1 || rope_dim < 2 ||
+ rope_dim > head_dim || q_rank < 1 || groups < 1 || heads % groups)
+ return set_error(error, error_size, "unsupported attention dimensions");
+
+ ColiTensorView wq_a, wq_b, wkv, wo_a, wo_b;
+ if (fp8_view(&wq_a, weights, "attn.wq_a") ||
+ fp8_view(&wq_b, weights, "attn.wq_b") ||
+ fp8_view(&wkv, weights, "attn.wkv") ||
+ fp8_view(&wo_a, weights, "attn.wo_a") ||
+ fp8_view(&wo_b, weights, "attn.wo_b"))
+ return set_error(error, error_size, "missing native FP8 attention tensor");
+
+ float *qa = calloc((size_t)q_rank, sizeof(*qa));
+ float *q = calloc((size_t)heads * head_dim, sizeof(*q));
+ float *kv = calloc((size_t)head_dim, sizeof(*kv));
+ float *attended = calloc((size_t)heads * head_dim, sizeof(*attended));
+ float *oa = calloc((size_t)groups * o_rank, sizeof(*oa));
+ float *norm_weight = calloc((size_t)(q_rank > head_dim ? q_rank : head_dim),
+ sizeof(*norm_weight));
+ float *cosines = calloc((size_t)rope_dim / 2, sizeof(*cosines));
+ float *sines = calloc((size_t)rope_dim / 2, sizeof(*sines));
+ int *compressed_indices = NULL;
+ int compressed_selected = 0;
+ if (!qa || !q || !kv || !attended || !oa || !norm_weight || !cosines || !sines) {
+ free(sines); free(cosines); free(norm_weight); free(oa);
+ free(attended); free(kv); free(q); free(qa);
+ return set_error(error, error_size, "out of memory in attention");
+ }
+
+ int result = coli_fp8_matvec_ref(qa, &wq_a, input);
+ coli_bf16_round_array(qa, (size_t)q_rank);
+ const void *q_norm = layer_data(weights, "attn.q_norm.weight", NULL);
+ if (!result && (!q_norm || decode_bf16(norm_weight, q_norm, (size_t)q_rank) ||
+ coli_v4_rmsnorm(qa, qa, norm_weight, q_rank,
+ config->rms_norm_eps))) result = -1;
+ if (!result) coli_bf16_round_array(qa, (size_t)q_rank);
+ if (!result && state && weights->plan.compression_ratio) {
+ result = prepare_compressed_state(state, weights, config,
+ error, error_size);
+ if (!result && (position + 1) % state->ratio == 0)
+ result = grow_compressed_state(state, error, error_size);
+ int produced = 0;
+ if (!result) result = coli_v4_compressor_step(
+ state->compressor,
+ state->compressed + (size_t)state->compressed_count * head_dim,
+ &produced, input, position, error, error_size);
+ if (!result && produced) state->compressed_count++;
+ if (!result && state->indexer) {
+ compressed_indices = malloc((size_t)config->index_topk *
+ sizeof(*compressed_indices));
+ if (!compressed_indices) result = -1;
+ else compressed_selected = coli_v4_indexer_step(
+ state->indexer, compressed_indices, config->index_topk,
+ qa, input, position, error, error_size);
+ if (compressed_selected < 0) result = -1;
+ }
+ }
+ if (!result) result = coli_fp8_matvec_ref(q, &wq_b, qa);
+ if (!result) coli_bf16_round_array(q, (size_t)heads * head_dim);
+ for (int head = 0; !result && head < heads; head++) {
+ float *values = q + (size_t)head * head_dim;
+ float mean_square = 0.0f;
+ for (int i = 0; i < head_dim; i++) mean_square += values[i] * values[i];
+ float scale = 1.0f / sqrtf(mean_square / head_dim + config->rms_norm_eps);
+ for (int i = 0; i < head_dim; i++) values[i] = coli_bf16_round(values[i] * scale);
+ }
+
+ if (!result) result = coli_fp8_matvec_ref(kv, &wkv, input);
+ if (!result) coli_bf16_round_array(kv, (size_t)head_dim);
+ const void *kv_norm = layer_data(weights, "attn.kv_norm.weight", NULL);
+ if (!result && (!kv_norm || decode_bf16(norm_weight, kv_norm, (size_t)head_dim) ||
+ coli_v4_rmsnorm(kv, kv, norm_weight, head_dim,
+ config->rms_norm_eps))) result = -1;
+ if (!result) coli_bf16_round_array(kv, (size_t)head_dim);
+
+ if (!result) {
+ float *all_cos = calloc((size_t)(position + 1) * rope_dim / 2, sizeof(*all_cos));
+ float *all_sin = calloc((size_t)(position + 1) * rope_dim / 2, sizeof(*all_sin));
+ int compressed = weights->plan.compression_ratio != 0;
+ if (!all_cos || !all_sin || coli_v4_rope_precompute(
+ all_cos, all_sin, rope_dim, position + 1,
+ compressed ? config->original_max_position_embeddings : 0,
+ compressed ? config->compress_rope_theta : config->rope_theta,
+ config->rope_factor,
+ config->rope_beta_fast, config->rope_beta_slow)) result = -1;
+ if (!result) {
+ memcpy(cosines, all_cos + (size_t)position * rope_dim / 2,
+ (size_t)rope_dim / 2 * sizeof(*cosines));
+ memcpy(sines, all_sin + (size_t)position * rope_dim / 2,
+ (size_t)rope_dim / 2 * sizeof(*sines));
+ }
+ free(all_sin); free(all_cos);
+ }
+ if (!result) {
+ for (int head = 0; head < heads; head++) {
+ float *rope = q + (size_t)head * head_dim + head_dim - rope_dim;
+ coli_v4_rope_apply(rope, 1, rope_dim, cosines, sines, 0);
+ coli_bf16_round_array(rope, (size_t)rope_dim);
+ }
+ float *kv_rope = kv + head_dim - rope_dim;
+ coli_v4_rope_apply(kv_rope, 1, rope_dim, cosines, sines, 0);
+ coli_bf16_round_array(kv_rope, (size_t)rope_dim);
+ size_t nope = (size_t)(head_dim - rope_dim);
+ float *qdq = malloc(nope * sizeof(*qdq));
+ uint8_t *scales = malloc((nope + 63) / 64);
+ if (!qdq || !scales || coli_fp8_activation_qdq_ref(qdq, scales, kv, nope, 64))
+ result = -1;
+ if (!result) {
+ memcpy(kv, qdq, nope * sizeof(*kv));
+ coli_bf16_round_array(kv, nope);
+ }
+ free(scales); free(qdq);
+ }
+
+ const float *sinks = layer_data(weights, "attn.attn_sink", NULL);
+ if (!result && state) {
+ int slot = position % state->window_size;
+ memcpy(state->kv + (size_t)slot * head_dim, kv,
+ (size_t)head_dim * sizeof(*kv));
+ if (!state->indexer) compressed_selected = state->compressed_count;
+ int topk = state->window_size + compressed_selected;
+ int kv_count = state->window_size + state->compressed_count;
+ int *indices = malloc((size_t)topk * sizeof(*indices));
+ float *all_kv = state->compressed_count
+ ? malloc((size_t)kv_count * head_dim * sizeof(*all_kv)) : NULL;
+ if (!indices || (state->compressed_count && !all_kv)) result = -1;
+ if (!result) {
+ if (position < state->window_size - 1) {
+ for (int i = 0; i < state->window_size; i++)
+ indices[i] = i <= position ? i : -1;
+ } else {
+ int oldest = (position + 1) % state->window_size;
+ for (int i = 0; i < state->window_size; i++)
+ indices[i] = (oldest + i) % state->window_size;
+ }
+ const float *kv_values = state->kv;
+ if (state->compressed_count) {
+ memcpy(all_kv, state->kv,
+ (size_t)state->window_size * head_dim * sizeof(*all_kv));
+ memcpy(all_kv + (size_t)state->window_size * head_dim,
+ state->compressed,
+ (size_t)state->compressed_count * head_dim * sizeof(*all_kv));
+ kv_values = all_kv;
+ }
+ for (int i = 0; i < compressed_selected; i++) {
+ int ordinal = state->indexer ? compressed_indices[i] : i;
+ indices[state->window_size + i] = state->window_size + ordinal;
+ }
+ result = coli_v4_sparse_attention_ref(
+ attended, q, kv_values, sinks, indices, heads, head_dim,
+ kv_count, topk,
+ 1.0f / sqrtf((float)head_dim));
+ }
+ free(all_kv);
+ free(indices);
+ } else for (int head = 0; !result && head < heads; head++) {
+ float *query = q + (size_t)head * head_dim;
+ float score = 0.0f;
+ for (int i = 0; i < head_dim; i++) score += query[i] * kv[i];
+ score *= 1.0f / sqrtf((float)head_dim);
+ float attention_weight = 1.0f / (1.0f + expf(sinks[head] - score));
+ float *head_output = attended + (size_t)head * head_dim;
+ for (int i = 0; i < head_dim; i++)
+ head_output[i] = coli_bf16_round(kv[i] * attention_weight);
+ }
+ for (int head = 0; !result && head < heads; head++) {
+ float *head_output = attended + (size_t)head * head_dim;
+ float *rope = head_output + head_dim - rope_dim;
+ coli_v4_rope_apply(rope, 1, rope_dim, cosines, sines, 1);
+ coli_bf16_round_array(rope, (size_t)rope_dim);
+ }
+
+ int heads_per_group = heads / groups;
+ int group_width = heads_per_group * head_dim;
+ int scale_columns = (hidden + 127) / 128;
+ int scale_rows_per_group = (o_rank + 127) / 128;
+ for (int group = 0; !result && group < groups; group++) {
+ ColiTensorView group_view = wo_a;
+ group_view.rows = o_rank;
+ group_view.columns = group_width;
+ group_view.data = (const uint8_t *)wo_a.data +
+ (size_t)group * o_rank * group_width;
+ group_view.scales = (const uint8_t *)wo_a.scales +
+ (size_t)group * scale_rows_per_group * scale_columns;
+ group_view.data_bytes = (size_t)o_rank * group_width;
+ group_view.scale_bytes = (size_t)scale_rows_per_group * scale_columns;
+ result = coli_fp8_matvec_ref(oa + (size_t)group * o_rank, &group_view,
+ attended + (size_t)group * group_width);
+ }
+ if (!result) coli_bf16_round_array(oa, (size_t)groups * o_rank);
+ if (!result) result = coli_fp8_matvec_ref(output, &wo_b, oa);
+ if (!result) coli_bf16_round_array(output, (size_t)hidden);
+
+ free(compressed_indices);
+ free(sines); free(cosines); free(norm_weight); free(oa);
+ free(attended); free(kv); free(q); free(qa);
+ if (result) return set_error(error, error_size, "attention computation failed");
+ return 0;
+}
+
+int coli_v4_attention_token_ref(float *output,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const float *input, int position,
+ char *error, size_t error_size) {
+ return attention_token_impl(output, NULL, weights, config, input, position,
+ error, error_size);
+}
+
+int coli_v4_attention_window_token_ref(
+ float *output, ColiDeepSeekV4WindowAttentionState *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, const float *input, int position,
+ char *error, size_t error_size) {
+ return attention_token_impl(output, state, weights, config, input, position,
+ error, error_size);
+}
+#endif /* COLI_V4_UNIT_ATTENTION */
+
+#ifdef COLI_V4_UNIT_ATTENTION_BATCH
+/* ######## deepseek_v4_attention_batch.c ######## */
+#define coli_v4_window_attention_create coli_v4_window_attention_batch_create_copy
+#define coli_v4_window_attention_reset coli_v4_window_attention_batch_reset_copy
+#define coli_v4_window_attention_destroy coli_v4_window_attention_batch_destroy_copy
+#define coli_v4_attention_token_ref coli_v4_attention_token_batch_serial_copy
+#define coli_v4_attention_window_token_ref coli_v4_attention_window_token_batch_serial_copy
+/* ---- begin include deepseek_v4_attention.c ---- */
+#include "deepseek_v4_internal.h"
+
+#include <math.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "native_quant.h"
+
+static int set_error(char *error, size_t size, const char *format, ...);
+
+struct ColiDeepSeekV4WindowAttentionState {
+ int window_size;
+ int head_dim;
+ int layer;
+ int ratio;
+ float *kv;
+ ColiDeepSeekV4CompressorState *compressor;
+ ColiDeepSeekV4Indexer *indexer;
+ float *compressed;
+ int compressed_count;
+ int compressed_capacity;
+};
+
+int coli_v4_window_attention_create(ColiDeepSeekV4WindowAttentionState **output,
+ const ColiDeepSeekV4Config *config) {
+ if (!output || !config || config->sliding_window < 1 || config->head_dim < 1)
+ return -1;
+ *output = calloc(1, sizeof(**output));
+ if (!*output) return -1;
+ (*output)->window_size = config->sliding_window;
+ (*output)->head_dim = config->head_dim;
+ (*output)->layer = -1;
+ (*output)->kv = calloc((size_t)config->sliding_window * config->head_dim,
+ sizeof(*(*output)->kv));
+ if (!(*output)->kv) {
+ free(*output);
+ *output = NULL;
+ return -1;
+ }
+ return 0;
+}
+
+void coli_v4_window_attention_reset(ColiDeepSeekV4WindowAttentionState *state) {
+ if (!state) return;
+ memset(state->kv, 0,
+ (size_t)state->window_size * state->head_dim * sizeof(*state->kv));
+ state->compressed_count = 0;
+ if (state->compressor) coli_v4_compressor_reset(state->compressor);
+ if (state->indexer) coli_v4_indexer_reset(state->indexer);
+}
+
+void coli_v4_window_attention_destroy(ColiDeepSeekV4WindowAttentionState *state) {
+ if (!state) return;
+ coli_v4_indexer_destroy(state->indexer);
+ coli_v4_compressor_destroy(state->compressor);
+ free(state->compressed);
+ free(state->kv);
+ free(state);
+}
+
+static int prepare_compressed_state(
+ ColiDeepSeekV4WindowAttentionState *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, char *error, size_t error_size) {
+ int ratio = weights->plan.compression_ratio;
+ if (!ratio) return 0;
+ if (state->layer < 0) {
+ state->layer = weights->plan.layer;
+ state->ratio = ratio;
+ state->compressed_capacity = 16;
+ state->compressed = calloc((size_t)state->compressed_capacity * state->head_dim,
+ sizeof(*state->compressed));
+ if (!state->compressed || coli_v4_compressor_create(
+ &state->compressor, weights, config, error, error_size)) return -1;
+ if (ratio == 4 && coli_v4_indexer_create(
+ &state->indexer, weights, config, config->max_position_embeddings,
+ error, error_size)) return -1;
+ } else if (state->layer != weights->plan.layer || state->ratio != ratio) {
+ return set_error(error, error_size, "attention state belongs to another layer");
+ }
+ if (coli_v4_compressor_bind_weights(state->compressor, weights,
+ error, error_size)) return -1;
+ if (state->indexer && coli_v4_indexer_bind_weights(
+ state->indexer, weights, error, error_size)) return -1;
+ return 0;
+}
+
+static int grow_compressed_state(ColiDeepSeekV4WindowAttentionState *state,
+ char *error, size_t error_size) {
+ if (state->compressed_count < state->compressed_capacity) return 0;
+ int capacity = state->compressed_capacity * 2;
+ float *grown = realloc(state->compressed,
+ (size_t)capacity * state->head_dim * sizeof(*grown));
+ if (!grown) return set_error(error, error_size, "cannot grow compressed KV cache");
+ state->compressed = grown;
+ state->compressed_capacity = capacity;
+ return 0;
+}
+
+static int set_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list arguments;
+ va_start(arguments, format);
+ vsnprintf(error, size, format, arguments);
+ va_end(arguments);
+ }
+ return -1;
+}
+
+static const void *layer_data(const ColiDeepSeekV4LayerWeights *weights,
+ const char *suffix,
+ const ColiDeepSeekV4TensorSpec **spec) {
+ char name[COLI_V4_MAX_TENSOR_NAME];
+ snprintf(name, sizeof(name), "layers.%d.%s", weights->plan.layer, suffix);
+ return coli_v4_layer_data(weights, name, spec);
+}
+
+static int fp8_view(ColiTensorView *view,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const char *prefix) {
+ char suffix[128];
+ const ColiDeepSeekV4TensorSpec *weight_spec = NULL, *scale_spec = NULL;
+ snprintf(suffix, sizeof(suffix), "%s.weight", prefix);
+ const void *data = layer_data(weights, suffix, &weight_spec);
+ snprintf(suffix, sizeof(suffix), "%s.scale", prefix);
+ const void *scales = layer_data(weights, suffix, &scale_spec);
+ if (!data || !scales || !weight_spec || !scale_spec ||
+ weight_spec->dtype != COLI_ST_F8_E4M3 ||
+ scale_spec->dtype != COLI_ST_F8_E8M0 || weight_spec->rank != 2)
+ return -1;
+ *view = (ColiTensorView){
+ COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_UE8M0, data, scales,
+ (size_t)(weight_spec->shape[0] * weight_spec->shape[1]),
+ (size_t)(scale_spec->shape[0] * scale_spec->shape[1]),
+ weight_spec->shape[0], weight_spec->shape[1], 128, 128
+ };
+ return 0;
+}
+
+static int decode_bf16(float *output, const void *data, size_t count) {
+ if (!output || !data) return -1;
+ const uint16_t *values = data;
+ for (size_t i = 0; i < count; i++) output[i] = coli_bf16_decode(values[i]);
+ return 0;
+}
+
+static int attention_token_impl(float *output,
+ ColiDeepSeekV4WindowAttentionState *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const float *input, int position,
+ char *error, size_t error_size) {
+ if (!output || !weights || !config || !input || position < 0 ||
+ (!state && weights->plan.compression_ratio != 0 && position != 0))
+ return set_error(error, error_size, "invalid uncompressed attention arguments");
+ int hidden = config->hidden_size;
+ int heads = config->num_attention_heads;
+ int head_dim = config->head_dim;
+ int rope_dim = config->qk_rope_head_dim;
+ int q_rank = config->q_lora_rank;
+ int groups = config->o_groups;
+ int o_rank = config->o_lora_rank;
+ if (hidden < 1 || heads < 1 || head_dim < 1 || rope_dim < 2 ||
+ rope_dim > head_dim || q_rank < 1 || groups < 1 || heads % groups)
+ return set_error(error, error_size, "unsupported attention dimensions");
+
+ ColiTensorView wq_a, wq_b, wkv, wo_a, wo_b;
+ if (fp8_view(&wq_a, weights, "attn.wq_a") ||
+ fp8_view(&wq_b, weights, "attn.wq_b") ||
+ fp8_view(&wkv, weights, "attn.wkv") ||
+ fp8_view(&wo_a, weights, "attn.wo_a") ||
+ fp8_view(&wo_b, weights, "attn.wo_b"))
+ return set_error(error, error_size, "missing native FP8 attention tensor");
+
+ float *qa = calloc((size_t)q_rank, sizeof(*qa));
+ float *q = calloc((size_t)heads * head_dim, sizeof(*q));
+ float *kv = calloc((size_t)head_dim, sizeof(*kv));
+ float *attended = calloc((size_t)heads * head_dim, sizeof(*attended));
+ float *oa = calloc((size_t)groups * o_rank, sizeof(*oa));
+ float *norm_weight = calloc((size_t)(q_rank > head_dim ? q_rank : head_dim),
+ sizeof(*norm_weight));
+ float *cosines = calloc((size_t)rope_dim / 2, sizeof(*cosines));
+ float *sines = calloc((size_t)rope_dim / 2, sizeof(*sines));
+ int *compressed_indices = NULL;
+ int compressed_selected = 0;
+ if (!qa || !q || !kv || !attended || !oa || !norm_weight || !cosines || !sines) {
+ free(sines); free(cosines); free(norm_weight); free(oa);
+ free(attended); free(kv); free(q); free(qa);
+ return set_error(error, error_size, "out of memory in attention");
+ }
+
+ int result = coli_fp8_matvec_ref(qa, &wq_a, input);
+ coli_bf16_round_array(qa, (size_t)q_rank);
+ const void *q_norm = layer_data(weights, "attn.q_norm.weight", NULL);
+ if (!result && (!q_norm || decode_bf16(norm_weight, q_norm, (size_t)q_rank) ||
+ coli_v4_rmsnorm(qa, qa, norm_weight, q_rank,
+ config->rms_norm_eps))) result = -1;
+ if (!result) coli_bf16_round_array(qa, (size_t)q_rank);
+ if (!result && state && weights->plan.compression_ratio) {
+ result = prepare_compressed_state(state, weights, config,
+ error, error_size);
+ if (!result && (position + 1) % state->ratio == 0)
+ result = grow_compressed_state(state, error, error_size);
+ int produced = 0;
+ if (!result) result = coli_v4_compressor_step(
+ state->compressor,
+ state->compressed + (size_t)state->compressed_count * head_dim,
+ &produced, input, position, error, error_size);
+ if (!result && produced) state->compressed_count++;
+ if (!result && state->indexer) {
+ compressed_indices = malloc((size_t)config->index_topk *
+ sizeof(*compressed_indices));
+ if (!compressed_indices) result = -1;
+ else compressed_selected = coli_v4_indexer_step(
+ state->indexer, compressed_indices, config->index_topk,
+ qa, input, position, error, error_size);
+ if (compressed_selected < 0) result = -1;
+ }
+ }
+ if (!result) result = coli_fp8_matvec_ref(q, &wq_b, qa);
+ if (!result) coli_bf16_round_array(q, (size_t)heads * head_dim);
+ for (int head = 0; !result && head < heads; head++) {
+ float *values = q + (size_t)head * head_dim;
+ float mean_square = 0.0f;
+ for (int i = 0; i < head_dim; i++) mean_square += values[i] * values[i];
+ float scale = 1.0f / sqrtf(mean_square / head_dim + config->rms_norm_eps);
+ for (int i = 0; i < head_dim; i++) values[i] = coli_bf16_round(values[i] * scale);
+ }
+
+ if (!result) result = coli_fp8_matvec_ref(kv, &wkv, input);
+ if (!result) coli_bf16_round_array(kv, (size_t)head_dim);
+ const void *kv_norm = layer_data(weights, "attn.kv_norm.weight", NULL);
+ if (!result && (!kv_norm || decode_bf16(norm_weight, kv_norm, (size_t)head_dim) ||
+ coli_v4_rmsnorm(kv, kv, norm_weight, head_dim,
+ config->rms_norm_eps))) result = -1;
+ if (!result) coli_bf16_round_array(kv, (size_t)head_dim);
+
+ if (!result) {
+ float *all_cos = calloc((size_t)(position + 1) * rope_dim / 2, sizeof(*all_cos));
+ float *all_sin = calloc((size_t)(position + 1) * rope_dim / 2, sizeof(*all_sin));
+ int compressed = weights->plan.compression_ratio != 0;
+ if (!all_cos || !all_sin || coli_v4_rope_precompute(
+ all_cos, all_sin, rope_dim, position + 1,
+ compressed ? config->original_max_position_embeddings : 0,
+ compressed ? config->compress_rope_theta : config->rope_theta,
+ config->rope_factor,
+ config->rope_beta_fast, config->rope_beta_slow)) result = -1;
+ if (!result) {
+ memcpy(cosines, all_cos + (size_t)position * rope_dim / 2,
+ (size_t)rope_dim / 2 * sizeof(*cosines));
+ memcpy(sines, all_sin + (size_t)position * rope_dim / 2,
+ (size_t)rope_dim / 2 * sizeof(*sines));
+ }
+ free(all_sin); free(all_cos);
+ }
+ if (!result) {
+ for (int head = 0; head < heads; head++) {
+ float *rope = q + (size_t)head * head_dim + head_dim - rope_dim;
+ coli_v4_rope_apply(rope, 1, rope_dim, cosines, sines, 0);
+ coli_bf16_round_array(rope, (size_t)rope_dim);
+ }
+ float *kv_rope = kv + head_dim - rope_dim;
+ coli_v4_rope_apply(kv_rope, 1, rope_dim, cosines, sines, 0);
+ coli_bf16_round_array(kv_rope, (size_t)rope_dim);
+ size_t nope = (size_t)(head_dim - rope_dim);
+ float *qdq = malloc(nope * sizeof(*qdq));
+ uint8_t *scales = malloc((nope + 63) / 64);
+ if (!qdq || !scales || coli_fp8_activation_qdq_ref(qdq, scales, kv, nope, 64))
+ result = -1;
+ if (!result) {
+ memcpy(kv, qdq, nope * sizeof(*kv));
+ coli_bf16_round_array(kv, nope);
+ }
+ free(scales); free(qdq);
+ }
+
+ const float *sinks = layer_data(weights, "attn.attn_sink", NULL);
+ if (!result && state) {
+ int slot = position % state->window_size;
+ memcpy(state->kv + (size_t)slot * head_dim, kv,
+ (size_t)head_dim * sizeof(*kv));
+ if (!state->indexer) compressed_selected = state->compressed_count;
+ int topk = state->window_size + compressed_selected;
+ int kv_count = state->window_size + state->compressed_count;
+ int *indices = malloc((size_t)topk * sizeof(*indices));
+ float *all_kv = state->compressed_count
+ ? malloc((size_t)kv_count * head_dim * sizeof(*all_kv)) : NULL;
+ if (!indices || (state->compressed_count && !all_kv)) result = -1;
+ if (!result) {
+ if (position < state->window_size - 1) {
+ for (int i = 0; i < state->window_size; i++)
+ indices[i] = i <= position ? i : -1;
+ } else {
+ int oldest = (position + 1) % state->window_size;
+ for (int i = 0; i < state->window_size; i++)
+ indices[i] = (oldest + i) % state->window_size;
+ }
+ const float *kv_values = state->kv;
+ if (state->compressed_count) {
+ memcpy(all_kv, state->kv,
+ (size_t)state->window_size * head_dim * sizeof(*all_kv));
+ memcpy(all_kv + (size_t)state->window_size * head_dim,
+ state->compressed,
+ (size_t)state->compressed_count * head_dim * sizeof(*all_kv));
+ kv_values = all_kv;
+ }
+ for (int i = 0; i < compressed_selected; i++) {
+ int ordinal = state->indexer ? compressed_indices[i] : i;
+ indices[state->window_size + i] = state->window_size + ordinal;
+ }
+ result = coli_v4_sparse_attention_ref(
+ attended, q, kv_values, sinks, indices, heads, head_dim,
+ kv_count, topk,
+ 1.0f / sqrtf((float)head_dim));
+ }
+ free(all_kv);
+ free(indices);
+ } else for (int head = 0; !result && head < heads; head++) {
+ float *query = q + (size_t)head * head_dim;
+ float score = 0.0f;
+ for (int i = 0; i < head_dim; i++) score += query[i] * kv[i];
+ score *= 1.0f / sqrtf((float)head_dim);
+ float attention_weight = 1.0f / (1.0f + expf(sinks[head] - score));
+ float *head_output = attended + (size_t)head * head_dim;
+ for (int i = 0; i < head_dim; i++)
+ head_output[i] = coli_bf16_round(kv[i] * attention_weight);
+ }
+ for (int head = 0; !result && head < heads; head++) {
+ float *head_output = attended + (size_t)head * head_dim;
+ float *rope = head_output + head_dim - rope_dim;
+ coli_v4_rope_apply(rope, 1, rope_dim, cosines, sines, 1);
+ coli_bf16_round_array(rope, (size_t)rope_dim);
+ }
+
+ int heads_per_group = heads / groups;
+ int group_width = heads_per_group * head_dim;
+ int scale_columns = (hidden + 127) / 128;
+ int scale_rows_per_group = (o_rank + 127) / 128;
+ for (int group = 0; !result && group < groups; group++) {
+ ColiTensorView group_view = wo_a;
+ group_view.rows = o_rank;
+ group_view.columns = group_width;
+ group_view.data = (const uint8_t *)wo_a.data +
+ (size_t)group * o_rank * group_width;
+ group_view.scales = (const uint8_t *)wo_a.scales +
+ (size_t)group * scale_rows_per_group * scale_columns;
+ group_view.data_bytes = (size_t)o_rank * group_width;
+ group_view.scale_bytes = (size_t)scale_rows_per_group * scale_columns;
+ result = coli_fp8_matvec_ref(oa + (size_t)group * o_rank, &group_view,
+ attended + (size_t)group * group_width);
+ }
+ if (!result) coli_bf16_round_array(oa, (size_t)groups * o_rank);
+ if (!result) result = coli_fp8_matvec_ref(output, &wo_b, oa);
+ if (!result) coli_bf16_round_array(output, (size_t)hidden);
+
+ free(compressed_indices);
+ free(sines); free(cosines); free(norm_weight); free(oa);
+ free(attended); free(kv); free(q); free(qa);
+ if (result) return set_error(error, error_size, "attention computation failed");
+ return 0;
+}
+
+int coli_v4_attention_token_ref(float *output,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const float *input, int position,
+ char *error, size_t error_size) {
+ return attention_token_impl(output, NULL, weights, config, input, position,
+ error, error_size);
+}
+
+int coli_v4_attention_window_token_ref(
+ float *output, ColiDeepSeekV4WindowAttentionState *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, const float *input, int position,
+ char *error, size_t error_size) {
+ return attention_token_impl(output, state, weights, config, input, position,
+ error, error_size);
+}
+/* ---- end include deepseek_v4_attention.c ---- */
+
+#undef coli_v4_window_attention_create
+#undef coli_v4_window_attention_reset
+#undef coli_v4_window_attention_destroy
+#undef coli_v4_attention_token_ref
+#undef coli_v4_attention_window_token_ref
+
+#include "deepseek_v4_internal.h"
+#include "native_quant_batch.h"
+
+int coli_v4_attention_window_batch_ref(
+ float *outputs, ColiDeepSeekV4WindowAttentionState *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, const float *inputs,
+ int start_position, int batch, char *error, size_t error_size) {
+ if (!outputs || !state || !weights || !config || !inputs ||
+ start_position < 0 || batch < 1 || batch > 64)
+ return set_error(error, error_size, "invalid batched attention arguments");
+ int hidden = config->hidden_size, heads = config->num_attention_heads;
+ int head_dim = config->head_dim, rope_dim = config->qk_rope_head_dim;
+ int q_rank = config->q_lora_rank, groups = config->o_groups;
+ int o_rank = config->o_lora_rank;
+ size_t q_width = (size_t)heads * head_dim;
+ size_t oa_width = (size_t)groups * o_rank;
+
+ ColiTensorView wq_a, wq_b, wkv, wo_a, wo_b;
+ if (fp8_view(&wq_a, weights, "attn.wq_a") ||
+ fp8_view(&wq_b, weights, "attn.wq_b") ||
+ fp8_view(&wkv, weights, "attn.wkv") ||
+ fp8_view(&wo_a, weights, "attn.wo_a") ||
+ fp8_view(&wo_b, weights, "attn.wo_b"))
+ return set_error(error, error_size, "missing batched attention tensor");
+
+ float *qa = calloc((size_t)batch * q_rank, sizeof(*qa));
+ float *q = calloc((size_t)batch * q_width, sizeof(*q));
+ float *kv = calloc((size_t)batch * head_dim, sizeof(*kv));
+ float *attended = calloc((size_t)batch * q_width, sizeof(*attended));
+ float *oa = calloc((size_t)batch * oa_width, sizeof(*oa));
+ float *norm = malloc((size_t)(q_rank > head_dim ? q_rank : head_dim) *
+ sizeof(*norm));
+ int *selected_counts = calloc((size_t)batch, sizeof(*selected_counts));
+ int *compressed_counts = calloc((size_t)batch, sizeof(*compressed_counts));
+ int *compressed_indices = malloc((size_t)batch * config->index_topk *
+ sizeof(*compressed_indices));
+ int end_position = start_position + batch;
+ size_t rope_pairs = (size_t)rope_dim / 2;
+ float *cosines = malloc((size_t)end_position * rope_pairs * sizeof(*cosines));
+ float *sines = malloc((size_t)end_position * rope_pairs * sizeof(*sines));
+ if (!qa || !q || !kv || !attended || !oa || !norm || !selected_counts ||
+ !compressed_counts || !compressed_indices || !cosines || !sines) {
+ free(sines); free(cosines); free(compressed_indices);
+ free(compressed_counts); free(selected_counts); free(norm); free(oa);
+ free(attended); free(kv); free(q); free(qa);
+ return set_error(error, error_size, "out of memory in batched attention");
+ }
+
+ int result = coli_fp8_matmul_batch_ref(qa, &wq_a, inputs, batch);
+ if (!result) coli_bf16_round_array(qa, (size_t)batch * q_rank);
+ const void *raw_q_norm = layer_data(weights, "attn.q_norm.weight", NULL);
+ if (!result && (!raw_q_norm || decode_bf16(norm, raw_q_norm, q_rank))) result = -1;
+ for (int item = 0; !result && item < batch; item++) {
+ float *item_qa = qa + (size_t)item * q_rank;
+ result = coli_v4_rmsnorm(item_qa, item_qa, norm, q_rank,
+ config->rms_norm_eps);
+ if (!result) coli_bf16_round_array(item_qa, (size_t)q_rank);
+ }
+
+ for (int item = 0; !result && item < batch; item++) {
+ int position = start_position + item;
+ if (weights->plan.compression_ratio) {
+ result = prepare_compressed_state(state, weights, config,
+ error, error_size);
+ if (!result && (position + 1) % state->ratio == 0)
+ result = grow_compressed_state(state, error, error_size);
+ int produced = 0;
+ if (!result) result = coli_v4_compressor_step(
+ state->compressor,
+ state->compressed + (size_t)state->compressed_count * head_dim,
+ &produced, inputs + (size_t)item * hidden, position,
+ error, error_size);
+ if (!result && produced) state->compressed_count++;
+ compressed_counts[item] = state->compressed_count;
+ if (!result && state->indexer) {
+ selected_counts[item] = coli_v4_indexer_step(
+ state->indexer,
+ compressed_indices + (size_t)item * config->index_topk,
+ config->index_topk, qa + (size_t)item * q_rank,
+ inputs + (size_t)item * hidden, position,
+ error, error_size);
+ if (selected_counts[item] < 0) result = -1;
+ } else if (!result) {
+ selected_counts[item] = state->compressed_count;
+ }
+ }
+ }
+
+ if (!result) result = coli_fp8_matmul_batch_ref(q, &wq_b, qa, batch);
+ if (!result) coli_bf16_round_array(q, (size_t)batch * q_width);
+ for (int item = 0; !result && item < batch; item++)
+ for (int head = 0; head < heads; head++) {
+ float *values = q + (size_t)item * q_width + (size_t)head * head_dim;
+ float square = 0.0f;
+ for (int i = 0; i < head_dim; i++) square += values[i] * values[i];
+ float scale = 1.0f / sqrtf(square / head_dim + config->rms_norm_eps);
+ for (int i = 0; i < head_dim; i++)
+ values[i] = coli_bf16_round(values[i] * scale);
+ }
+
+ if (!result) result = coli_fp8_matmul_batch_ref(kv, &wkv, inputs, batch);
+ if (!result) coli_bf16_round_array(kv, (size_t)batch * head_dim);
+ const void *raw_kv_norm = layer_data(weights, "attn.kv_norm.weight", NULL);
+ if (!result && (!raw_kv_norm || decode_bf16(norm, raw_kv_norm, head_dim))) result = -1;
+ for (int item = 0; !result && item < batch; item++) {
+ float *item_kv = kv + (size_t)item * head_dim;
+ result = coli_v4_rmsnorm(item_kv, item_kv, norm, head_dim,
+ config->rms_norm_eps);
+ if (!result) coli_bf16_round_array(item_kv, (size_t)head_dim);
+ }
+
+ int compressed = weights->plan.compression_ratio != 0;
+ if (!result) result = coli_v4_rope_precompute(
+ cosines, sines, rope_dim, end_position,
+ compressed ? config->original_max_position_embeddings : 0,
+ compressed ? config->compress_rope_theta : config->rope_theta,
+ config->rope_factor, config->rope_beta_fast, config->rope_beta_slow);
+ for (int item = 0; !result && item < batch; item++) {
+ int position = start_position + item;
+ const float *item_cos = cosines + (size_t)position * rope_pairs;
+ const float *item_sin = sines + (size_t)position * rope_pairs;
+ float *item_q = q + (size_t)item * q_width;
+ float *item_kv = kv + (size_t)item * head_dim;
+ for (int head = 0; head < heads; head++) {
+ float *rope = item_q + (size_t)head * head_dim + head_dim - rope_dim;
+ coli_v4_rope_apply(rope, 1, rope_dim, item_cos, item_sin, 0);
+ coli_bf16_round_array(rope, (size_t)rope_dim);
+ }
+ float *kv_rope = item_kv + head_dim - rope_dim;
+ coli_v4_rope_apply(kv_rope, 1, rope_dim, item_cos, item_sin, 0);
+ coli_bf16_round_array(kv_rope, (size_t)rope_dim);
+ size_t nope = (size_t)(head_dim - rope_dim);
+ float *qdq = malloc(nope * sizeof(*qdq));
+ uint8_t *scales = malloc((nope + 63) / 64);
+ if (!qdq || !scales || coli_fp8_activation_qdq_ref(
+ qdq, scales, item_kv, nope, 64)) result = -1;
+ if (!result) {
+ memcpy(item_kv, qdq, nope * sizeof(*item_kv));
+ coli_bf16_round_array(item_kv, nope);
+ }
+ free(scales); free(qdq);
+ }
+
+ const float *sinks = layer_data(weights, "attn.attn_sink", NULL);
+ for (int item = 0; !result && item < batch; item++) {
+ int position = start_position + item;
+ float *item_kv = kv + (size_t)item * head_dim;
+ float *item_q = q + (size_t)item * q_width;
+ float *item_attended = attended + (size_t)item * q_width;
+ int slot = position % state->window_size;
+ memcpy(state->kv + (size_t)slot * head_dim, item_kv,
+ (size_t)head_dim * sizeof(*item_kv));
+ int selected = selected_counts[item];
+ int compressed_count = compressed_counts[item];
+ int topk = state->window_size + selected;
+ int kv_count = state->window_size + compressed_count;
+ int *indices = malloc((size_t)topk * sizeof(*indices));
+ float *all_kv = compressed_count
+ ? malloc((size_t)kv_count * head_dim * sizeof(*all_kv)) : NULL;
+ if (!indices || (compressed_count && !all_kv)) result = -1;
+ if (!result) {
+ if (position < state->window_size - 1)
+ for (int i = 0; i < state->window_size; i++)
+ indices[i] = i <= position ? i : -1;
+ else {
+ int oldest = (position + 1) % state->window_size;
+ for (int i = 0; i < state->window_size; i++)
+ indices[i] = (oldest + i) % state->window_size;
+ }
+ const float *values = state->kv;
+ if (compressed_count) {
+ memcpy(all_kv, state->kv,
+ (size_t)state->window_size * head_dim * sizeof(*all_kv));
+ memcpy(all_kv + (size_t)state->window_size * head_dim,
+ state->compressed,
+ (size_t)compressed_count * head_dim * sizeof(*all_kv));
+ values = all_kv;
+ }
+ for (int i = 0; i < selected; i++) {
+ int ordinal = state->indexer
+ ? compressed_indices[(size_t)item * config->index_topk + i] : i;
+ indices[state->window_size + i] = state->window_size + ordinal;
+ }
+ result = coli_v4_sparse_attention_ref(
+ item_attended, item_q, values, sinks, indices, heads, head_dim,
+ kv_count, topk, 1.0f / sqrtf((float)head_dim));
+ }
+ free(all_kv); free(indices);
+ const float *item_cos = cosines + (size_t)position * rope_pairs;
+ const float *item_sin = sines + (size_t)position * rope_pairs;
+ for (int head = 0; !result && head < heads; head++) {
+ float *rope = item_attended + (size_t)head * head_dim +
+ head_dim - rope_dim;
+ coli_v4_rope_apply(rope, 1, rope_dim, item_cos, item_sin, 1);
+ coli_bf16_round_array(rope, (size_t)rope_dim);
+ }
+ }
+
+ int heads_per_group = heads / groups;
+ int group_width = heads_per_group * head_dim;
+ int scale_columns = (hidden + 127) / 128;
+ int scale_rows = (o_rank + 127) / 128;
+ float *group_inputs = malloc((size_t)batch * group_width * sizeof(*group_inputs));
+ float *group_outputs = malloc((size_t)batch * o_rank * sizeof(*group_outputs));
+ if (!group_inputs || !group_outputs) result = -1;
+ for (int group = 0; !result && group < groups; group++) {
+ for (int item = 0; item < batch; item++)
+ memcpy(group_inputs + (size_t)item * group_width,
+ attended + (size_t)item * q_width + (size_t)group * group_width,
+ (size_t)group_width * sizeof(*group_inputs));
+ ColiTensorView group_view = wo_a;
+ group_view.rows = o_rank;
+ group_view.columns = group_width;
+ group_view.data = (const uint8_t *)wo_a.data +
+ (size_t)group * o_rank * group_width;
+ group_view.scales = (const uint8_t *)wo_a.scales +
+ (size_t)group * scale_rows * scale_columns;
+ group_view.data_bytes = (size_t)o_rank * group_width;
+ group_view.scale_bytes = (size_t)scale_rows * scale_columns;
+ result = coli_fp8_matmul_batch_ref(
+ group_outputs, &group_view, group_inputs, batch);
+ for (int item = 0; !result && item < batch; item++)
+ memcpy(oa + (size_t)item * oa_width + (size_t)group * o_rank,
+ group_outputs + (size_t)item * o_rank,
+ (size_t)o_rank * sizeof(*oa));
+ }
+ if (!result) coli_bf16_round_array(oa, (size_t)batch * oa_width);
+ if (!result) result = coli_fp8_matmul_batch_ref(outputs, &wo_b, oa, batch);
+ if (!result) coli_bf16_round_array(outputs, (size_t)batch * hidden);
+
+ free(group_outputs); free(group_inputs); free(sines); free(cosines);
+ free(compressed_indices); free(compressed_counts); free(selected_counts);
+ free(norm); free(oa); free(attended); free(kv); free(q); free(qa);
+ return result ? set_error(error, error_size, "batched attention failed") : 0;
+}
+#endif /* COLI_V4_UNIT_ATTENTION_BATCH */
+
+#ifdef COLI_V4_UNIT_COMPRESSOR
+/* ######## deepseek_v4_compressor.c ######## */
+#include "deepseek_v4_internal.h"
+
+#include <math.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "deepseek_v4_internal.h"
+#include "native_quant.h"
+
+struct ColiDeepSeekV4CompressorState {
+ const ColiDeepSeekV4LayerWeights *weights;
+ const ColiDeepSeekV4Config *config;
+ int ratio;
+ int layer;
+ int hidden;
+ int head_dim;
+ int projection_dim;
+ int state_rows;
+ int rope_dim;
+ int rotate_fp4;
+ char prefix[96];
+ float *kv_state;
+ float *score_state;
+};
+
+static int set_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list arguments;
+ va_start(arguments, format);
+ vsnprintf(error, size, format, arguments);
+ va_end(arguments);
+ }
+ return -1;
+}
+
+static const void *layer_value(const ColiDeepSeekV4LayerWeights *weights,
+ const char *suffix) {
+ char name[COLI_V4_MAX_TENSOR_NAME];
+ snprintf(name, sizeof(name), "layers.%d.%s", weights->plan.layer, suffix);
+ return coli_v4_layer_data(weights, name, NULL);
+}
+
+int coli_v4_compressor_create(ColiDeepSeekV4CompressorState **output,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ char *error, size_t error_size) {
+ ColiDeepSeekV4CompressorOptions options = {
+ "attn.compressor", config ? config->head_dim : 0, 0
+ };
+ return coli_v4_compressor_create_with_options(
+ output, weights, config, &options, error, error_size);
+}
+
+int coli_v4_compressor_create_with_options(
+ ColiDeepSeekV4CompressorState **output,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const ColiDeepSeekV4CompressorOptions *options,
+ char *error, size_t error_size) {
+ if (!output || !weights || !config || !options || !options->prefix ||
+ !options->prefix[0] || options->head_dimension <= 0 ||
+ weights->plan.compression_ratio < 1)
+ return set_error(error, error_size, "unsupported compressor ratio");
+ if (strlen(options->prefix) >= sizeof(((ColiDeepSeekV4CompressorState *)0)->prefix))
+ return set_error(error, error_size, "compressor prefix is too long");
+ *output = NULL;
+ ColiDeepSeekV4CompressorState *state = calloc(1, sizeof(*state));
+ if (!state) return set_error(error, error_size, "out of memory creating compressor");
+ state->weights = weights;
+ state->config = config;
+ state->ratio = weights->plan.compression_ratio;
+ state->layer = weights->plan.layer;
+ state->hidden = config->hidden_size;
+ state->head_dim = options->head_dimension;
+ state->rotate_fp4 = options->rotate_fp4 != 0;
+ memcpy(state->prefix, options->prefix, strlen(options->prefix) + 1);
+ int overlap = state->ratio == 4;
+ state->projection_dim = (1 + overlap) * state->head_dim;
+ state->state_rows = (1 + overlap) * state->ratio;
+ state->rope_dim = config->qk_rope_head_dim;
+ size_t count = (size_t)state->state_rows * state->projection_dim;
+ state->kv_state = calloc(count, sizeof(*state->kv_state));
+ state->score_state = malloc(count * sizeof(*state->score_state));
+ if (!state->kv_state || !state->score_state) {
+ coli_v4_compressor_destroy(state);
+ return set_error(error, error_size, "out of memory allocating compressor state");
+ }
+ for (size_t i = 0; i < count; i++) state->score_state[i] = -INFINITY;
+ *output = state;
+ return 0;
+}
+
+int coli_v4_compressor_bind_weights(ColiDeepSeekV4CompressorState *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ char *error, size_t error_size) {
+ if (!state || !weights ||
+ weights->plan.layer != state->layer ||
+ weights->plan.compression_ratio != state->ratio)
+ return set_error(error, error_size, "incompatible compressor weights");
+ state->weights = weights;
+ return 0;
+}
+
+void coli_v4_compressor_reset(ColiDeepSeekV4CompressorState *state) {
+ if (!state) return;
+ size_t count = (size_t)state->state_rows * state->projection_dim;
+ memset(state->kv_state, 0, count * sizeof(*state->kv_state));
+ for (size_t i = 0; i < count; i++) state->score_state[i] = -INFINITY;
+}
+
+void coli_v4_compressor_destroy(ColiDeepSeekV4CompressorState *state) {
+ if (!state) return;
+ free(state->score_state);
+ free(state->kv_state);
+ free(state);
+}
+
+int coli_v4_compressor_step(ColiDeepSeekV4CompressorState *state,
+ float *output, int *produced,
+ const float *input, int position,
+ char *error, size_t error_size) {
+ if (!state || !produced || !input || position < 0)
+ return set_error(error, error_size, "invalid compressor step arguments");
+ *produced = 0;
+ int slot = position % state->ratio;
+ int hidden = state->hidden, dimension = state->head_dim;
+ int projection = state->projection_dim;
+ int state_row = state->ratio == 4 ? state->ratio + slot : slot;
+ char suffix[128];
+ snprintf(suffix, sizeof(suffix), "%s.wkv.weight", state->prefix);
+ const uint16_t *wkv = layer_value(state->weights, suffix);
+ snprintf(suffix, sizeof(suffix), "%s.wgate.weight", state->prefix);
+ const uint16_t *wgate = layer_value(state->weights, suffix);
+ snprintf(suffix, sizeof(suffix), "%s.ape", state->prefix);
+ const float *ape = layer_value(state->weights, suffix);
+ if (!wkv || !wgate || !ape)
+ return set_error(error, error_size, "missing compressor tensor for %s", state->prefix);
+ float *kv_row = state->kv_state + (size_t)state_row * projection;
+ float *score_row = state->score_state + (size_t)state_row * projection;
+ #pragma omp parallel for
+ for (int row = 0; row < projection; row++) {
+ float kv_sum = 0.0f, gate_sum = 0.0f;
+ const uint16_t *kv_weight = wkv + (size_t)row * hidden;
+ const uint16_t *gate_weight = wgate + (size_t)row * hidden;
+ for (int column = 0; column < hidden; column++) {
+ float value = input[column];
+ kv_sum += coli_bf16_decode(kv_weight[column]) * value;
+ gate_sum += coli_bf16_decode(gate_weight[column]) * value;
+ }
+ kv_row[row] = kv_sum;
+ score_row[row] = gate_sum + ape[(size_t)slot * projection + row];
+ }
+ if ((position + 1) % state->ratio != 0) return 0;
+ if (!output) return set_error(error, error_size, "compressor output is required");
+
+ #pragma omp parallel for
+ for (int column = 0; column < dimension; column++) {
+ float maximum = -INFINITY;
+ int pool_rows = state->ratio == 4 ? 2 * state->ratio : state->ratio;
+ for (int row = 0; row < pool_rows; row++) {
+ int source_row = row;
+ int source_column = column;
+ if (state->ratio == 4 && row >= state->ratio)
+ source_column += dimension;
+ float score = state->score_state[
+ (size_t)source_row * projection + source_column];
+ if (score > maximum) maximum = score;
+ }
+ float total = 0.0f, weighted = 0.0f;
+ for (int row = 0; row < pool_rows; row++) {
+ int source_column = column;
+ if (state->ratio == 4 && row >= state->ratio)
+ source_column += dimension;
+ float weight = expf(state->score_state[
+ (size_t)row * projection + source_column] - maximum);
+ total += weight;
+ weighted += state->kv_state[
+ (size_t)row * projection + source_column] * weight;
+ }
+ output[column] = weighted / total;
+ }
+ if (state->ratio == 4) {
+ memcpy(state->kv_state,
+ state->kv_state + (size_t)state->ratio * projection,
+ (size_t)state->ratio * projection * sizeof(*state->kv_state));
+ memcpy(state->score_state,
+ state->score_state + (size_t)state->ratio * projection,
+ (size_t)state->ratio * projection * sizeof(*state->score_state));
+ }
+ coli_bf16_round_array(output, (size_t)dimension);
+ snprintf(suffix, sizeof(suffix), "%s.norm.weight", state->prefix);
+ const uint16_t *raw_norm = layer_value(state->weights, suffix);
+ float *norm = malloc((size_t)dimension * sizeof(*norm));
+ if (!raw_norm || !norm) {
+ free(norm);
+ return set_error(error, error_size, "missing compressor norm");
+ }
+ for (int i = 0; i < dimension; i++) norm[i] = coli_bf16_decode(raw_norm[i]);
+ coli_v4_rmsnorm(output, output, norm, dimension, state->config->rms_norm_eps);
+ coli_bf16_round_array(output, (size_t)dimension);
+ free(norm);
+
+ int rope_position = position + 1 - state->ratio;
+ int pairs = state->rope_dim / 2;
+ size_t table_count = (size_t)(rope_position + 1) * pairs;
+ float *cosines = malloc(table_count * sizeof(*cosines));
+ float *sines = malloc(table_count * sizeof(*sines));
+ if (!cosines || !sines || coli_v4_rope_precompute(
+ cosines, sines, state->rope_dim, rope_position + 1,
+ state->config->original_max_position_embeddings,
+ state->config->compress_rope_theta, state->config->rope_factor,
+ state->config->rope_beta_fast, state->config->rope_beta_slow)) {
+ free(sines); free(cosines);
+ return set_error(error, error_size, "cannot create compressor RoPE table");
+ }
+ float *rope = output + dimension - state->rope_dim;
+ coli_v4_rope_apply(rope, 1, state->rope_dim,
+ cosines + (size_t)rope_position * pairs,
+ sines + (size_t)rope_position * pairs, 0);
+ coli_bf16_round_array(rope, (size_t)state->rope_dim);
+ free(sines); free(cosines);
+
+ size_t quantized = state->rotate_fp4
+ ? (size_t)dimension : (size_t)(dimension - state->rope_dim);
+ size_t block = state->rotate_fp4 ? 32u : 64u;
+ float *qdq = malloc(quantized * sizeof(*qdq));
+ uint8_t *scales = malloc((quantized + block - 1) / block);
+ if (!qdq || !scales) {
+ free(scales); free(qdq);
+ return set_error(error, error_size, "compressor activation quantization failed");
+ }
+ int quant_error = 0;
+ if (state->rotate_fp4)
+ quant_error = coli_hadamard_bf16_ref(output, (size_t)dimension) ||
+ coli_fp4_activation_qdq_ref(qdq, scales, output,
+ quantized, block);
+ else
+ quant_error = coli_fp8_activation_qdq_ref(qdq, scales, output,
+ quantized, block);
+ if (quant_error) {
+ free(scales); free(qdq);
+ return set_error(error, error_size, "compressor activation quantization failed");
+ }
+ memcpy(output, qdq, quantized * sizeof(*output));
+ coli_bf16_round_array(output, quantized);
+ free(scales); free(qdq);
+ *produced = 1;
+ return 0;
+}
+#endif /* COLI_V4_UNIT_COMPRESSOR */
+
+#ifdef COLI_V4_UNIT_INDEXER
+/* ######## deepseek_v4_indexer.c ######## */
+#include "deepseek_v4_internal.h"
+
+#include <math.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "native_quant.h"
+
+struct ColiDeepSeekV4Indexer {
+ const ColiDeepSeekV4LayerWeights *weights;
+ const ColiDeepSeekV4Config *config;
+ ColiDeepSeekV4CompressorState *compressor;
+ int layer;
+ int capacity;
+ int count;
+ float *compressed;
+};
+
+typedef struct { float score; int index; } IndexScore;
+
+static int set_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list arguments;
+ va_start(arguments, format);
+ vsnprintf(error, size, format, arguments);
+ va_end(arguments);
+ }
+ return -1;
+}
+
+static const void *value(const ColiDeepSeekV4LayerWeights *weights,
+ const char *suffix,
+ const ColiDeepSeekV4TensorSpec **spec) {
+ char name[COLI_V4_MAX_TENSOR_NAME];
+ snprintf(name, sizeof(name), "layers.%d.%s", weights->plan.layer, suffix);
+ return coli_v4_layer_data(weights, name, spec);
+}
+
+static int fp8_view(ColiTensorView *view,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const char *prefix) {
+ char suffix[128];
+ const ColiDeepSeekV4TensorSpec *ws = NULL, *ss = NULL;
+ snprintf(suffix, sizeof(suffix), "%s.weight", prefix);
+ const void *data = value(weights, suffix, &ws);
+ snprintf(suffix, sizeof(suffix), "%s.scale", prefix);
+ const void *scales = value(weights, suffix, &ss);
+ if (!data || !scales || !ws || !ss || ws->rank != 2 ||
+ ws->dtype != COLI_ST_F8_E4M3 || ss->dtype != COLI_ST_F8_E8M0)
+ return -1;
+ *view = (ColiTensorView){
+ COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_UE8M0, data, scales,
+ (size_t)(ws->shape[0] * ws->shape[1]),
+ (size_t)(ss->shape[0] * ss->shape[1]),
+ ws->shape[0], ws->shape[1], 128, 128
+ };
+ return 0;
+}
+
+static int descending_score(const void *left, const void *right) {
+ const IndexScore *a = left, *b = right;
+ if (a->score < b->score) return 1;
+ if (a->score > b->score) return -1;
+ return a->index - b->index;
+}
+
+int coli_v4_indexer_create(ColiDeepSeekV4Indexer **output,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ int max_context, char *error, size_t error_size) {
+ if (!output || !weights || !config || !weights->plan.has_indexer ||
+ max_context < 4 || config->index_head_dim < 1 ||
+ config->index_n_heads < 1)
+ return set_error(error, error_size, "invalid indexer options");
+ *output = NULL;
+ ColiDeepSeekV4Indexer *state = calloc(1, sizeof(*state));
+ if (!state) return set_error(error, error_size, "out of memory creating indexer");
+ state->weights = weights;
+ state->config = config;
+ state->layer = weights->plan.layer;
+ state->capacity = (max_context + 3) / 4;
+ if (state->capacity > 128) state->capacity = 128;
+ state->compressed = calloc((size_t)state->capacity * config->index_head_dim,
+ sizeof(*state->compressed));
+ ColiDeepSeekV4CompressorOptions options = {
+ "attn.indexer.compressor", config->index_head_dim, 1
+ };
+ if (!state->compressed || coli_v4_compressor_create_with_options(
+ &state->compressor, weights, config, &options, error, error_size)) {
+ coli_v4_indexer_destroy(state);
+ return set_error(error, error_size, "cannot create indexer compressor");
+ }
+ *output = state;
+ return 0;
+}
+
+int coli_v4_indexer_bind_weights(ColiDeepSeekV4Indexer *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ char *error, size_t error_size) {
+ if (!state || !weights || weights->plan.layer != state->layer ||
+ !weights->plan.has_indexer)
+ return set_error(error, error_size, "incompatible indexer weights");
+ state->weights = weights;
+ return coli_v4_compressor_bind_weights(state->compressor, weights,
+ error, error_size);
+}
+
+void coli_v4_indexer_reset(ColiDeepSeekV4Indexer *state) {
+ if (!state) return;
+ state->count = 0;
+ memset(state->compressed, 0,
+ (size_t)state->capacity * state->config->index_head_dim * sizeof(float));
+ coli_v4_compressor_reset(state->compressor);
+}
+
+void coli_v4_indexer_destroy(ColiDeepSeekV4Indexer *state) {
+ if (!state) return;
+ coli_v4_compressor_destroy(state->compressor);
+ free(state->compressed);
+ free(state);
+}
+
+static int apply_position_rope(float *queries,
+ const ColiDeepSeekV4Config *config,
+ int position) {
+ int heads = config->index_n_heads, dimension = config->index_head_dim;
+ int rope_dim = config->qk_rope_head_dim, pairs = rope_dim / 2;
+ size_t count = (size_t)(position + 1) * pairs;
+ float *cosines = malloc(count * sizeof(*cosines));
+ float *sines = malloc(count * sizeof(*sines));
+ if (!cosines || !sines || coli_v4_rope_precompute(
+ cosines, sines, rope_dim, position + 1,
+ config->original_max_position_embeddings,
+ config->compress_rope_theta, config->rope_factor,
+ config->rope_beta_fast, config->rope_beta_slow)) {
+ free(sines); free(cosines); return -1;
+ }
+ for (int head = 0; head < heads; head++) {
+ float *query = queries + (size_t)head * dimension;
+ coli_v4_rope_apply(query + dimension - rope_dim, 1, rope_dim,
+ cosines + (size_t)position * pairs,
+ sines + (size_t)position * pairs, 0);
+ coli_bf16_round_array(query + dimension - rope_dim, (size_t)rope_dim);
+ }
+ free(sines); free(cosines);
+ return 0;
+}
+
+int coli_v4_indexer_step(ColiDeepSeekV4Indexer *state, int *indices,
+ int index_capacity, const float *query_rank,
+ const float *input, int position,
+ char *error, size_t error_size) {
+ if (!state || !indices || index_capacity < 1 || !query_rank || !input ||
+ position < 0)
+ return set_error(error, error_size, "invalid indexer step arguments");
+ int dimension = state->config->index_head_dim;
+ int heads = state->config->index_n_heads;
+ int produced = 0;
+ if ((position + 1) % 4 == 0 && state->count >= state->capacity) {
+ int next_capacity = state->capacity * 2;
+ float *grown = realloc(state->compressed,
+ (size_t)next_capacity * dimension * sizeof(*grown));
+ if (!grown) return set_error(error, error_size, "cannot grow indexer cache");
+ memset(grown + (size_t)state->capacity * dimension, 0,
+ (size_t)(next_capacity - state->capacity) * dimension * sizeof(*grown));
+ state->compressed = grown;
+ state->capacity = next_capacity;
+ }
+ float *next = state->count < state->capacity
+ ? state->compressed + (size_t)state->count * dimension : NULL;
+ if (coli_v4_compressor_step(state->compressor, next, &produced, input,
+ position, error, error_size)) return -1;
+ if (produced) {
+ if (state->count >= state->capacity)
+ return set_error(error, error_size, "indexer cache capacity exceeded");
+ state->count++;
+ }
+ if (!state->count) return 0;
+
+ ColiTensorView wq;
+ if (fp8_view(&wq, state->weights, "attn.indexer.wq_b"))
+ return set_error(error, error_size, "missing indexer query weight");
+ float *queries = malloc((size_t)heads * dimension * sizeof(*queries));
+ float *head_weights = malloc((size_t)heads * sizeof(*head_weights));
+ IndexScore *scores = malloc((size_t)state->count * sizeof(*scores));
+ uint8_t *scales = malloc((size_t)dimension / 32);
+ float *qdq = malloc((size_t)dimension * sizeof(*qdq));
+ const uint16_t *raw_weights = value(
+ state->weights, "attn.indexer.weights_proj.weight", NULL);
+ if (!queries || !head_weights || !scores || !scales || !qdq || !raw_weights) {
+ free(qdq); free(scales); free(scores); free(head_weights); free(queries);
+ return set_error(error, error_size, "out of memory scoring indexer");
+ }
+ int result = coli_fp8_matvec_ref(queries, &wq, query_rank);
+ if (!result) coli_bf16_round_array(queries, (size_t)heads * dimension);
+ if (!result) result = apply_position_rope(queries, state->config, position);
+ for (int head = 0; !result && head < heads; head++) {
+ float *query = queries + (size_t)head * dimension;
+ result = coli_hadamard_bf16_ref(query, (size_t)dimension);
+ if (!result) result = coli_fp4_activation_qdq_ref(
+ qdq, scales, query, (size_t)dimension, 32);
+ if (!result) {
+ memcpy(query, qdq, (size_t)dimension * sizeof(*query));
+ coli_bf16_round_array(query, (size_t)dimension);
+ }
+ }
+ float weight_scale = 1.0f / sqrtf((float)(dimension * heads));
+ for (int head = 0; !result && head < heads; head++) {
+ float sum = 0.0f;
+ const uint16_t *row = raw_weights + (size_t)head * state->config->hidden_size;
+ for (int column = 0; column < state->config->hidden_size; column++)
+ sum += coli_bf16_decode(row[column]) * input[column];
+ head_weights[head] = sum * weight_scale;
+ }
+ for (int candidate = 0; !result && candidate < state->count; candidate++) {
+ const float *key = state->compressed + (size_t)candidate * dimension;
+ float score = 0.0f;
+ for (int head = 0; head < heads; head++) {
+ const float *query = queries + (size_t)head * dimension;
+ float dot = 0.0f;
+ for (int i = 0; i < dimension; i++) dot += query[i] * key[i];
+ score += fmaxf(dot, 0.0f) * head_weights[head];
+ }
+ scores[candidate] = (IndexScore){score, candidate};
+ }
+ if (!result) qsort(scores, (size_t)state->count, sizeof(*scores), descending_score);
+ int selected = state->count;
+ if (selected > state->config->index_topk) selected = state->config->index_topk;
+ if (selected > index_capacity) selected = index_capacity;
+ for (int i = 0; !result && i < selected; i++) indices[i] = scores[i].index;
+ free(qdq); free(scales); free(scores); free(head_weights); free(queries);
+ return result ? set_error(error, error_size, "indexer scoring failed") : selected;
+}
+
+const float *coli_v4_indexer_compressed_values(
+ const ColiDeepSeekV4Indexer *state) {
+ return state ? state->compressed : NULL;
+}
+
+int coli_v4_indexer_compressed_count(const ColiDeepSeekV4Indexer *state) {
+ return state ? state->count : 0;
+}
+#endif /* COLI_V4_UNIT_INDEXER */
+
+#ifdef COLI_V4_UNIT_SPARSE_ATTENTION
+/* ######## deepseek_v4_sparse_attention.c ######## */
+#include "deepseek_v4_internal.h"
+
+#include <math.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "native_quant.h"
+
+int coli_v4_sparse_attention_ref(float *output, const float *queries,
+ const float *kv, const float *sinks,
+ const int *indices, int heads,
+ int head_dimension, int kv_count, int topk,
+ float softmax_scale) {
+ if (!output || !queries || !kv || !sinks || !indices || heads < 1 ||
+ head_dimension < 1 || kv_count < 1 || topk < 1 || !(softmax_scale > 0.0f))
+ return -1;
+ float *scores = malloc((size_t)topk * sizeof(*scores));
+ if (!scores) return -1;
+ for (int head = 0; head < heads; head++) {
+ const float *query = queries + (size_t)head * head_dimension;
+ float maximum = -INFINITY;
+ for (int rank = 0; rank < topk; rank++) {
+ int index = indices[rank];
+ if (index < 0) {
+ scores[rank] = -INFINITY;
+ continue;
+ }
+ if (index >= kv_count) {
+ free(scores);
+ return -1;
+ }
+ const float *key = kv + (size_t)index * head_dimension;
+ float score = 0.0f;
+ for (int column = 0; column < head_dimension; column++)
+ score += query[column] * key[column];
+ score *= softmax_scale;
+ scores[rank] = score;
+ if (score > maximum) maximum = score;
+ }
+ if (!isfinite(maximum)) {
+ free(scores);
+ return -1;
+ }
+ float denominator = expf(sinks[head] - maximum);
+ float *head_output = output + (size_t)head * head_dimension;
+ memset(head_output, 0, (size_t)head_dimension * sizeof(*head_output));
+ for (int rank = 0; rank < topk; rank++) {
+ if (indices[rank] < 0) continue;
+ float probability = expf(scores[rank] - maximum);
+ denominator += probability;
+ /* TileLang casts the exp fragment to BF16 before value GEMM. */
+ probability = coli_bf16_round(probability);
+ const float *value = kv + (size_t)indices[rank] * head_dimension;
+ for (int column = 0; column < head_dimension; column++)
+ head_output[column] += probability * value[column];
+ }
+ for (int column = 0; column < head_dimension; column++)
+ head_output[column] = coli_bf16_round(head_output[column] / denominator);
+ }
+ free(scores);
+ return 0;
+}
+#endif /* COLI_V4_UNIT_SPARSE_ATTENTION */
+
+#ifdef COLI_V4_UNIT_BLOCK_HYBRID
+/* ######## deepseek_v4_block_hybrid.c ######## */
+/* Accepted decode pipeline plus batched causal attention for prompt prefill. */
+/* ---- begin include deepseek_v4_block_pipeline.c ---- */
+/* Windows-native routed-expert I/O pipeline. The original block implementation
+ * is retained under serial symbols. Public entry points normally keep three
+ * persistent lookup workers so reads N+1..N+3 overlap ordered expert compute.
+ * COLI_V4_DISABLE_DUAL_EXPERT_LOADER restores the one-worker pipeline. */
+#if !defined(COLI_V4_DISABLE_DUAL_EXPERT_LOADER) && \
+ !defined(COLI_V4_DISABLE_PERSISTENT_EXPERT_LOADER) && \
+ !defined(COLI_V4_EXPERIMENTAL_SYNC_EXPERT_LOOKUP) && \
+ !defined(COLI_V4_EXPERIMENTAL_DUAL_EXPERT_LOADER)
+#define COLI_V4_EXPERIMENTAL_DUAL_EXPERT_LOADER
+#endif
+#define coli_v4_block_token_ref coli_v4_block_token_serial_ref
+#define coli_v4_block_window_token_ref coli_v4_block_window_token_serial_ref
+/* ---- begin include deepseek_v4_block.c ---- */
+#include "deepseek_v4_internal.h"
+
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "native_quant.h"
+
+static int set_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list arguments;
+ va_start(arguments, format);
+ vsnprintf(error, size, format, arguments);
+ va_end(arguments);
+ }
+ return -1;
+}
+
+static const void *value(const ColiDeepSeekV4LayerWeights *weights,
+ const char *suffix,
+ const ColiDeepSeekV4TensorSpec **spec) {
+ char name[COLI_V4_MAX_TENSOR_NAME];
+ snprintf(name, sizeof(name), "layers.%d.%s", weights->plan.layer, suffix);
+ return coli_v4_layer_data(weights, name, spec);
+}
+
+static int fp8_view(ColiTensorView *view,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const char *prefix) {
+ char name[128];
+ const ColiDeepSeekV4TensorSpec *ws = NULL, *ss = NULL;
+ snprintf(name, sizeof(name), "%s.weight", prefix);
+ const void *data = value(weights, name, &ws);
+ snprintf(name, sizeof(name), "%s.scale", prefix);
+ const void *scales = value(weights, name, &ss);
+ if (!data || !scales || !ws || !ss || ws->rank != 2) return -1;
+ *view = (ColiTensorView){
+ COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_UE8M0, data, scales,
+ (size_t)(ws->shape[0] * ws->shape[1]),
+ (size_t)(ss->shape[0] * ss->shape[1]),
+ ws->shape[0], ws->shape[1], 128, 128
+ };
+ return 0;
+}
+
+static void decode_bf16(float *output, const uint16_t *input, size_t count) {
+ for (size_t i = 0; i < count; i++) output[i] = coli_bf16_decode(input[i]);
+}
+
+static int normalized_hc_pre(float *reduced, float *post, float *comb,
+ float *normalized, const float *input_hc,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const char *branch, const char *norm_name) {
+ char name[64];
+ snprintf(name, sizeof(name), "hc_%s_fn", branch);
+ const float *function = value(weights, name, NULL);
+ snprintf(name, sizeof(name), "hc_%s_scale", branch);
+ const float *scale = value(weights, name, NULL);
+ snprintf(name, sizeof(name), "hc_%s_base", branch);
+ const float *base = value(weights, name, NULL);
+ const uint16_t *raw_norm = value(weights, norm_name, NULL);
+ int d = config->hidden_size;
+ float *norm = malloc((size_t)d * sizeof(*norm));
+ if (!function || !scale || !base || !raw_norm || !norm) {
+ free(norm);
+ return -1;
+ }
+ decode_bf16(norm, raw_norm, (size_t)d);
+ int result = coli_v4_hc_pre(reduced, post, comb, input_hc, function,
+ scale, base, config->hc_mult, d,
+ config->hc_sinkhorn_iters,
+ config->rms_norm_eps, config->hc_eps);
+ if (!result) {
+ coli_bf16_round_array(reduced, (size_t)d);
+ result = coli_v4_rmsnorm(normalized, reduced, norm, d,
+ config->rms_norm_eps);
+ coli_bf16_round_array(normalized, (size_t)d);
+ }
+ free(norm);
+ return result;
+}
+
+static int moe_token(float *output,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ ColiExpertStore *store, const float *input, int token) {
+ int d = config->hidden_size;
+ int n = config->n_routed_experts;
+ int topk = config->num_experts_per_tok;
+ size_t gate_count = (size_t)n * d;
+ float *gate = malloc(gate_count * sizeof(*gate));
+ float *route_weights = malloc((size_t)topk * sizeof(*route_weights));
+ int *indices = malloc((size_t)topk * sizeof(*indices));
+ float *expert_output = malloc((size_t)d * sizeof(*expert_output));
+ float *shared_output = malloc((size_t)d * sizeof(*shared_output));
+ if (!gate || !route_weights || !indices || !expert_output || !shared_output) {
+ free(shared_output); free(expert_output); free(indices);
+ free(route_weights); free(gate);
+ return -1;
+ }
+ decode_bf16(gate, value(weights, "ffn.gate.weight", NULL), gate_count);
+ const int64_t *table = value(weights, "ffn.gate.tid2eid", NULL);
+ const float *bias = value(weights, "ffn.gate.bias", NULL);
+ int result = token < 0 || token >= config->vocab_size;
+ if (!result && weights->plan.uses_hash_router) {
+ if (!table) result = -1;
+ }
+ if (!result && weights->plan.uses_hash_router) {
+ for (int i = 0; i < topk; i++)
+ indices[i] = (int)table[(size_t)token * topk + i];
+ }
+ if (!result) result = coli_v4_route(
+ route_weights, indices, input, gate, bias,
+ weights->plan.uses_hash_router ? indices : NULL,
+ n, d, topk, config->routed_scaling_factor);
+
+ ColiTensorView w1, w2, w3;
+ if (!result && (fp8_view(&w1, weights, "ffn.shared_experts.w1") ||
+ fp8_view(&w2, weights, "ffn.shared_experts.w2") ||
+ fp8_view(&w3, weights, "ffn.shared_experts.w3"))) result = -1;
+ if (!result) result = coli_v4_shared_expert_forward_ref(
+ shared_output, &w1, &w2, &w3, input, config->swiglu_limit);
+ if (!result) memset(output, 0, (size_t)d * sizeof(*output));
+ for (int expert_id = 0; !result && expert_id < n; expert_id++) {
+ int rank = -1;
+ for (int candidate = 0; candidate < topk; candidate++)
+ if (indices[candidate] == expert_id) rank = candidate;
+ if (rank < 0) continue;
+ ColiExpertView expert;
+ if (coli_expert_lookup(store,
+ (ColiExpertKey){weights->plan.layer, expert_id},
+ &expert)) {
+ result = -1;
+ break;
+ }
+ result = coli_v4_expert_forward_ref(expert_output, &expert, input,
+ route_weights[rank],
+ config->swiglu_limit);
+ coli_expert_release(store, &expert);
+ if (!result)
+ for (int i = 0; i < d; i++) output[i] += expert_output[i];
+ }
+ if (!result)
+ for (int i = 0; i < d; i++)
+ output[i] = coli_bf16_round(output[i] + shared_output[i]);
+ free(shared_output); free(expert_output); free(indices);
+ free(route_weights); free(gate);
+ return result;
+}
+
+static int block_token_impl(float *output_hc,
+ ColiDeepSeekV4WindowAttentionState *attention,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ ColiExpertStore *experts,
+ const float *input_hc, int token, int position,
+ char *error, size_t error_size) {
+ if (!output_hc || !weights || !config || !experts || !input_hc)
+ return set_error(error, error_size, "invalid block arguments");
+ int d = config->hidden_size, hc = config->hc_mult;
+ size_t hd = (size_t)hc * d;
+ float *residual = malloc(hd * sizeof(*residual));
+ float *state = malloc(hd * sizeof(*state));
+ float *reduced = malloc((size_t)d * sizeof(*reduced));
+ float *normalized = malloc((size_t)d * sizeof(*normalized));
+ float *branch = malloc((size_t)d * sizeof(*branch));
+ float *post = malloc((size_t)hc * sizeof(*post));
+ float *comb = malloc((size_t)hc * hc * sizeof(*comb));
+ if (!residual || !state || !reduced || !normalized || !branch || !post || !comb) {
+ free(comb); free(post); free(branch); free(normalized);
+ free(reduced); free(state); free(residual);
+ return set_error(error, error_size, "out of memory in block");
+ }
+ memcpy(residual, input_hc, hd * sizeof(*residual));
+ int result = normalized_hc_pre(reduced, post, comb, normalized, input_hc,
+ weights, config, "attn", "attn_norm.weight");
+ if (!result) result = attention
+ ? coli_v4_attention_window_token_ref(branch, attention, weights, config,
+ normalized, position, error, error_size)
+ : coli_v4_attention_token_ref(branch, weights, config, normalized,
+ position, error, error_size);
+ if (!result) result = coli_v4_hc_post(state, branch, residual, post, comb, hc, d);
+ if (!result) coli_bf16_round_array(state, hd);
+
+ if (!result) memcpy(residual, state, hd * sizeof(*residual));
+ if (!result) result = normalized_hc_pre(reduced, post, comb, normalized, state,
+ weights, config, "ffn", "ffn_norm.weight");
+ if (!result) result = moe_token(branch, weights, config, experts, normalized, token);
+ if (!result) result = coli_v4_hc_post(output_hc, branch, residual, post, comb, hc, d);
+ if (!result) coli_bf16_round_array(output_hc, hd);
+
+ free(comb); free(post); free(branch); free(normalized);
+ free(reduced); free(state); free(residual);
+ return result ? set_error(error, error_size, "block computation failed") : 0;
+}
+
+int coli_v4_block_token_ref(float *output_hc,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ ColiExpertStore *experts,
+ const float *input_hc, int token, int position,
+ char *error, size_t error_size) {
+ return block_token_impl(output_hc, NULL, weights, config, experts, input_hc,
+ token, position, error, error_size);
+}
+
+int coli_v4_block_window_token_ref(
+ float *output_hc, ColiDeepSeekV4WindowAttentionState *attention,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, ColiExpertStore *experts,
+ const float *input_hc, int token, int position,
+ char *error, size_t error_size) {
+ return block_token_impl(output_hc, attention, weights, config, experts,
+ input_hc, token, position, error, error_size);
+}
+/* ---- end include deepseek_v4_block.c ---- */
+
+#undef coli_v4_block_token_ref
+#undef coli_v4_block_window_token_ref
+
+#ifndef COLI_V4_DISABLE_BF16_ROUTE
+int coli_v4_route_bf16(float *weights, int *indices, const float *hidden,
+ const uint16_t *gate, const float *bias,
+ const int *forced_indices, int experts, int dimension,
+ int topk, float route_scale);
+#endif
+
+typedef struct {
+ ColiExpertStore *store;
+ ColiExpertKey key;
+ ColiExpertView view;
+ int result;
+} ExpertLoadJob;
+
+#ifdef COLI_V4_EXPERIMENTAL_PREFETCH
+static int expert_prefetch_enabled(void) {
+ static int enabled = -1;
+ if (enabled < 0) {
+ const char *text = getenv("COLI_V4_EXPERT_PREFETCH");
+ enabled = text && *text && atoi(text) != 0;
+ }
+ return enabled;
+}
+#endif
+
+
+static void *expert_load_worker(void *argument) {
+ ExpertLoadJob *job = argument;
+ job->result = coli_expert_lookup(job->store, job->key, &job->view);
+ return NULL;
+}
+
+typedef struct {
+ pthread_t thread;
+ int active;
+#ifdef COLI_V4_EXPERIMENTAL_DUAL_EXPERT_LOADER
+ int loader_slot;
+#endif
+} ExpertLoadHandle;
+
+#ifdef COLI_V4_EXPERIMENTAL_DUAL_EXPERT_LOADER
+#ifndef COLI_V4_EXPERT_LOADER_COUNT
+#define COLI_V4_EXPERT_LOADER_COUNT 3
+#endif
+enum { DUAL_EXPERT_LOADER_COUNT = COLI_V4_EXPERT_LOADER_COUNT };
+
+typedef struct {
+ pthread_t thread;
+ ExpertLoadJob *job;
+ int pending;
+ int completed;
+ int stopping;
+ int available;
+} DualExpertLoaderSlot;
+
+typedef struct {
+ pthread_mutex_t mutex;
+ pthread_cond_t ready;
+ pthread_cond_t complete;
+ pthread_cond_t idle;
+ DualExpertLoaderSlot slots[DUAL_EXPERT_LOADER_COUNT];
+} DualExpertLoaderPool;
+
+static DualExpertLoaderPool dual_loader_pool = {
+ .mutex = PTHREAD_MUTEX_INITIALIZER,
+ .ready = PTHREAD_COND_INITIALIZER,
+ .complete = PTHREAD_COND_INITIALIZER,
+ .idle = PTHREAD_COND_INITIALIZER,
+};
+static pthread_once_t dual_loader_once = PTHREAD_ONCE_INIT;
+
+static void *dual_expert_loader_worker(void *argument) {
+ DualExpertLoaderSlot *slot = argument;
+ pthread_mutex_lock(&dual_loader_pool.mutex);
+ for (;;) {
+ while (!slot->pending && !slot->stopping)
+ pthread_cond_wait(&dual_loader_pool.ready,
+ &dual_loader_pool.mutex);
+ if (slot->stopping) break;
+ ExpertLoadJob *job = slot->job;
+ slot->pending = 0;
+ pthread_mutex_unlock(&dual_loader_pool.mutex);
+ job->result = coli_expert_lookup(job->store, job->key, &job->view);
+ pthread_mutex_lock(&dual_loader_pool.mutex);
+ slot->completed = 1;
+ pthread_cond_broadcast(&dual_loader_pool.complete);
+ }
+ pthread_mutex_unlock(&dual_loader_pool.mutex);
+ return NULL;
+}
+
+static void dual_expert_loader_shutdown(void) {
+ pthread_mutex_lock(&dual_loader_pool.mutex);
+ for (int i = 0; i < DUAL_EXPERT_LOADER_COUNT; i++)
+ dual_loader_pool.slots[i].stopping = 1;
+ pthread_cond_broadcast(&dual_loader_pool.ready);
+ pthread_mutex_unlock(&dual_loader_pool.mutex);
+ for (int i = 0; i < DUAL_EXPERT_LOADER_COUNT; i++)
+ if (dual_loader_pool.slots[i].available) {
+ pthread_join(dual_loader_pool.slots[i].thread, NULL);
+ dual_loader_pool.slots[i].available = 0;
+ }
+}
+
+static void dual_expert_loader_init(void) {
+ int available = 0;
+ for (int i = 0; i < DUAL_EXPERT_LOADER_COUNT; i++) {
+ DualExpertLoaderSlot *slot = &dual_loader_pool.slots[i];
+ if (!pthread_create(&slot->thread, NULL,
+ dual_expert_loader_worker, slot)) {
+ slot->available = 1;
+ available++;
+ }
+ }
+ if (available) atexit(dual_expert_loader_shutdown);
+}
+
+static int dual_expert_load_start(ExpertLoadHandle *handle,
+ ExpertLoadJob *job) {
+ pthread_once(&dual_loader_once, dual_expert_loader_init);
+ pthread_mutex_lock(&dual_loader_pool.mutex);
+ int selected = -1;
+ while (selected < 0) {
+ int available = 0;
+ for (int i = 0; i < DUAL_EXPERT_LOADER_COUNT; i++) {
+ DualExpertLoaderSlot *slot = &dual_loader_pool.slots[i];
+ if (!slot->available) continue;
+ available++;
+ if (!slot->job && !slot->pending) { selected = i; break; }
+ }
+ if (!available ||
+ (selected < 0 && available < DUAL_EXPERT_LOADER_COUNT)) {
+ pthread_mutex_unlock(&dual_loader_pool.mutex);
+ return -1;
+ }
+ if (selected < 0)
+ pthread_cond_wait(&dual_loader_pool.idle,
+ &dual_loader_pool.mutex);
+ }
+ DualExpertLoaderSlot *slot = &dual_loader_pool.slots[selected];
+ slot->job = job;
+ slot->completed = 0;
+ slot->pending = 1;
+ handle->active = 1;
+ handle->loader_slot = selected;
+ pthread_cond_broadcast(&dual_loader_pool.ready);
+ pthread_mutex_unlock(&dual_loader_pool.mutex);
+ return 0;
+}
+
+static int dual_expert_load_finish(ExpertLoadHandle *handle) {
+ if (!handle->active || handle->loader_slot < 0 ||
+ handle->loader_slot >= DUAL_EXPERT_LOADER_COUNT) return -1;
+ pthread_mutex_lock(&dual_loader_pool.mutex);
+ DualExpertLoaderSlot *slot =
+ &dual_loader_pool.slots[handle->loader_slot];
+ while (!slot->completed)
+ pthread_cond_wait(&dual_loader_pool.complete,
+ &dual_loader_pool.mutex);
+ slot->job = NULL;
+ pthread_cond_broadcast(&dual_loader_pool.idle);
+ pthread_mutex_unlock(&dual_loader_pool.mutex);
+ handle->active = 0;
+ return 0;
+}
+#endif
+
+#if !defined(COLI_V4_DISABLE_PERSISTENT_EXPERT_LOADER) && \
+ !defined(COLI_V4_EXPERIMENTAL_DUAL_EXPERT_LOADER)
+typedef struct {
+ pthread_mutex_t mutex;
+ pthread_cond_t ready;
+ pthread_cond_t complete;
+ pthread_cond_t idle;
+ pthread_t thread;
+ ExpertLoadJob *job;
+ int pending;
+ int completed;
+ int stopping;
+ int available;
+} PersistentExpertLoader;
+
+static PersistentExpertLoader persistent_loader = {
+ .mutex = PTHREAD_MUTEX_INITIALIZER,
+ .ready = PTHREAD_COND_INITIALIZER,
+ .complete = PTHREAD_COND_INITIALIZER,
+ .idle = PTHREAD_COND_INITIALIZER,
+};
+static pthread_once_t persistent_loader_once = PTHREAD_ONCE_INIT;
+
+static void *persistent_expert_loader_worker(void *unused) {
+ (void)unused;
+ pthread_mutex_lock(&persistent_loader.mutex);
+ for (;;) {
+ while (!persistent_loader.pending && !persistent_loader.stopping)
+ pthread_cond_wait(&persistent_loader.ready,
+ &persistent_loader.mutex);
+ if (persistent_loader.stopping) break;
+ ExpertLoadJob *job = persistent_loader.job;
+ persistent_loader.pending = 0;
+ pthread_mutex_unlock(&persistent_loader.mutex);
+ job->result = coli_expert_lookup(job->store, job->key, &job->view);
+ pthread_mutex_lock(&persistent_loader.mutex);
+ persistent_loader.completed = 1;
+ pthread_cond_signal(&persistent_loader.complete);
+ }
+ pthread_mutex_unlock(&persistent_loader.mutex);
+ return NULL;
+}
+
+static void persistent_expert_loader_shutdown(void) {
+ if (!persistent_loader.available) return;
+ pthread_mutex_lock(&persistent_loader.mutex);
+ persistent_loader.stopping = 1;
+ pthread_cond_signal(&persistent_loader.ready);
+ pthread_mutex_unlock(&persistent_loader.mutex);
+ pthread_join(persistent_loader.thread, NULL);
+ persistent_loader.available = 0;
+}
+
+static void persistent_expert_loader_init(void) {
+ if (!pthread_create(&persistent_loader.thread, NULL,
+ persistent_expert_loader_worker, NULL)) {
+ persistent_loader.available = 1;
+ atexit(persistent_expert_loader_shutdown);
+ }
+}
+#endif
+
+static int expert_load_start(ExpertLoadHandle *handle, ExpertLoadJob *job) {
+#ifdef COLI_V4_EXPERIMENTAL_DUAL_EXPERT_LOADER
+ if (!dual_expert_load_start(handle, job)) return 0;
+ if (pthread_create(&handle->thread, NULL, expert_load_worker, job))
+ return -1;
+ handle->loader_slot = -1;
+ handle->active = 1;
+ return 0;
+#elif defined(COLI_V4_EXPERIMENTAL_SYNC_EXPERT_LOOKUP)
+ job->result = coli_expert_lookup(job->store, job->key, &job->view);
+ handle->active = 1;
+ return 0;
+#elif !defined(COLI_V4_DISABLE_PERSISTENT_EXPERT_LOADER)
+ pthread_once(&persistent_loader_once, persistent_expert_loader_init);
+ if (!persistent_loader.available) return -1;
+ pthread_mutex_lock(&persistent_loader.mutex);
+ while (persistent_loader.job || persistent_loader.pending)
+ pthread_cond_wait(&persistent_loader.idle,
+ &persistent_loader.mutex);
+ persistent_loader.job = job;
+ persistent_loader.completed = 0;
+ persistent_loader.pending = 1;
+ handle->active = 1;
+ pthread_cond_signal(&persistent_loader.ready);
+ pthread_mutex_unlock(&persistent_loader.mutex);
+ return 0;
+#else
+ if (pthread_create(&handle->thread, NULL, expert_load_worker, job))
+ return -1;
+ handle->active = 1;
+ return 0;
+#endif
+}
+
+static int expert_load_finish(ExpertLoadHandle *handle) {
+ if (!handle->active) return -1;
+#ifdef COLI_V4_EXPERIMENTAL_DUAL_EXPERT_LOADER
+ if (handle->loader_slot >= 0)
+ return dual_expert_load_finish(handle);
+ int result = pthread_join(handle->thread, NULL);
+ handle->active = 0;
+ return result;
+#elif defined(COLI_V4_EXPERIMENTAL_SYNC_EXPERT_LOOKUP)
+ handle->active = 0;
+ return 0;
+#elif !defined(COLI_V4_DISABLE_PERSISTENT_EXPERT_LOADER)
+ pthread_mutex_lock(&persistent_loader.mutex);
+ while (!persistent_loader.completed)
+ pthread_cond_wait(&persistent_loader.complete,
+ &persistent_loader.mutex);
+ persistent_loader.job = NULL;
+ pthread_cond_broadcast(&persistent_loader.idle);
+ pthread_mutex_unlock(&persistent_loader.mutex);
+ handle->active = 0;
+ return 0;
+#else
+ int result = pthread_join(handle->thread, NULL);
+ handle->active = 0;
+ return result;
+#endif
+}
+
+#ifdef COLI_V4_EXPERIMENTAL_BLOCK_OTHER_PROFILE
+enum {
+ COLI_V4_BLOCK_PROFILE_MOE_TOTAL = 0,
+ COLI_V4_BLOCK_PROFILE_GATE_DECODE = 1,
+ COLI_V4_BLOCK_PROFILE_LOADER_START = 2,
+ COLI_V4_BLOCK_PROFILE_LOADER_WAIT = 3,
+};
+double coli_v4_block_profile_now(void);
+void coli_v4_block_profile_add(int kind, double seconds);
+#endif
+
+static int profiled_expert_load_start(ExpertLoadHandle *handle,
+ ExpertLoadJob *job) {
+#ifdef COLI_V4_EXPERIMENTAL_BLOCK_OTHER_PROFILE
+ double began = coli_v4_block_profile_now();
+#endif
+ int result = expert_load_start(handle, job);
+#ifdef COLI_V4_EXPERIMENTAL_BLOCK_OTHER_PROFILE
+ coli_v4_block_profile_add(COLI_V4_BLOCK_PROFILE_LOADER_START,
+ coli_v4_block_profile_now() - began);
+#endif
+ return result;
+}
+
+static int profiled_expert_load_finish(ExpertLoadHandle *handle) {
+#ifdef COLI_V4_EXPERIMENTAL_BLOCK_OTHER_PROFILE
+ double began = coli_v4_block_profile_now();
+#endif
+ int result = expert_load_finish(handle);
+#ifdef COLI_V4_EXPERIMENTAL_BLOCK_OTHER_PROFILE
+ coli_v4_block_profile_add(COLI_V4_BLOCK_PROFILE_LOADER_WAIT,
+ coli_v4_block_profile_now() - began);
+#endif
+ return result;
+}
+
+static int moe_token_pipeline(float *output,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ ColiExpertStore *store,
+ const float *input, int token) {
+#ifdef COLI_V4_EXPERIMENTAL_BLOCK_OTHER_PROFILE
+ double profile_moe_began = coli_v4_block_profile_now();
+#endif
+ int d = config->hidden_size;
+ int n = config->n_routed_experts;
+ int topk = config->num_experts_per_tok;
+#ifndef COLI_V4_DISABLE_BF16_ROUTE
+ float *gate = NULL;
+ const uint16_t *raw_gate = value(weights, "ffn.gate.weight", NULL);
+ int missing_gate = !raw_gate;
+#else
+ size_t gate_count = (size_t)n * d;
+ float *gate = malloc(gate_count * sizeof(*gate));
+ int missing_gate = !gate;
+#endif
+ float *route_weights = malloc((size_t)topk * sizeof(*route_weights));
+ int *indices = malloc((size_t)topk * sizeof(*indices));
+ int *expert_ids = malloc((size_t)topk * sizeof(*expert_ids));
+ float *expert_weights = malloc((size_t)topk * sizeof(*expert_weights));
+ float *expert_output = malloc((size_t)d * sizeof(*expert_output));
+ float *shared_output = malloc((size_t)d * sizeof(*shared_output));
+ if (missing_gate || !route_weights || !indices || !expert_ids || !expert_weights ||
+ !expert_output || !shared_output) {
+ free(shared_output); free(expert_output); free(expert_weights);
+ free(expert_ids); free(indices); free(route_weights); free(gate);
+ return -1;
+ }
+#ifdef COLI_V4_DISABLE_BF16_ROUTE
+#ifdef COLI_V4_EXPERIMENTAL_BLOCK_OTHER_PROFILE
+ double profile_gate_began = coli_v4_block_profile_now();
+#endif
+ decode_bf16(gate, value(weights, "ffn.gate.weight", NULL), gate_count);
+#ifdef COLI_V4_EXPERIMENTAL_BLOCK_OTHER_PROFILE
+ coli_v4_block_profile_add(COLI_V4_BLOCK_PROFILE_GATE_DECODE,
+ coli_v4_block_profile_now() - profile_gate_began);
+#endif
+#endif
+ const int64_t *table = value(weights, "ffn.gate.tid2eid", NULL);
+ const float *bias = value(weights, "ffn.gate.bias", NULL);
+ int result = token < 0 || token >= config->vocab_size;
+ if (!result && weights->plan.uses_hash_router && !table) result = -1;
+ if (!result && weights->plan.uses_hash_router)
+ for (int i = 0; i < topk; i++)
+ indices[i] = (int)table[(size_t)token * topk + i];
+#ifndef COLI_V4_DISABLE_BF16_ROUTE
+ if (!result) result = coli_v4_route_bf16(
+ route_weights, indices, input, raw_gate, bias,
+ weights->plan.uses_hash_router ? indices : NULL,
+ n, d, topk, config->routed_scaling_factor);
+#else
+ if (!result) result = coli_v4_route(
+ route_weights, indices, input, gate, bias,
+ weights->plan.uses_hash_router ? indices : NULL,
+ n, d, topk, config->routed_scaling_factor);
+#endif
+
+ int selected = 0;
+ for (int expert_id = 0; !result && expert_id < n; expert_id++) {
+ for (int rank = 0; rank < topk; rank++) {
+ if (indices[rank] == expert_id) {
+ expert_ids[selected] = expert_id;
+ expert_weights[selected] = route_weights[rank];
+ selected++;
+ }
+ }
+ }
+ if (!result && selected != topk) result = -1;
+
+#ifdef COLI_V4_EXPERIMENTAL_PREFETCH
+ if (!result && expert_prefetch_enabled() && store->ops->prefetch) {
+ ColiExpertKey *keys = malloc((size_t)selected * sizeof(*keys));
+ if (keys) {
+ for (int i = 0; i < selected; i++)
+ keys[i] = (ColiExpertKey){weights->plan.layer, expert_ids[i]};
+ store->ops->prefetch(store, keys, (size_t)selected);
+ free(keys);
+ }
+ }
+#endif
+
+
+#ifdef COLI_V4_EXPERIMENTAL_DUAL_EXPERT_LOADER
+ ExpertLoadJob jobs[DUAL_EXPERT_LOADER_COUNT] = {{0}};
+ ExpertLoadHandle loaders[DUAL_EXPERT_LOADER_COUNT] = {{0}};
+ int loader_active[DUAL_EXPERT_LOADER_COUNT] = {0};
+ if (!result) {
+ int preload = selected < DUAL_EXPERT_LOADER_COUNT
+ ? selected : DUAL_EXPERT_LOADER_COUNT;
+ for (int i = 0; i < preload; i++) {
+ jobs[i].store = store;
+ jobs[i].key = (ColiExpertKey){weights->plan.layer, expert_ids[i]};
+ jobs[i].result = -1;
+ if (profiled_expert_load_start(&loaders[i], &jobs[i]) != 0) {
+ result = -1;
+ break;
+ }
+ loader_active[i] = 1;
+ }
+ }
+#else
+ ExpertLoadJob job = {0};
+ ExpertLoadHandle loader = {0};
+ int loader_active = 0;
+ if (!result) {
+ job.store = store;
+ job.key = (ColiExpertKey){weights->plan.layer, expert_ids[0]};
+ job.result = -1;
+ if (profiled_expert_load_start(&loader, &job) != 0)
+ result = -1;
+ else
+ loader_active = 1;
+ }
+#endif
+
+ ColiTensorView w1, w2, w3;
+ if (!result && (fp8_view(&w1, weights, "ffn.shared_experts.w1") ||
+ fp8_view(&w2, weights, "ffn.shared_experts.w2") ||
+ fp8_view(&w3, weights, "ffn.shared_experts.w3"))) result = -1;
+ if (!result) result = coli_v4_shared_expert_forward_ref(
+ shared_output, &w1, &w2, &w3, input, config->swiglu_limit);
+ if (!result) memset(output, 0, (size_t)d * sizeof(*output));
+
+#ifdef COLI_V4_EXPERIMENTAL_DUAL_EXPERT_LOADER
+ for (int current = 0; !result && current < selected; current++) {
+ int slot = current % DUAL_EXPERT_LOADER_COUNT;
+ if (!loader_active[slot] ||
+ profiled_expert_load_finish(&loaders[slot]) != 0) {
+ result = -1; break;
+ }
+ loader_active[slot] = 0;
+ if (jobs[slot].result) { result = -1; break; }
+ ColiExpertView expert = jobs[slot].view;
+
+ int next = current + DUAL_EXPERT_LOADER_COUNT;
+ if (next < selected) {
+ memset(&jobs[slot], 0, sizeof(jobs[slot]));
+ jobs[slot].store = store;
+ jobs[slot].key = (ColiExpertKey){weights->plan.layer,
+ expert_ids[next]};
+ jobs[slot].result = -1;
+ if (profiled_expert_load_start(&loaders[slot],
+ &jobs[slot]) != 0)
+ result = -1;
+ else
+ loader_active[slot] = 1;
+ }
+ if (!result) result = coli_v4_expert_forward_ref(
+ expert_output, &expert, input, expert_weights[current],
+ config->swiglu_limit);
+ coli_expert_release(store, &expert);
+ if (!result)
+ for (int i = 0; i < d; i++) output[i] += expert_output[i];
+ }
+ for (int slot = 0; slot < DUAL_EXPERT_LOADER_COUNT; slot++)
+ if (loader_active[slot]) {
+ profiled_expert_load_finish(&loaders[slot]);
+ if (!jobs[slot].result)
+ coli_expert_release(store, &jobs[slot].view);
+ }
+#else
+ for (int current = 0; current < selected && loader_active; current++) {
+ if (profiled_expert_load_finish(&loader) != 0) {
+ result = -1; loader_active = 0; break;
+ }
+ loader_active = 0;
+ if (job.result) { result = -1; break; }
+ ColiExpertView expert = job.view;
+
+ if (current + 1 < selected) {
+ memset(&job, 0, sizeof(job));
+ job.store = store;
+ job.key = (ColiExpertKey){weights->plan.layer,
+ expert_ids[current + 1]};
+ job.result = -1;
+ if (profiled_expert_load_start(&loader, &job) != 0)
+ result = -1;
+ else
+ loader_active = 1;
+ }
+ if (!result) result = coli_v4_expert_forward_ref(
+ expert_output, &expert, input, expert_weights[current],
+ config->swiglu_limit);
+ coli_expert_release(store, &expert);
+ if (!result)
+ for (int i = 0; i < d; i++) output[i] += expert_output[i];
+ }
+ if (loader_active) {
+ profiled_expert_load_finish(&loader);
+ if (!job.result) coli_expert_release(store, &job.view);
+ }
+#endif
+ if (!result)
+ for (int i = 0; i < d; i++)
+ output[i] = coli_bf16_round(output[i] + shared_output[i]);
+
+ free(shared_output); free(expert_output); free(expert_weights);
+ free(expert_ids); free(indices); free(route_weights); free(gate);
+#ifdef COLI_V4_EXPERIMENTAL_BLOCK_OTHER_PROFILE
+ coli_v4_block_profile_add(COLI_V4_BLOCK_PROFILE_MOE_TOTAL,
+ coli_v4_block_profile_now() - profile_moe_began);
+#endif
+ return result;
+}
+
+static int block_token_pipeline(float *output_hc,
+ ColiDeepSeekV4WindowAttentionState *attention,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ ColiExpertStore *experts,
+ const float *input_hc, int token, int position,
+ char *error, size_t error_size) {
+ if (!output_hc || !weights || !config || !experts || !input_hc)
+ return set_error(error, error_size, "invalid block arguments");
+ int d = config->hidden_size, hc = config->hc_mult;
+ size_t hd = (size_t)hc * d;
+ float *residual = malloc(hd * sizeof(*residual));
+ float *state = malloc(hd * sizeof(*state));
+ float *reduced = malloc((size_t)d * sizeof(*reduced));
+ float *normalized = malloc((size_t)d * sizeof(*normalized));
+ float *branch = malloc((size_t)d * sizeof(*branch));
+ float *post = malloc((size_t)hc * sizeof(*post));
+ float *comb = malloc((size_t)hc * hc * sizeof(*comb));
+ if (!residual || !state || !reduced || !normalized || !branch ||
+ !post || !comb) {
+ free(comb); free(post); free(branch); free(normalized);
+ free(reduced); free(state); free(residual); return -1;
+ }
+ memcpy(residual, input_hc, hd * sizeof(*residual));
+ int result = normalized_hc_pre(reduced, post, comb, normalized, input_hc,
+ weights, config, "attn", "attn_norm.weight");
+ if (!result) result = attention
+ ? coli_v4_attention_window_token_ref(branch, attention, weights, config,
+ normalized, position, error, error_size)
+ : coli_v4_attention_token_ref(branch, weights, config, normalized,
+ position, error, error_size);
+ if (!result) result = coli_v4_hc_post(state, branch, residual,
+ post, comb, hc, d);
+ if (!result) coli_bf16_round_array(state, hd);
+ if (!result) memcpy(residual, state, hd * sizeof(*residual));
+ if (!result) result = normalized_hc_pre(reduced, post, comb, normalized, state,
+ weights, config, "ffn",
+ "ffn_norm.weight");
+ if (!result) result = moe_token_pipeline(branch, weights, config, experts,
+ normalized, token);
+ if (!result) result = coli_v4_hc_post(output_hc, branch, residual,
+ post, comb, hc, d);
+ if (!result) coli_bf16_round_array(output_hc, hd);
+ free(comb); free(post); free(branch); free(normalized);
+ free(reduced); free(state); free(residual);
+ return result ? set_error(error, error_size, "block computation failed") : 0;
+}
+
+int coli_v4_block_token_ref(float *output_hc,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ ColiExpertStore *experts,
+ const float *input_hc, int token, int position,
+ char *error, size_t error_size) {
+ return block_token_pipeline(output_hc, NULL, weights, config, experts,
+ input_hc, token, position, error, error_size);
+}
+
+int coli_v4_block_window_token_ref(
+ float *output_hc, ColiDeepSeekV4WindowAttentionState *attention,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, ColiExpertStore *experts,
+ const float *input_hc, int token, int position,
+ char *error, size_t error_size) {
+ return block_token_pipeline(output_hc, attention, weights, config, experts,
+ input_hc, token, position, error, error_size);
+}
+/* ---- end include deepseek_v4_block_pipeline.c ---- */
+
+
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+
+int coli_v4_block_window_batch_ref(
+ float *outputs_hc, ColiDeepSeekV4WindowAttentionState *attention,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, ColiExpertStore *experts,
+ const float *inputs_hc, const int *tokens, int start_position, int batch,
+ char *error, size_t error_size) {
+ if (!outputs_hc || !attention || !weights || !config || !experts ||
+ !inputs_hc || !tokens || batch < 1 || batch > 64) return -1;
+ int d = config->hidden_size, hc = config->hc_mult;
+ size_t hd = (size_t)hc * d;
+ float *states = malloc((size_t)batch * hd * sizeof(*states));
+ float *normalized = malloc((size_t)batch * d * sizeof(*normalized));
+ float *branches = malloc((size_t)batch * d * sizeof(*branches));
+ float *posts = malloc((size_t)batch * hc * sizeof(*posts));
+ float *combs = malloc((size_t)batch * hc * hc * sizeof(*combs));
+ float *reduced = malloc((size_t)d * sizeof(*reduced));
+ float *ffn_normalized = malloc((size_t)d * sizeof(*ffn_normalized));
+ float *ffn_branch = malloc((size_t)d * sizeof(*ffn_branch));
+ float *ffn_post = malloc((size_t)hc * sizeof(*ffn_post));
+ float *ffn_comb = malloc((size_t)hc * hc * sizeof(*ffn_comb));
+ if (!states || !normalized || !branches || !posts || !combs || !reduced ||
+ !ffn_normalized || !ffn_branch || !ffn_post || !ffn_comb) {
+ free(ffn_comb); free(ffn_post); free(ffn_branch); free(ffn_normalized);
+ free(reduced); free(combs); free(posts); free(branches);
+ free(normalized); free(states); return -1;
+ }
+ int result = 0;
+ const char *phase = "attention hyper-connection";
+ for (int item = 0; !result && item < batch; item++)
+ result = normalized_hc_pre(
+ reduced, posts + (size_t)item * hc,
+ combs + (size_t)item * hc * hc,
+ normalized + (size_t)item * d,
+ inputs_hc + (size_t)item * hd,
+ weights, config, "attn", "attn_norm.weight");
+ phase = "attention";
+ if (!result) result = coli_v4_attention_window_batch_ref(
+ branches, attention, weights, config, normalized,
+ start_position, batch, error, error_size);
+ if (!result) phase = "attention post / FFN hyper-connection";
+ for (int item = 0; !result && item < batch; item++) {
+ float *state = states + (size_t)item * hd;
+ result = coli_v4_hc_post(
+ state, branches + (size_t)item * d,
+ inputs_hc + (size_t)item * hd,
+ posts + (size_t)item * hc,
+ combs + (size_t)item * hc * hc, hc, d);
+ if (!result) coli_bf16_round_array(state, hd);
+ if (!result) phase = "FFN hyper-connection";
+ if (!result) result = normalized_hc_pre(
+ reduced, ffn_post, ffn_comb, ffn_normalized, state,
+ weights, config, "ffn", "ffn_norm.weight");
+ if (!result) phase = "MoE";
+ if (!result) result = moe_token_pipeline(
+ ffn_branch, weights, config, experts,
+ ffn_normalized, tokens[item]);
+ if (!result) result = coli_v4_hc_post(
+ outputs_hc + (size_t)item * hd, ffn_branch, state,
+ ffn_post, ffn_comb, hc, d);
+ if (!result) coli_bf16_round_array(
+ outputs_hc + (size_t)item * hd, hd);
+ }
+ free(ffn_comb); free(ffn_post); free(ffn_branch); free(ffn_normalized);
+ free(reduced); free(combs); free(posts); free(branches);
+ free(normalized); free(states);
+ if (!result) return 0;
+ if (error && error_size && error[0]) return -1;
+ return set_error(error, error_size, "hybrid batched block failed in %s", phase);
+}
+#endif /* COLI_V4_UNIT_BLOCK_HYBRID */
+
+#ifdef COLI_V4_UNIT_COMPRESSOR_SNAPSHOT
+/* ######## deepseek_v4_compressor_snapshot.c ######## */
+#define coli_v4_compressor_create snapshot_copy_compressor_create
+#define coli_v4_compressor_create_with_options snapshot_copy_compressor_create_with_options
+#define coli_v4_compressor_reset snapshot_copy_compressor_reset
+#define coli_v4_compressor_bind_weights snapshot_copy_compressor_bind_weights
+#define coli_v4_compressor_destroy snapshot_copy_compressor_destroy
+#define coli_v4_compressor_step snapshot_copy_compressor_step
+/* ---- begin include deepseek_v4_compressor.c ---- */
+#include "deepseek_v4_internal.h"
+
+#include <math.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "deepseek_v4_internal.h"
+#include "native_quant.h"
+
+struct ColiDeepSeekV4CompressorState {
+ const ColiDeepSeekV4LayerWeights *weights;
+ const ColiDeepSeekV4Config *config;
+ int ratio;
+ int layer;
+ int hidden;
+ int head_dim;
+ int projection_dim;
+ int state_rows;
+ int rope_dim;
+ int rotate_fp4;
+ char prefix[96];
+ float *kv_state;
+ float *score_state;
+};
+
+static int set_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list arguments;
+ va_start(arguments, format);
+ vsnprintf(error, size, format, arguments);
+ va_end(arguments);
+ }
+ return -1;
+}
+
+static const void *layer_value(const ColiDeepSeekV4LayerWeights *weights,
+ const char *suffix) {
+ char name[COLI_V4_MAX_TENSOR_NAME];
+ snprintf(name, sizeof(name), "layers.%d.%s", weights->plan.layer, suffix);
+ return coli_v4_layer_data(weights, name, NULL);
+}
+
+int coli_v4_compressor_create(ColiDeepSeekV4CompressorState **output,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ char *error, size_t error_size) {
+ ColiDeepSeekV4CompressorOptions options = {
+ "attn.compressor", config ? config->head_dim : 0, 0
+ };
+ return coli_v4_compressor_create_with_options(
+ output, weights, config, &options, error, error_size);
+}
+
+int coli_v4_compressor_create_with_options(
+ ColiDeepSeekV4CompressorState **output,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const ColiDeepSeekV4CompressorOptions *options,
+ char *error, size_t error_size) {
+ if (!output || !weights || !config || !options || !options->prefix ||
+ !options->prefix[0] || options->head_dimension <= 0 ||
+ weights->plan.compression_ratio < 1)
+ return set_error(error, error_size, "unsupported compressor ratio");
+ if (strlen(options->prefix) >= sizeof(((ColiDeepSeekV4CompressorState *)0)->prefix))
+ return set_error(error, error_size, "compressor prefix is too long");
+ *output = NULL;
+ ColiDeepSeekV4CompressorState *state = calloc(1, sizeof(*state));
+ if (!state) return set_error(error, error_size, "out of memory creating compressor");
+ state->weights = weights;
+ state->config = config;
+ state->ratio = weights->plan.compression_ratio;
+ state->layer = weights->plan.layer;
+ state->hidden = config->hidden_size;
+ state->head_dim = options->head_dimension;
+ state->rotate_fp4 = options->rotate_fp4 != 0;
+ memcpy(state->prefix, options->prefix, strlen(options->prefix) + 1);
+ int overlap = state->ratio == 4;
+ state->projection_dim = (1 + overlap) * state->head_dim;
+ state->state_rows = (1 + overlap) * state->ratio;
+ state->rope_dim = config->qk_rope_head_dim;
+ size_t count = (size_t)state->state_rows * state->projection_dim;
+ state->kv_state = calloc(count, sizeof(*state->kv_state));
+ state->score_state = malloc(count * sizeof(*state->score_state));
+ if (!state->kv_state || !state->score_state) {
+ coli_v4_compressor_destroy(state);
+ return set_error(error, error_size, "out of memory allocating compressor state");
+ }
+ for (size_t i = 0; i < count; i++) state->score_state[i] = -INFINITY;
+ *output = state;
+ return 0;
+}
+
+int coli_v4_compressor_bind_weights(ColiDeepSeekV4CompressorState *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ char *error, size_t error_size) {
+ if (!state || !weights ||
+ weights->plan.layer != state->layer ||
+ weights->plan.compression_ratio != state->ratio)
+ return set_error(error, error_size, "incompatible compressor weights");
+ state->weights = weights;
+ return 0;
+}
+
+void coli_v4_compressor_reset(ColiDeepSeekV4CompressorState *state) {
+ if (!state) return;
+ size_t count = (size_t)state->state_rows * state->projection_dim;
+ memset(state->kv_state, 0, count * sizeof(*state->kv_state));
+ for (size_t i = 0; i < count; i++) state->score_state[i] = -INFINITY;
+}
+
+void coli_v4_compressor_destroy(ColiDeepSeekV4CompressorState *state) {
+ if (!state) return;
+ free(state->score_state);
+ free(state->kv_state);
+ free(state);
+}
+
+int coli_v4_compressor_step(ColiDeepSeekV4CompressorState *state,
+ float *output, int *produced,
+ const float *input, int position,
+ char *error, size_t error_size) {
+ if (!state || !produced || !input || position < 0)
+ return set_error(error, error_size, "invalid compressor step arguments");
+ *produced = 0;
+ int slot = position % state->ratio;
+ int hidden = state->hidden, dimension = state->head_dim;
+ int projection = state->projection_dim;
+ int state_row = state->ratio == 4 ? state->ratio + slot : slot;
+ char suffix[128];
+ snprintf(suffix, sizeof(suffix), "%s.wkv.weight", state->prefix);
+ const uint16_t *wkv = layer_value(state->weights, suffix);
+ snprintf(suffix, sizeof(suffix), "%s.wgate.weight", state->prefix);
+ const uint16_t *wgate = layer_value(state->weights, suffix);
+ snprintf(suffix, sizeof(suffix), "%s.ape", state->prefix);
+ const float *ape = layer_value(state->weights, suffix);
+ if (!wkv || !wgate || !ape)
+ return set_error(error, error_size, "missing compressor tensor for %s", state->prefix);
+ float *kv_row = state->kv_state + (size_t)state_row * projection;
+ float *score_row = state->score_state + (size_t)state_row * projection;
+ #pragma omp parallel for
+ for (int row = 0; row < projection; row++) {
+ float kv_sum = 0.0f, gate_sum = 0.0f;
+ const uint16_t *kv_weight = wkv + (size_t)row * hidden;
+ const uint16_t *gate_weight = wgate + (size_t)row * hidden;
+ for (int column = 0; column < hidden; column++) {
+ float value = input[column];
+ kv_sum += coli_bf16_decode(kv_weight[column]) * value;
+ gate_sum += coli_bf16_decode(gate_weight[column]) * value;
+ }
+ kv_row[row] = kv_sum;
+ score_row[row] = gate_sum + ape[(size_t)slot * projection + row];
+ }
+ if ((position + 1) % state->ratio != 0) return 0;
+ if (!output) return set_error(error, error_size, "compressor output is required");
+
+ #pragma omp parallel for
+ for (int column = 0; column < dimension; column++) {
+ float maximum = -INFINITY;
+ int pool_rows = state->ratio == 4 ? 2 * state->ratio : state->ratio;
+ for (int row = 0; row < pool_rows; row++) {
+ int source_row = row;
+ int source_column = column;
+ if (state->ratio == 4 && row >= state->ratio)
+ source_column += dimension;
+ float score = state->score_state[
+ (size_t)source_row * projection + source_column];
+ if (score > maximum) maximum = score;
+ }
+ float total = 0.0f, weighted = 0.0f;
+ for (int row = 0; row < pool_rows; row++) {
+ int source_column = column;
+ if (state->ratio == 4 && row >= state->ratio)
+ source_column += dimension;
+ float weight = expf(state->score_state[
+ (size_t)row * projection + source_column] - maximum);
+ total += weight;
+ weighted += state->kv_state[
+ (size_t)row * projection + source_column] * weight;
+ }
+ output[column] = weighted / total;
+ }
+ if (state->ratio == 4) {
+ memcpy(state->kv_state,
+ state->kv_state + (size_t)state->ratio * projection,
+ (size_t)state->ratio * projection * sizeof(*state->kv_state));
+ memcpy(state->score_state,
+ state->score_state + (size_t)state->ratio * projection,
+ (size_t)state->ratio * projection * sizeof(*state->score_state));
+ }
+ coli_bf16_round_array(output, (size_t)dimension);
+ snprintf(suffix, sizeof(suffix), "%s.norm.weight", state->prefix);
+ const uint16_t *raw_norm = layer_value(state->weights, suffix);
+ float *norm = malloc((size_t)dimension * sizeof(*norm));
+ if (!raw_norm || !norm) {
+ free(norm);
+ return set_error(error, error_size, "missing compressor norm");
+ }
+ for (int i = 0; i < dimension; i++) norm[i] = coli_bf16_decode(raw_norm[i]);
+ coli_v4_rmsnorm(output, output, norm, dimension, state->config->rms_norm_eps);
+ coli_bf16_round_array(output, (size_t)dimension);
+ free(norm);
+
+ int rope_position = position + 1 - state->ratio;
+ int pairs = state->rope_dim / 2;
+ size_t table_count = (size_t)(rope_position + 1) * pairs;
+ float *cosines = malloc(table_count * sizeof(*cosines));
+ float *sines = malloc(table_count * sizeof(*sines));
+ if (!cosines || !sines || coli_v4_rope_precompute(
+ cosines, sines, state->rope_dim, rope_position + 1,
+ state->config->original_max_position_embeddings,
+ state->config->compress_rope_theta, state->config->rope_factor,
+ state->config->rope_beta_fast, state->config->rope_beta_slow)) {
+ free(sines); free(cosines);
+ return set_error(error, error_size, "cannot create compressor RoPE table");
+ }
+ float *rope = output + dimension - state->rope_dim;
+ coli_v4_rope_apply(rope, 1, state->rope_dim,
+ cosines + (size_t)rope_position * pairs,
+ sines + (size_t)rope_position * pairs, 0);
+ coli_bf16_round_array(rope, (size_t)state->rope_dim);
+ free(sines); free(cosines);
+
+ size_t quantized = state->rotate_fp4
+ ? (size_t)dimension : (size_t)(dimension - state->rope_dim);
+ size_t block = state->rotate_fp4 ? 32u : 64u;
+ float *qdq = malloc(quantized * sizeof(*qdq));
+ uint8_t *scales = malloc((quantized + block - 1) / block);
+ if (!qdq || !scales) {
+ free(scales); free(qdq);
+ return set_error(error, error_size, "compressor activation quantization failed");
+ }
+ int quant_error = 0;
+ if (state->rotate_fp4)
+ quant_error = coli_hadamard_bf16_ref(output, (size_t)dimension) ||
+ coli_fp4_activation_qdq_ref(qdq, scales, output,
+ quantized, block);
+ else
+ quant_error = coli_fp8_activation_qdq_ref(qdq, scales, output,
+ quantized, block);
+ if (quant_error) {
+ free(scales); free(qdq);
+ return set_error(error, error_size, "compressor activation quantization failed");
+ }
+ memcpy(output, qdq, quantized * sizeof(*output));
+ coli_bf16_round_array(output, quantized);
+ free(scales); free(qdq);
+ *produced = 1;
+ return 0;
+}
+/* ---- end include deepseek_v4_compressor.c ---- */
+
+#undef coli_v4_compressor_step
+#undef coli_v4_compressor_destroy
+#undef coli_v4_compressor_bind_weights
+#undef coli_v4_compressor_reset
+#undef coli_v4_compressor_create_with_options
+#undef coli_v4_compressor_create
+
+#include "deepseek_v4_internal.h"
+
+struct ColiV4CompressorSnapshot {
+ size_t count;
+ float *kv_state;
+ float *score_state;
+};
+
+int coli_v4_compressor_snapshot_create(
+ const ColiDeepSeekV4CompressorState *state,
+ ColiV4CompressorSnapshot **output) {
+ if (!state || !output) return -1;
+ *output = calloc(1, sizeof(**output));
+ if (!*output) return -1;
+ (*output)->count = (size_t)state->state_rows * state->projection_dim;
+ (*output)->kv_state = malloc((*output)->count * sizeof(float));
+ (*output)->score_state = malloc((*output)->count * sizeof(float));
+ if (!(*output)->kv_state || !(*output)->score_state) {
+ coli_v4_compressor_snapshot_destroy(*output); *output = NULL; return -1;
+ }
+ memcpy((*output)->kv_state, state->kv_state,
+ (*output)->count * sizeof(float));
+ memcpy((*output)->score_state, state->score_state,
+ (*output)->count * sizeof(float));
+ return 0;
+}
+
+int coli_v4_compressor_snapshot_restore(
+ ColiDeepSeekV4CompressorState *state,
+ const ColiV4CompressorSnapshot *snapshot) {
+ if (!state || !snapshot || snapshot->count !=
+ (size_t)state->state_rows * state->projection_dim) return -1;
+ memcpy(state->kv_state, snapshot->kv_state, snapshot->count * sizeof(float));
+ memcpy(state->score_state, snapshot->score_state,
+ snapshot->count * sizeof(float));
+ return 0;
+}
+
+void coli_v4_compressor_snapshot_destroy(ColiV4CompressorSnapshot *snapshot) {
+ if (!snapshot) return;
+ free(snapshot->score_state); free(snapshot->kv_state); free(snapshot);
+}
+#endif /* COLI_V4_UNIT_COMPRESSOR_SNAPSHOT */
+
+#ifdef COLI_V4_UNIT_INDEXER_SNAPSHOT
+/* ######## deepseek_v4_indexer_snapshot.c ######## */
+#define coli_v4_indexer_create snapshot_copy_indexer_create
+#define coli_v4_indexer_bind_weights snapshot_copy_indexer_bind_weights
+#define coli_v4_indexer_reset snapshot_copy_indexer_reset
+#define coli_v4_indexer_destroy snapshot_copy_indexer_destroy
+#define coli_v4_indexer_step snapshot_copy_indexer_step
+#define coli_v4_indexer_compressed_values snapshot_copy_indexer_values
+#define coli_v4_indexer_compressed_count snapshot_copy_indexer_count
+/* ---- begin include deepseek_v4_indexer.c ---- */
+#include "deepseek_v4_internal.h"
+
+#include <math.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "native_quant.h"
+
+struct ColiDeepSeekV4Indexer {
+ const ColiDeepSeekV4LayerWeights *weights;
+ const ColiDeepSeekV4Config *config;
+ ColiDeepSeekV4CompressorState *compressor;
+ int layer;
+ int capacity;
+ int count;
+ float *compressed;
+};
+
+typedef struct { float score; int index; } IndexScore;
+
+static int set_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list arguments;
+ va_start(arguments, format);
+ vsnprintf(error, size, format, arguments);
+ va_end(arguments);
+ }
+ return -1;
+}
+
+static const void *value(const ColiDeepSeekV4LayerWeights *weights,
+ const char *suffix,
+ const ColiDeepSeekV4TensorSpec **spec) {
+ char name[COLI_V4_MAX_TENSOR_NAME];
+ snprintf(name, sizeof(name), "layers.%d.%s", weights->plan.layer, suffix);
+ return coli_v4_layer_data(weights, name, spec);
+}
+
+static int fp8_view(ColiTensorView *view,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const char *prefix) {
+ char suffix[128];
+ const ColiDeepSeekV4TensorSpec *ws = NULL, *ss = NULL;
+ snprintf(suffix, sizeof(suffix), "%s.weight", prefix);
+ const void *data = value(weights, suffix, &ws);
+ snprintf(suffix, sizeof(suffix), "%s.scale", prefix);
+ const void *scales = value(weights, suffix, &ss);
+ if (!data || !scales || !ws || !ss || ws->rank != 2 ||
+ ws->dtype != COLI_ST_F8_E4M3 || ss->dtype != COLI_ST_F8_E8M0)
+ return -1;
+ *view = (ColiTensorView){
+ COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_UE8M0, data, scales,
+ (size_t)(ws->shape[0] * ws->shape[1]),
+ (size_t)(ss->shape[0] * ss->shape[1]),
+ ws->shape[0], ws->shape[1], 128, 128
+ };
+ return 0;
+}
+
+static int descending_score(const void *left, const void *right) {
+ const IndexScore *a = left, *b = right;
+ if (a->score < b->score) return 1;
+ if (a->score > b->score) return -1;
+ return a->index - b->index;
+}
+
+int coli_v4_indexer_create(ColiDeepSeekV4Indexer **output,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ int max_context, char *error, size_t error_size) {
+ if (!output || !weights || !config || !weights->plan.has_indexer ||
+ max_context < 4 || config->index_head_dim < 1 ||
+ config->index_n_heads < 1)
+ return set_error(error, error_size, "invalid indexer options");
+ *output = NULL;
+ ColiDeepSeekV4Indexer *state = calloc(1, sizeof(*state));
+ if (!state) return set_error(error, error_size, "out of memory creating indexer");
+ state->weights = weights;
+ state->config = config;
+ state->layer = weights->plan.layer;
+ state->capacity = (max_context + 3) / 4;
+ if (state->capacity > 128) state->capacity = 128;
+ state->compressed = calloc((size_t)state->capacity * config->index_head_dim,
+ sizeof(*state->compressed));
+ ColiDeepSeekV4CompressorOptions options = {
+ "attn.indexer.compressor", config->index_head_dim, 1
+ };
+ if (!state->compressed || coli_v4_compressor_create_with_options(
+ &state->compressor, weights, config, &options, error, error_size)) {
+ coli_v4_indexer_destroy(state);
+ return set_error(error, error_size, "cannot create indexer compressor");
+ }
+ *output = state;
+ return 0;
+}
+
+int coli_v4_indexer_bind_weights(ColiDeepSeekV4Indexer *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ char *error, size_t error_size) {
+ if (!state || !weights || weights->plan.layer != state->layer ||
+ !weights->plan.has_indexer)
+ return set_error(error, error_size, "incompatible indexer weights");
+ state->weights = weights;
+ return coli_v4_compressor_bind_weights(state->compressor, weights,
+ error, error_size);
+}
+
+void coli_v4_indexer_reset(ColiDeepSeekV4Indexer *state) {
+ if (!state) return;
+ state->count = 0;
+ memset(state->compressed, 0,
+ (size_t)state->capacity * state->config->index_head_dim * sizeof(float));
+ coli_v4_compressor_reset(state->compressor);
+}
+
+void coli_v4_indexer_destroy(ColiDeepSeekV4Indexer *state) {
+ if (!state) return;
+ coli_v4_compressor_destroy(state->compressor);
+ free(state->compressed);
+ free(state);
+}
+
+static int apply_position_rope(float *queries,
+ const ColiDeepSeekV4Config *config,
+ int position) {
+ int heads = config->index_n_heads, dimension = config->index_head_dim;
+ int rope_dim = config->qk_rope_head_dim, pairs = rope_dim / 2;
+ size_t count = (size_t)(position + 1) * pairs;
+ float *cosines = malloc(count * sizeof(*cosines));
+ float *sines = malloc(count * sizeof(*sines));
+ if (!cosines || !sines || coli_v4_rope_precompute(
+ cosines, sines, rope_dim, position + 1,
+ config->original_max_position_embeddings,
+ config->compress_rope_theta, config->rope_factor,
+ config->rope_beta_fast, config->rope_beta_slow)) {
+ free(sines); free(cosines); return -1;
+ }
+ for (int head = 0; head < heads; head++) {
+ float *query = queries + (size_t)head * dimension;
+ coli_v4_rope_apply(query + dimension - rope_dim, 1, rope_dim,
+ cosines + (size_t)position * pairs,
+ sines + (size_t)position * pairs, 0);
+ coli_bf16_round_array(query + dimension - rope_dim, (size_t)rope_dim);
+ }
+ free(sines); free(cosines);
+ return 0;
+}
+
+int coli_v4_indexer_step(ColiDeepSeekV4Indexer *state, int *indices,
+ int index_capacity, const float *query_rank,
+ const float *input, int position,
+ char *error, size_t error_size) {
+ if (!state || !indices || index_capacity < 1 || !query_rank || !input ||
+ position < 0)
+ return set_error(error, error_size, "invalid indexer step arguments");
+ int dimension = state->config->index_head_dim;
+ int heads = state->config->index_n_heads;
+ int produced = 0;
+ if ((position + 1) % 4 == 0 && state->count >= state->capacity) {
+ int next_capacity = state->capacity * 2;
+ float *grown = realloc(state->compressed,
+ (size_t)next_capacity * dimension * sizeof(*grown));
+ if (!grown) return set_error(error, error_size, "cannot grow indexer cache");
+ memset(grown + (size_t)state->capacity * dimension, 0,
+ (size_t)(next_capacity - state->capacity) * dimension * sizeof(*grown));
+ state->compressed = grown;
+ state->capacity = next_capacity;
+ }
+ float *next = state->count < state->capacity
+ ? state->compressed + (size_t)state->count * dimension : NULL;
+ if (coli_v4_compressor_step(state->compressor, next, &produced, input,
+ position, error, error_size)) return -1;
+ if (produced) {
+ if (state->count >= state->capacity)
+ return set_error(error, error_size, "indexer cache capacity exceeded");
+ state->count++;
+ }
+ if (!state->count) return 0;
+
+ ColiTensorView wq;
+ if (fp8_view(&wq, state->weights, "attn.indexer.wq_b"))
+ return set_error(error, error_size, "missing indexer query weight");
+ float *queries = malloc((size_t)heads * dimension * sizeof(*queries));
+ float *head_weights = malloc((size_t)heads * sizeof(*head_weights));
+ IndexScore *scores = malloc((size_t)state->count * sizeof(*scores));
+ uint8_t *scales = malloc((size_t)dimension / 32);
+ float *qdq = malloc((size_t)dimension * sizeof(*qdq));
+ const uint16_t *raw_weights = value(
+ state->weights, "attn.indexer.weights_proj.weight", NULL);
+ if (!queries || !head_weights || !scores || !scales || !qdq || !raw_weights) {
+ free(qdq); free(scales); free(scores); free(head_weights); free(queries);
+ return set_error(error, error_size, "out of memory scoring indexer");
+ }
+ int result = coli_fp8_matvec_ref(queries, &wq, query_rank);
+ if (!result) coli_bf16_round_array(queries, (size_t)heads * dimension);
+ if (!result) result = apply_position_rope(queries, state->config, position);
+ for (int head = 0; !result && head < heads; head++) {
+ float *query = queries + (size_t)head * dimension;
+ result = coli_hadamard_bf16_ref(query, (size_t)dimension);
+ if (!result) result = coli_fp4_activation_qdq_ref(
+ qdq, scales, query, (size_t)dimension, 32);
+ if (!result) {
+ memcpy(query, qdq, (size_t)dimension * sizeof(*query));
+ coli_bf16_round_array(query, (size_t)dimension);
+ }
+ }
+ float weight_scale = 1.0f / sqrtf((float)(dimension * heads));
+ for (int head = 0; !result && head < heads; head++) {
+ float sum = 0.0f;
+ const uint16_t *row = raw_weights + (size_t)head * state->config->hidden_size;
+ for (int column = 0; column < state->config->hidden_size; column++)
+ sum += coli_bf16_decode(row[column]) * input[column];
+ head_weights[head] = sum * weight_scale;
+ }
+ for (int candidate = 0; !result && candidate < state->count; candidate++) {
+ const float *key = state->compressed + (size_t)candidate * dimension;
+ float score = 0.0f;
+ for (int head = 0; head < heads; head++) {
+ const float *query = queries + (size_t)head * dimension;
+ float dot = 0.0f;
+ for (int i = 0; i < dimension; i++) dot += query[i] * key[i];
+ score += fmaxf(dot, 0.0f) * head_weights[head];
+ }
+ scores[candidate] = (IndexScore){score, candidate};
+ }
+ if (!result) qsort(scores, (size_t)state->count, sizeof(*scores), descending_score);
+ int selected = state->count;
+ if (selected > state->config->index_topk) selected = state->config->index_topk;
+ if (selected > index_capacity) selected = index_capacity;
+ for (int i = 0; !result && i < selected; i++) indices[i] = scores[i].index;
+ free(qdq); free(scales); free(scores); free(head_weights); free(queries);
+ return result ? set_error(error, error_size, "indexer scoring failed") : selected;
+}
+
+const float *coli_v4_indexer_compressed_values(
+ const ColiDeepSeekV4Indexer *state) {
+ return state ? state->compressed : NULL;
+}
+
+int coli_v4_indexer_compressed_count(const ColiDeepSeekV4Indexer *state) {
+ return state ? state->count : 0;
+}
+/* ---- end include deepseek_v4_indexer.c ---- */
+
+#undef coli_v4_indexer_compressed_count
+#undef coli_v4_indexer_compressed_values
+#undef coli_v4_indexer_step
+#undef coli_v4_indexer_destroy
+#undef coli_v4_indexer_reset
+#undef coli_v4_indexer_bind_weights
+#undef coli_v4_indexer_create
+
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+
+struct ColiV4IndexerSnapshot {
+ int count;
+ int head_dim;
+ float *compressed;
+ ColiV4CompressorSnapshot *compressor;
+};
+
+int coli_v4_indexer_snapshot_create(const ColiDeepSeekV4Indexer *state,
+ ColiV4IndexerSnapshot **output) {
+ if (!state || !output || !state->config) return -1;
+ *output = calloc(1, sizeof(**output));
+ if (!*output) return -1;
+ (*output)->count = state->count;
+ (*output)->head_dim = state->config->index_head_dim;
+ if (state->count) {
+ (*output)->compressed = malloc((size_t)state->count *
+ (*output)->head_dim * sizeof(float));
+ if (!(*output)->compressed) {
+ coli_v4_indexer_snapshot_destroy(*output); *output = NULL; return -1;
+ }
+ memcpy((*output)->compressed, state->compressed,
+ (size_t)state->count * (*output)->head_dim * sizeof(float));
+ }
+ if (coli_v4_compressor_snapshot_create(state->compressor,
+ &(*output)->compressor)) {
+ coli_v4_indexer_snapshot_destroy(*output); *output = NULL; return -1;
+ }
+ return 0;
+}
+
+int coli_v4_indexer_snapshot_restore(ColiDeepSeekV4Indexer *state,
+ const ColiV4IndexerSnapshot *snapshot) {
+ if (!state || !snapshot || !state->config ||
+ state->config->index_head_dim != snapshot->head_dim ||
+ snapshot->count > state->capacity) return -1;
+ state->count = snapshot->count;
+ if (snapshot->count)
+ memcpy(state->compressed, snapshot->compressed,
+ (size_t)snapshot->count * snapshot->head_dim * sizeof(float));
+ return coli_v4_compressor_snapshot_restore(state->compressor,
+ snapshot->compressor);
+}
+
+void coli_v4_indexer_snapshot_destroy(ColiV4IndexerSnapshot *snapshot) {
+ if (!snapshot) return;
+ coli_v4_compressor_snapshot_destroy(snapshot->compressor);
+ free(snapshot->compressed); free(snapshot);
+}
+#endif /* COLI_V4_UNIT_INDEXER_SNAPSHOT */
+
+#ifdef COLI_V4_UNIT_ATTENTION_TRANSACTION
+/* ######## deepseek_v4_attention_transaction.c ######## */
+#define coli_v4_window_attention_create transaction_copy_attention_create
+#define coli_v4_window_attention_reset transaction_copy_attention_reset
+#define coli_v4_window_attention_destroy transaction_copy_attention_destroy
+#define coli_v4_attention_token_ref transaction_copy_attention_token
+#define coli_v4_attention_window_token_ref transaction_copy_attention_window_token
+/* ---- begin include deepseek_v4_attention.c ---- */
+#include "deepseek_v4_internal.h"
+
+#include <math.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "native_quant.h"
+
+static int set_error(char *error, size_t size, const char *format, ...);
+
+struct ColiDeepSeekV4WindowAttentionState {
+ int window_size;
+ int head_dim;
+ int layer;
+ int ratio;
+ float *kv;
+ ColiDeepSeekV4CompressorState *compressor;
+ ColiDeepSeekV4Indexer *indexer;
+ float *compressed;
+ int compressed_count;
+ int compressed_capacity;
+};
+
+int coli_v4_window_attention_create(ColiDeepSeekV4WindowAttentionState **output,
+ const ColiDeepSeekV4Config *config) {
+ if (!output || !config || config->sliding_window < 1 || config->head_dim < 1)
+ return -1;
+ *output = calloc(1, sizeof(**output));
+ if (!*output) return -1;
+ (*output)->window_size = config->sliding_window;
+ (*output)->head_dim = config->head_dim;
+ (*output)->layer = -1;
+ (*output)->kv = calloc((size_t)config->sliding_window * config->head_dim,
+ sizeof(*(*output)->kv));
+ if (!(*output)->kv) {
+ free(*output);
+ *output = NULL;
+ return -1;
+ }
+ return 0;
+}
+
+void coli_v4_window_attention_reset(ColiDeepSeekV4WindowAttentionState *state) {
+ if (!state) return;
+ memset(state->kv, 0,
+ (size_t)state->window_size * state->head_dim * sizeof(*state->kv));
+ state->compressed_count = 0;
+ if (state->compressor) coli_v4_compressor_reset(state->compressor);
+ if (state->indexer) coli_v4_indexer_reset(state->indexer);
+}
+
+void coli_v4_window_attention_destroy(ColiDeepSeekV4WindowAttentionState *state) {
+ if (!state) return;
+ coli_v4_indexer_destroy(state->indexer);
+ coli_v4_compressor_destroy(state->compressor);
+ free(state->compressed);
+ free(state->kv);
+ free(state);
+}
+
+static int prepare_compressed_state(
+ ColiDeepSeekV4WindowAttentionState *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, char *error, size_t error_size) {
+ int ratio = weights->plan.compression_ratio;
+ if (!ratio) return 0;
+ if (state->layer < 0) {
+ state->layer = weights->plan.layer;
+ state->ratio = ratio;
+ state->compressed_capacity = 16;
+ state->compressed = calloc((size_t)state->compressed_capacity * state->head_dim,
+ sizeof(*state->compressed));
+ if (!state->compressed || coli_v4_compressor_create(
+ &state->compressor, weights, config, error, error_size)) return -1;
+ if (ratio == 4 && coli_v4_indexer_create(
+ &state->indexer, weights, config, config->max_position_embeddings,
+ error, error_size)) return -1;
+ } else if (state->layer != weights->plan.layer || state->ratio != ratio) {
+ return set_error(error, error_size, "attention state belongs to another layer");
+ }
+ if (coli_v4_compressor_bind_weights(state->compressor, weights,
+ error, error_size)) return -1;
+ if (state->indexer && coli_v4_indexer_bind_weights(
+ state->indexer, weights, error, error_size)) return -1;
+ return 0;
+}
+
+static int grow_compressed_state(ColiDeepSeekV4WindowAttentionState *state,
+ char *error, size_t error_size) {
+ if (state->compressed_count < state->compressed_capacity) return 0;
+ int capacity = state->compressed_capacity * 2;
+ float *grown = realloc(state->compressed,
+ (size_t)capacity * state->head_dim * sizeof(*grown));
+ if (!grown) return set_error(error, error_size, "cannot grow compressed KV cache");
+ state->compressed = grown;
+ state->compressed_capacity = capacity;
+ return 0;
+}
+
+static int set_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list arguments;
+ va_start(arguments, format);
+ vsnprintf(error, size, format, arguments);
+ va_end(arguments);
+ }
+ return -1;
+}
+
+static const void *layer_data(const ColiDeepSeekV4LayerWeights *weights,
+ const char *suffix,
+ const ColiDeepSeekV4TensorSpec **spec) {
+ char name[COLI_V4_MAX_TENSOR_NAME];
+ snprintf(name, sizeof(name), "layers.%d.%s", weights->plan.layer, suffix);
+ return coli_v4_layer_data(weights, name, spec);
+}
+
+static int fp8_view(ColiTensorView *view,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const char *prefix) {
+ char suffix[128];
+ const ColiDeepSeekV4TensorSpec *weight_spec = NULL, *scale_spec = NULL;
+ snprintf(suffix, sizeof(suffix), "%s.weight", prefix);
+ const void *data = layer_data(weights, suffix, &weight_spec);
+ snprintf(suffix, sizeof(suffix), "%s.scale", prefix);
+ const void *scales = layer_data(weights, suffix, &scale_spec);
+ if (!data || !scales || !weight_spec || !scale_spec ||
+ weight_spec->dtype != COLI_ST_F8_E4M3 ||
+ scale_spec->dtype != COLI_ST_F8_E8M0 || weight_spec->rank != 2)
+ return -1;
+ *view = (ColiTensorView){
+ COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_UE8M0, data, scales,
+ (size_t)(weight_spec->shape[0] * weight_spec->shape[1]),
+ (size_t)(scale_spec->shape[0] * scale_spec->shape[1]),
+ weight_spec->shape[0], weight_spec->shape[1], 128, 128
+ };
+ return 0;
+}
+
+static int decode_bf16(float *output, const void *data, size_t count) {
+ if (!output || !data) return -1;
+ const uint16_t *values = data;
+ for (size_t i = 0; i < count; i++) output[i] = coli_bf16_decode(values[i]);
+ return 0;
+}
+
+static int attention_token_impl(float *output,
+ ColiDeepSeekV4WindowAttentionState *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const float *input, int position,
+ char *error, size_t error_size) {
+ if (!output || !weights || !config || !input || position < 0 ||
+ (!state && weights->plan.compression_ratio != 0 && position != 0))
+ return set_error(error, error_size, "invalid uncompressed attention arguments");
+ int hidden = config->hidden_size;
+ int heads = config->num_attention_heads;
+ int head_dim = config->head_dim;
+ int rope_dim = config->qk_rope_head_dim;
+ int q_rank = config->q_lora_rank;
+ int groups = config->o_groups;
+ int o_rank = config->o_lora_rank;
+ if (hidden < 1 || heads < 1 || head_dim < 1 || rope_dim < 2 ||
+ rope_dim > head_dim || q_rank < 1 || groups < 1 || heads % groups)
+ return set_error(error, error_size, "unsupported attention dimensions");
+
+ ColiTensorView wq_a, wq_b, wkv, wo_a, wo_b;
+ if (fp8_view(&wq_a, weights, "attn.wq_a") ||
+ fp8_view(&wq_b, weights, "attn.wq_b") ||
+ fp8_view(&wkv, weights, "attn.wkv") ||
+ fp8_view(&wo_a, weights, "attn.wo_a") ||
+ fp8_view(&wo_b, weights, "attn.wo_b"))
+ return set_error(error, error_size, "missing native FP8 attention tensor");
+
+ float *qa = calloc((size_t)q_rank, sizeof(*qa));
+ float *q = calloc((size_t)heads * head_dim, sizeof(*q));
+ float *kv = calloc((size_t)head_dim, sizeof(*kv));
+ float *attended = calloc((size_t)heads * head_dim, sizeof(*attended));
+ float *oa = calloc((size_t)groups * o_rank, sizeof(*oa));
+ float *norm_weight = calloc((size_t)(q_rank > head_dim ? q_rank : head_dim),
+ sizeof(*norm_weight));
+ float *cosines = calloc((size_t)rope_dim / 2, sizeof(*cosines));
+ float *sines = calloc((size_t)rope_dim / 2, sizeof(*sines));
+ int *compressed_indices = NULL;
+ int compressed_selected = 0;
+ if (!qa || !q || !kv || !attended || !oa || !norm_weight || !cosines || !sines) {
+ free(sines); free(cosines); free(norm_weight); free(oa);
+ free(attended); free(kv); free(q); free(qa);
+ return set_error(error, error_size, "out of memory in attention");
+ }
+
+ int result = coli_fp8_matvec_ref(qa, &wq_a, input);
+ coli_bf16_round_array(qa, (size_t)q_rank);
+ const void *q_norm = layer_data(weights, "attn.q_norm.weight", NULL);
+ if (!result && (!q_norm || decode_bf16(norm_weight, q_norm, (size_t)q_rank) ||
+ coli_v4_rmsnorm(qa, qa, norm_weight, q_rank,
+ config->rms_norm_eps))) result = -1;
+ if (!result) coli_bf16_round_array(qa, (size_t)q_rank);
+ if (!result && state && weights->plan.compression_ratio) {
+ result = prepare_compressed_state(state, weights, config,
+ error, error_size);
+ if (!result && (position + 1) % state->ratio == 0)
+ result = grow_compressed_state(state, error, error_size);
+ int produced = 0;
+ if (!result) result = coli_v4_compressor_step(
+ state->compressor,
+ state->compressed + (size_t)state->compressed_count * head_dim,
+ &produced, input, position, error, error_size);
+ if (!result && produced) state->compressed_count++;
+ if (!result && state->indexer) {
+ compressed_indices = malloc((size_t)config->index_topk *
+ sizeof(*compressed_indices));
+ if (!compressed_indices) result = -1;
+ else compressed_selected = coli_v4_indexer_step(
+ state->indexer, compressed_indices, config->index_topk,
+ qa, input, position, error, error_size);
+ if (compressed_selected < 0) result = -1;
+ }
+ }
+ if (!result) result = coli_fp8_matvec_ref(q, &wq_b, qa);
+ if (!result) coli_bf16_round_array(q, (size_t)heads * head_dim);
+ for (int head = 0; !result && head < heads; head++) {
+ float *values = q + (size_t)head * head_dim;
+ float mean_square = 0.0f;
+ for (int i = 0; i < head_dim; i++) mean_square += values[i] * values[i];
+ float scale = 1.0f / sqrtf(mean_square / head_dim + config->rms_norm_eps);
+ for (int i = 0; i < head_dim; i++) values[i] = coli_bf16_round(values[i] * scale);
+ }
+
+ if (!result) result = coli_fp8_matvec_ref(kv, &wkv, input);
+ if (!result) coli_bf16_round_array(kv, (size_t)head_dim);
+ const void *kv_norm = layer_data(weights, "attn.kv_norm.weight", NULL);
+ if (!result && (!kv_norm || decode_bf16(norm_weight, kv_norm, (size_t)head_dim) ||
+ coli_v4_rmsnorm(kv, kv, norm_weight, head_dim,
+ config->rms_norm_eps))) result = -1;
+ if (!result) coli_bf16_round_array(kv, (size_t)head_dim);
+
+ if (!result) {
+ float *all_cos = calloc((size_t)(position + 1) * rope_dim / 2, sizeof(*all_cos));
+ float *all_sin = calloc((size_t)(position + 1) * rope_dim / 2, sizeof(*all_sin));
+ int compressed = weights->plan.compression_ratio != 0;
+ if (!all_cos || !all_sin || coli_v4_rope_precompute(
+ all_cos, all_sin, rope_dim, position + 1,
+ compressed ? config->original_max_position_embeddings : 0,
+ compressed ? config->compress_rope_theta : config->rope_theta,
+ config->rope_factor,
+ config->rope_beta_fast, config->rope_beta_slow)) result = -1;
+ if (!result) {
+ memcpy(cosines, all_cos + (size_t)position * rope_dim / 2,
+ (size_t)rope_dim / 2 * sizeof(*cosines));
+ memcpy(sines, all_sin + (size_t)position * rope_dim / 2,
+ (size_t)rope_dim / 2 * sizeof(*sines));
+ }
+ free(all_sin); free(all_cos);
+ }
+ if (!result) {
+ for (int head = 0; head < heads; head++) {
+ float *rope = q + (size_t)head * head_dim + head_dim - rope_dim;
+ coli_v4_rope_apply(rope, 1, rope_dim, cosines, sines, 0);
+ coli_bf16_round_array(rope, (size_t)rope_dim);
+ }
+ float *kv_rope = kv + head_dim - rope_dim;
+ coli_v4_rope_apply(kv_rope, 1, rope_dim, cosines, sines, 0);
+ coli_bf16_round_array(kv_rope, (size_t)rope_dim);
+ size_t nope = (size_t)(head_dim - rope_dim);
+ float *qdq = malloc(nope * sizeof(*qdq));
+ uint8_t *scales = malloc((nope + 63) / 64);
+ if (!qdq || !scales || coli_fp8_activation_qdq_ref(qdq, scales, kv, nope, 64))
+ result = -1;
+ if (!result) {
+ memcpy(kv, qdq, nope * sizeof(*kv));
+ coli_bf16_round_array(kv, nope);
+ }
+ free(scales); free(qdq);
+ }
+
+ const float *sinks = layer_data(weights, "attn.attn_sink", NULL);
+ if (!result && state) {
+ int slot = position % state->window_size;
+ memcpy(state->kv + (size_t)slot * head_dim, kv,
+ (size_t)head_dim * sizeof(*kv));
+ if (!state->indexer) compressed_selected = state->compressed_count;
+ int topk = state->window_size + compressed_selected;
+ int kv_count = state->window_size + state->compressed_count;
+ int *indices = malloc((size_t)topk * sizeof(*indices));
+ float *all_kv = state->compressed_count
+ ? malloc((size_t)kv_count * head_dim * sizeof(*all_kv)) : NULL;
+ if (!indices || (state->compressed_count && !all_kv)) result = -1;
+ if (!result) {
+ if (position < state->window_size - 1) {
+ for (int i = 0; i < state->window_size; i++)
+ indices[i] = i <= position ? i : -1;
+ } else {
+ int oldest = (position + 1) % state->window_size;
+ for (int i = 0; i < state->window_size; i++)
+ indices[i] = (oldest + i) % state->window_size;
+ }
+ const float *kv_values = state->kv;
+ if (state->compressed_count) {
+ memcpy(all_kv, state->kv,
+ (size_t)state->window_size * head_dim * sizeof(*all_kv));
+ memcpy(all_kv + (size_t)state->window_size * head_dim,
+ state->compressed,
+ (size_t)state->compressed_count * head_dim * sizeof(*all_kv));
+ kv_values = all_kv;
+ }
+ for (int i = 0; i < compressed_selected; i++) {
+ int ordinal = state->indexer ? compressed_indices[i] : i;
+ indices[state->window_size + i] = state->window_size + ordinal;
+ }
+ result = coli_v4_sparse_attention_ref(
+ attended, q, kv_values, sinks, indices, heads, head_dim,
+ kv_count, topk,
+ 1.0f / sqrtf((float)head_dim));
+ }
+ free(all_kv);
+ free(indices);
+ } else for (int head = 0; !result && head < heads; head++) {
+ float *query = q + (size_t)head * head_dim;
+ float score = 0.0f;
+ for (int i = 0; i < head_dim; i++) score += query[i] * kv[i];
+ score *= 1.0f / sqrtf((float)head_dim);
+ float attention_weight = 1.0f / (1.0f + expf(sinks[head] - score));
+ float *head_output = attended + (size_t)head * head_dim;
+ for (int i = 0; i < head_dim; i++)
+ head_output[i] = coli_bf16_round(kv[i] * attention_weight);
+ }
+ for (int head = 0; !result && head < heads; head++) {
+ float *head_output = attended + (size_t)head * head_dim;
+ float *rope = head_output + head_dim - rope_dim;
+ coli_v4_rope_apply(rope, 1, rope_dim, cosines, sines, 1);
+ coli_bf16_round_array(rope, (size_t)rope_dim);
+ }
+
+ int heads_per_group = heads / groups;
+ int group_width = heads_per_group * head_dim;
+ int scale_columns = (hidden + 127) / 128;
+ int scale_rows_per_group = (o_rank + 127) / 128;
+ for (int group = 0; !result && group < groups; group++) {
+ ColiTensorView group_view = wo_a;
+ group_view.rows = o_rank;
+ group_view.columns = group_width;
+ group_view.data = (const uint8_t *)wo_a.data +
+ (size_t)group * o_rank * group_width;
+ group_view.scales = (const uint8_t *)wo_a.scales +
+ (size_t)group * scale_rows_per_group * scale_columns;
+ group_view.data_bytes = (size_t)o_rank * group_width;
+ group_view.scale_bytes = (size_t)scale_rows_per_group * scale_columns;
+ result = coli_fp8_matvec_ref(oa + (size_t)group * o_rank, &group_view,
+ attended + (size_t)group * group_width);
+ }
+ if (!result) coli_bf16_round_array(oa, (size_t)groups * o_rank);
+ if (!result) result = coli_fp8_matvec_ref(output, &wo_b, oa);
+ if (!result) coli_bf16_round_array(output, (size_t)hidden);
+
+ free(compressed_indices);
+ free(sines); free(cosines); free(norm_weight); free(oa);
+ free(attended); free(kv); free(q); free(qa);
+ if (result) return set_error(error, error_size, "attention computation failed");
+ return 0;
+}
+
+int coli_v4_attention_token_ref(float *output,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const float *input, int position,
+ char *error, size_t error_size) {
+ return attention_token_impl(output, NULL, weights, config, input, position,
+ error, error_size);
+}
+
+int coli_v4_attention_window_token_ref(
+ float *output, ColiDeepSeekV4WindowAttentionState *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, const float *input, int position,
+ char *error, size_t error_size) {
+ return attention_token_impl(output, state, weights, config, input, position,
+ error, error_size);
+}
+/* ---- end include deepseek_v4_attention.c ---- */
+
+#undef coli_v4_attention_window_token_ref
+#undef coli_v4_attention_token_ref
+#undef coli_v4_window_attention_destroy
+#undef coli_v4_window_attention_reset
+#undef coli_v4_window_attention_create
+
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+
+struct ColiV4AttentionSnapshot {
+ int window_size, head_dim, compressed_count;
+ float *kv;
+ float *compressed;
+ ColiV4CompressorSnapshot *compressor;
+ ColiV4IndexerSnapshot *indexer;
+};
+
+int coli_v4_attention_snapshot_create(
+ const ColiDeepSeekV4WindowAttentionState *state,
+ ColiV4AttentionSnapshot **output) {
+ if (!state || !output) return -1;
+ *output = calloc(1, sizeof(**output));
+ if (!*output) return -1;
+ (*output)->window_size = state->window_size;
+ (*output)->head_dim = state->head_dim;
+ (*output)->compressed_count = state->compressed_count;
+ size_t kv_count = (size_t)state->window_size * state->head_dim;
+ (*output)->kv = malloc(kv_count * sizeof(float));
+ if (!(*output)->kv) goto failed;
+ memcpy((*output)->kv, state->kv, kv_count * sizeof(float));
+ if (state->compressed_count) {
+ size_t count = (size_t)state->compressed_count * state->head_dim;
+ (*output)->compressed = malloc(count * sizeof(float));
+ if (!(*output)->compressed) goto failed;
+ memcpy((*output)->compressed, state->compressed, count * sizeof(float));
+ }
+ if (state->compressor && coli_v4_compressor_snapshot_create(
+ state->compressor, &(*output)->compressor)) goto failed;
+ if (state->indexer && coli_v4_indexer_snapshot_create(
+ state->indexer, &(*output)->indexer)) goto failed;
+ return 0;
+failed:
+ coli_v4_attention_snapshot_destroy(*output); *output = NULL; return -1;
+}
+
+int coli_v4_attention_snapshot_restore(
+ ColiDeepSeekV4WindowAttentionState *state,
+ const ColiV4AttentionSnapshot *snapshot) {
+ if (!state || !snapshot || state->window_size != snapshot->window_size ||
+ state->head_dim != snapshot->head_dim ||
+ snapshot->compressed_count > state->compressed_capacity) return -1;
+ memcpy(state->kv, snapshot->kv,
+ (size_t)state->window_size * state->head_dim * sizeof(float));
+ state->compressed_count = snapshot->compressed_count;
+ if (snapshot->compressed_count)
+ memcpy(state->compressed, snapshot->compressed,
+ (size_t)snapshot->compressed_count * state->head_dim * sizeof(float));
+ if ((state->compressor != NULL) != (snapshot->compressor != NULL) ||
+ (state->indexer != NULL) != (snapshot->indexer != NULL)) return -1;
+ if (state->compressor && coli_v4_compressor_snapshot_restore(
+ state->compressor, snapshot->compressor)) return -1;
+ if (state->indexer && coli_v4_indexer_snapshot_restore(
+ state->indexer, snapshot->indexer)) return -1;
+ return 0;
+}
+
+void coli_v4_attention_snapshot_destroy(ColiV4AttentionSnapshot *snapshot) {
+ if (!snapshot) return;
+ coli_v4_indexer_snapshot_destroy(snapshot->indexer);
+ coli_v4_compressor_snapshot_destroy(snapshot->compressor);
+ free(snapshot->compressed); free(snapshot->kv); free(snapshot);
+}
+#endif /* COLI_V4_UNIT_ATTENTION_TRANSACTION */
+
+#ifdef COLI_V4_UNIT_EXPERT_STORE_HOT_ROWS16
+/* ######## deepseek_v4_expert_store_hot_rows16.c ######## */
+/* Hot target experts are converted in-place to a 16-row resident layout.
+ * Cold experts keep the official row-major FP4 representation. */
+#define coli_deepseek_v4_expert_store_open \
+ coli_deepseek_v4_expert_store_open_base
+/* ---- begin include deepseek_v4_expert_store.c ---- */
+#define _GNU_SOURCE
+#include "deepseek_v4_internal.h"
+
+#include <assert.h>
+#include <pthread.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "safetensors_index.h"
+
+#ifdef COLI_V4_EXPERIMENTAL_PREFETCH_BATCH
+int coli_st_prefetch_many(
+ const ColiSafetensorsIndex *index, const int *shards,
+ const uint64_t *offsets, const size_t *lengths, size_t count);
+#endif
+
+enum { V4_W1 = 0, V4_W2 = 1, V4_W3 = 2, V4_MATRIX_COUNT = 3 };
+
+typedef struct {
+ const ColiSafetensorsTensor *weight[V4_MATRIX_COUNT];
+ const ColiSafetensorsTensor *scale[V4_MATRIX_COUNT];
+ int shard;
+ uint64_t scale_offset;
+ uint64_t scale_bytes;
+ uint64_t weight_offset;
+ uint64_t weight_bytes;
+ uint64_t record_bytes;
+} V4ExpertRecord;
+
+typedef struct {
+ int expert;
+ unsigned references;
+ uint64_t used;
+ unsigned char *slab;
+} V4ExpertSlot;
+
+typedef struct {
+ ColiSafetensorsIndex *index;
+ int layers;
+ int experts_per_layer;
+ int slots_per_layer;
+ uint64_t record_bytes;
+ V4ExpertRecord *records;
+ V4ExpertSlot *slots;
+ uint64_t clock;
+ unsigned active_leases;
+ ColiExpertStoreStats stats;
+ pthread_mutex_t mutex;
+} V4ExpertStoreState;
+
+static int set_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list args;
+ va_start(args, format);
+ vsnprintf(error, size, format, args);
+ va_end(args);
+ }
+ return -1;
+}
+
+static int compare_tensors(const void *left, const void *right) {
+ const ColiSafetensorsTensor *const *a = left;
+ const ColiSafetensorsTensor *const *b = right;
+ return ((*a)->offset > (*b)->offset) - ((*a)->offset < (*b)->offset);
+}
+
+static int contiguous_group(const ColiSafetensorsTensor *const input[3],
+ int *shard, uint64_t *offset, uint64_t *bytes) {
+ const ColiSafetensorsTensor *parts[3] = {input[0], input[1], input[2]};
+ qsort(parts, 3, sizeof(parts[0]), compare_tensors);
+ if (parts[0]->shard != parts[1]->shard || parts[1]->shard != parts[2]->shard ||
+ parts[0]->offset + parts[0]->nbytes != parts[1]->offset ||
+ parts[1]->offset + parts[1]->nbytes != parts[2]->offset)
+ return -1;
+ *shard = parts[0]->shard;
+ *offset = parts[0]->offset;
+ *bytes = parts[2]->offset + parts[2]->nbytes - parts[0]->offset;
+ return 0;
+}
+
+static int validate_matrix(const ColiSafetensorsTensor *weight,
+ const ColiSafetensorsTensor *scale) {
+ if (!weight || !scale || weight->dtype != COLI_ST_I8 ||
+ scale->dtype != COLI_ST_F8_E8M0 || weight->rank != 2 || scale->rank != 2 ||
+ weight->shape[0] != scale->shape[0] || weight->shape[1] <= 0 ||
+ scale->shape[1] <= 0)
+ return -1;
+ int64_t logical_columns = weight->shape[1] * 2;
+ return scale->shape[1] * 32 == logical_columns ? 0 : -1;
+}
+
+static int build_record(V4ExpertStoreState *state, int layer, int expert,
+ V4ExpertRecord *record, char *error, size_t error_size) {
+ static const char *matrix_names[V4_MATRIX_COUNT] = {"w1", "w2", "w3"};
+ char name[160];
+ memset(record, 0, sizeof(*record));
+ for (int matrix = 0; matrix < V4_MATRIX_COUNT; matrix++) {
+ snprintf(name, sizeof(name), "layers.%d.ffn.experts.%d.%s.weight",
+ layer, expert, matrix_names[matrix]);
+ record->weight[matrix] = coli_st_find(state->index, name);
+ snprintf(name, sizeof(name), "layers.%d.ffn.experts.%d.%s.scale",
+ layer, expert, matrix_names[matrix]);
+ record->scale[matrix] = coli_st_find(state->index, name);
+ if (validate_matrix(record->weight[matrix], record->scale[matrix]) != 0)
+ return set_error(error, error_size,
+ "invalid native FP4 expert matrix: layer=%d expert=%d %s",
+ layer, expert, matrix_names[matrix]);
+ }
+ int scale_shard = -1, weight_shard = -1;
+ if (contiguous_group(record->scale, &scale_shard, &record->scale_offset,
+ &record->scale_bytes) != 0 ||
+ contiguous_group(record->weight, &weight_shard, &record->weight_offset,
+ &record->weight_bytes) != 0 || scale_shard != weight_shard)
+ return set_error(error, error_size,
+ "expert is not two contiguous ranges: layer=%d expert=%d",
+ layer, expert);
+ record->shard = scale_shard;
+ record->record_bytes = record->scale_bytes + record->weight_bytes;
+ return 0;
+}
+
+static V4ExpertRecord *get_record(V4ExpertStoreState *state, ColiExpertKey key) {
+ if (key.layer < 0 || key.layer >= state->layers || key.expert < 0 ||
+ key.expert >= state->experts_per_layer)
+ return NULL;
+ return &state->records[(size_t)key.layer * state->experts_per_layer + key.expert];
+}
+
+static V4ExpertSlot *layer_slots(V4ExpertStoreState *state, int layer) {
+ return state->slots + (size_t)layer * state->slots_per_layer;
+}
+
+static void fill_tensor_view(ColiTensorView *view,
+ const V4ExpertRecord *record,
+ const V4ExpertSlot *slot, int matrix) {
+ const ColiSafetensorsTensor *weight = record->weight[matrix];
+ const ColiSafetensorsTensor *scale = record->scale[matrix];
+ memset(view, 0, sizeof(*view));
+ view->format = COLI_TENSOR_FP4_NATIVE_BLOCK;
+ view->scale_format = COLI_SCALE_UE8M0;
+ view->data = slot->slab + record->scale_bytes +
+ (weight->offset - record->weight_offset);
+ view->scales = slot->slab + (scale->offset - record->scale_offset);
+ view->data_bytes = (size_t)weight->nbytes;
+ view->scale_bytes = (size_t)scale->nbytes;
+ view->rows = weight->shape[0];
+ view->columns = weight->shape[1] * 2;
+ view->block_rows = 1;
+ view->block_columns = 32;
+}
+
+static int lookup(ColiExpertStore *store, ColiExpertKey key,
+ ColiExpertView *view) {
+ if (!store || !store->state || !view) {
+ if (view) memset(view, 0, sizeof(*view));
+ return -1;
+ }
+ V4ExpertStoreState *state = store->state;
+ V4ExpertRecord *record = get_record(state, key);
+ if (!record) {
+ memset(view, 0, sizeof(*view));
+ return -1;
+ }
+ pthread_mutex_lock(&state->mutex);
+ state->stats.requests++;
+ V4ExpertSlot *slots = layer_slots(state, key.layer);
+ V4ExpertSlot *slot = NULL;
+ for (int i = 0; i < state->slots_per_layer; i++) {
+ if (slots[i].slab && slots[i].expert == key.expert) {
+ slot = &slots[i];
+ state->stats.hits++;
+ break;
+ }
+ }
+ if (!slot) {
+ for (int i = 0; i < state->slots_per_layer; i++) {
+ if (!slots[i].references && (!slot || !slots[i].slab ||
+ (slot->slab && slots[i].used < slot->used)))
+ slot = &slots[i];
+ }
+ if (!slot) {
+ pthread_mutex_unlock(&state->mutex);
+ memset(view, 0, sizeof(*view));
+ return -1;
+ }
+ if (!slot->slab) {
+ slot->slab = malloc((size_t)state->record_bytes);
+ if (!slot->slab) {
+ pthread_mutex_unlock(&state->mutex);
+ memset(view, 0, sizeof(*view));
+ return -1;
+ }
+ state->stats.resident_bytes += state->record_bytes;
+ }
+ /* A short read must never expose a partially overwritten old slot. */
+ slot->expert = -1;
+ if (coli_st_read_at(state->index, record->shard, record->scale_offset,
+ (size_t)record->scale_bytes, slot->slab) != 0 ||
+ coli_st_read_at(state->index, record->shard, record->weight_offset,
+ (size_t)record->weight_bytes,
+ slot->slab + record->scale_bytes) != 0) {
+ pthread_mutex_unlock(&state->mutex);
+ memset(view, 0, sizeof(*view));
+ return -1;
+ }
+ slot->expert = key.expert;
+ state->stats.misses++;
+ state->stats.bytes_read += record->record_bytes;
+ }
+ slot->references++;
+ state->active_leases++;
+ slot->used = ++state->clock;
+ memset(view, 0, sizeof(*view));
+ view->key = key;
+ fill_tensor_view(&view->gate, record, slot, V4_W1);
+ fill_tensor_view(&view->down, record, slot, V4_W2);
+ fill_tensor_view(&view->up, record, slot, V4_W3);
+ view->lease = slot;
+ pthread_mutex_unlock(&state->mutex);
+ return 0;
+}
+
+static void release(ColiExpertStore *store, ColiExpertView *view) {
+ if (!store || !store->state || !view || !view->lease) {
+ if (view) memset(view, 0, sizeof(*view));
+ return;
+ }
+ V4ExpertStoreState *state = store->state;
+ V4ExpertSlot *slot = view->lease;
+ pthread_mutex_lock(&state->mutex);
+ if (slot->references) slot->references--;
+ if (state->active_leases) state->active_leases--;
+ pthread_mutex_unlock(&state->mutex);
+ memset(view, 0, sizeof(*view));
+}
+
+static int prefetch(ColiExpertStore *store, const ColiExpertKey *keys,
+ size_t count) {
+ if (!store || !store->state || (!keys && count)) return 0;
+ V4ExpertStoreState *state = store->state;
+ int accepted = 0;
+#ifdef COLI_V4_EXPERIMENTAL_PREFETCH_BATCH
+ size_t capacity = count * 2, ranges = 0;
+ int *shards = malloc(capacity * sizeof(*shards));
+ uint64_t *offsets = malloc(capacity * sizeof(*offsets));
+ size_t *lengths = malloc(capacity * sizeof(*lengths));
+ int candidates = 0;
+ if ((!shards || !offsets || !lengths) && capacity) {
+ free(lengths); free(offsets); free(shards); return 0;
+ }
+ pthread_mutex_lock(&state->mutex);
+ for (size_t i = 0; i < count; i++) {
+ V4ExpertRecord *record = get_record(state, keys[i]);
+ if (!record) continue;
+ int resident = 0;
+ V4ExpertSlot *slots = layer_slots(state, keys[i].layer);
+ for (int slot = 0; slot < state->slots_per_layer; slot++)
+ if (slots[slot].slab && slots[slot].expert == keys[i].expert) {
+ resident = 1; break;
+ }
+ if (resident) continue;
+ shards[ranges] = record->shard;
+ offsets[ranges] = record->scale_offset;
+ lengths[ranges++] = (size_t)record->scale_bytes;
+ shards[ranges] = record->shard;
+ offsets[ranges] = record->weight_offset;
+ lengths[ranges++] = (size_t)record->weight_bytes;
+ candidates++;
+ }
+ pthread_mutex_unlock(&state->mutex);
+ if (candidates && !coli_st_prefetch_many(
+ state->index, shards, offsets, lengths, ranges))
+ accepted = candidates;
+ free(lengths); free(offsets); free(shards);
+#else
+ for (size_t i = 0; i < count; i++) {
+ V4ExpertRecord *record = get_record(state, keys[i]);
+ if (!record) continue;
+ if (coli_st_prefetch_at(state->index, record->shard, record->scale_offset,
+ (size_t)record->scale_bytes) == 0 &&
+ coli_st_prefetch_at(state->index, record->shard, record->weight_offset,
+ (size_t)record->weight_bytes) == 0)
+ accepted++;
+ }
+#endif
+ pthread_mutex_lock(&state->mutex);
+ state->stats.prefetched += (uint64_t)accepted;
+ pthread_mutex_unlock(&state->mutex);
+ return accepted;
+}
+
+static void stats(const ColiExpertStore *store, ColiExpertStoreStats *output) {
+ if (!store || !store->state || !output) return;
+ V4ExpertStoreState *state = store->state;
+ pthread_mutex_lock(&state->mutex);
+ *output = state->stats;
+ pthread_mutex_unlock(&state->mutex);
+}
+
+static void destroy(ColiExpertStore *store) {
+ if (!store) return;
+ V4ExpertStoreState *state = store->state;
+ if (state) {
+ assert(state->active_leases == 0 && "destroy with active expert leases");
+ for (int i = 0; i < state->layers * state->slots_per_layer; i++)
+ free(state->slots[i].slab);
+ pthread_mutex_destroy(&state->mutex);
+ coli_st_index_close(state->index);
+ free(state->records);
+ free(state->slots);
+ free(state);
+ }
+ free(store);
+}
+
+int coli_deepseek_v4_expert_store_open(
+ const ColiDeepSeekV4ExpertStoreOptions *options, ColiExpertStore **output,
+ char *error, size_t error_size) {
+ static const ColiExpertStoreOps operations = {
+ lookup, release, prefetch, stats, destroy
+ };
+ if (!options || !output || !options->model_dir || options->layers < 1 ||
+ options->experts_per_layer < 1 || !options->cache_bytes)
+ return set_error(error, error_size, "invalid DeepSeek-V4 ExpertStore options");
+ *output = NULL;
+ ColiExpertStore *store = calloc(1, sizeof(*store));
+ V4ExpertStoreState *state = calloc(1, sizeof(*state));
+ if (!store || !state) {
+ free(store);
+ free(state);
+ return set_error(error, error_size, "out of memory creating ExpertStore");
+ }
+ pthread_mutex_init(&state->mutex, NULL);
+ state->layers = options->layers;
+ state->experts_per_layer = options->experts_per_layer;
+ if (coli_st_index_open(&state->index, options->model_dir, error, error_size) != 0)
+ goto fail;
+ size_t record_count = (size_t)state->layers * state->experts_per_layer;
+ state->records = calloc(record_count, sizeof(*state->records));
+ if (!state->records) {
+ set_error(error, error_size, "out of memory creating expert manifest");
+ goto fail;
+ }
+ for (int layer = 0; layer < state->layers; layer++) {
+ for (int expert = 0; expert < state->experts_per_layer; expert++) {
+ V4ExpertRecord *record = &state->records[
+ (size_t)layer * state->experts_per_layer + expert];
+ if (build_record(state, layer, expert, record, error, error_size) != 0)
+ goto fail;
+ if (!state->record_bytes) state->record_bytes = record->record_bytes;
+ if (record->record_bytes != state->record_bytes) {
+ set_error(error, error_size, "non-uniform expert size at layer=%d expert=%d",
+ layer, expert);
+ goto fail;
+ }
+ }
+ }
+ state->slots_per_layer = (int)(options->cache_bytes /
+ ((uint64_t)state->layers * state->record_bytes));
+ int minimum_slots = state->experts_per_layer < 6
+ ? state->experts_per_layer : 6;
+ if (state->slots_per_layer < minimum_slots) {
+ set_error(error, error_size,
+ "cache budget cannot hold %d active experts per layer "
+ "(need %llu bytes)", minimum_slots,
+ (unsigned long long)((uint64_t)state->layers * minimum_slots *
+ state->record_bytes));
+ goto fail;
+ }
+ if (state->slots_per_layer > state->experts_per_layer)
+ state->slots_per_layer = state->experts_per_layer;
+ state->slots = calloc((size_t)state->layers * state->slots_per_layer,
+ sizeof(*state->slots));
+ if (!state->slots) {
+ set_error(error, error_size, "out of memory creating expert cache slots");
+ goto fail;
+ }
+ for (int i = 0; i < state->layers * state->slots_per_layer; i++)
+ state->slots[i].expert = -1;
+ state->stats.capacity_bytes = (uint64_t)state->layers *
+ state->slots_per_layer * state->record_bytes;
+ store->ops = &operations;
+ store->state = state;
+ *output = store;
+ return 0;
+
+fail:
+ if (state->slots) free(state->slots);
+ free(state->records);
+ coli_st_index_close(state->index);
+ pthread_mutex_destroy(&state->mutex);
+ free(state);
+ free(store);
+ return -1;
+}
+/* ---- end include deepseek_v4_expert_store.c ---- */
+
+#undef coli_deepseek_v4_expert_store_open
+
+#include "native_quant_fp4_rows16.h"
+
+#include "deepseek_v4_internal.h"
+
+typedef struct V4HotPolicy {
+ ColiExpertStore *store;
+ int pin_count;
+ uint64_t repin_interval;
+ uint64_t *usage;
+ uint64_t *layer_requests;
+ int *pins;
+ unsigned char *packed;
+ uint64_t packed_slots;
+ struct V4HotPolicy *next;
+} V4HotPolicy;
+
+static pthread_mutex_t hot_policies_mutex = PTHREAD_MUTEX_INITIALIZER;
+static V4HotPolicy *hot_policies;
+
+
+static V4HotPolicy *hot_find(ColiExpertStore *store) {
+ pthread_mutex_lock(&hot_policies_mutex);
+ V4HotPolicy *policy = hot_policies;
+ while (policy && policy->store != store) policy = policy->next;
+ pthread_mutex_unlock(&hot_policies_mutex);
+ return policy;
+}
+
+#ifndef COLI_V4_PIN_RAMP_REQUESTS
+#define COLI_V4_PIN_RAMP_REQUESTS 0
+#endif
+
+static int hot_is_pinned(const V4HotPolicy *policy, int layer, int expert) {
+ if (!policy || policy->pin_count < 1) return 0;
+ int active = policy->pin_count;
+#if COLI_V4_PIN_RAMP_REQUESTS > 0
+ if (active > 4) {
+ uint64_t grown = policy->layer_requests[layer] /
+ COLI_V4_PIN_RAMP_REQUESTS;
+ active = grown >= (uint64_t)(active - 4)
+ ? active : 4 + (int)grown;
+ }
+#endif
+ const int *pins = policy->pins + (size_t)layer * policy->pin_count;
+ for (int i = 0; i < active; i++)
+ if (pins[i] == expert) return 1;
+ return 0;
+}
+
+static size_t hot_slot_index(const V4ExpertStoreState *state,
+ const V4ExpertSlot *slot) {
+ return (size_t)(slot - state->slots);
+}
+
+static void hot_fill_view(ColiTensorView *view,
+ const V4ExpertRecord *record,
+ const V4ExpertSlot *slot, int matrix,
+ const V4HotPolicy *policy,
+ const V4ExpertStoreState *state) {
+ fill_tensor_view(view, record, slot, matrix);
+#ifdef __AVX512F__
+ if (policy->packed[hot_slot_index(state, slot)]) view->block_rows = 16;
+#endif
+}
+
+static int hot_pack_matrix(ColiTensorView *view, unsigned char *scratch) {
+#ifndef __AVX512F__
+ (void)view; (void)scratch; return -1;
+#else
+ unsigned char *packed_scales = scratch + view->data_bytes;
+ if (coli_fp4_pack_rows16_v10(scratch, packed_scales, view)) return -1;
+ memcpy((void *)view->data, scratch, view->data_bytes);
+ memcpy((void *)view->scales, packed_scales, view->scale_bytes);
+ return 0;
+#endif
+}
+
+/* state->mutex is held and the slot has at least one reference. */
+static int hot_pack_slot_locked(V4HotPolicy *policy,
+ V4ExpertStoreState *state,
+ const V4ExpertRecord *record,
+ V4ExpertSlot *slot) {
+#ifndef __AVX512F__
+ (void)policy; (void)state; (void)record; (void)slot; return -1;
+#else
+ size_t slot_index = hot_slot_index(state, slot);
+ if (policy->packed[slot_index]) return 0;
+ ColiTensorView gate, down, up;
+ fill_tensor_view(&gate, record, slot, V4_W1);
+ fill_tensor_view(&down, record, slot, V4_W2);
+ fill_tensor_view(&up, record, slot, V4_W3);
+ const ColiTensorView *views[3] = {&gate, &down, &up};
+ size_t scratch_size = 0;
+ for (int i = 0; i < 3; i++) {
+ size_t needed = views[i]->data_bytes + views[i]->scale_bytes;
+ if (needed > scratch_size) scratch_size = needed;
+ }
+ unsigned char *scratch = malloc(scratch_size);
+ if (!scratch) return -1;
+ int result = hot_pack_matrix(&gate, scratch) ||
+ hot_pack_matrix(&down, scratch) ||
+ hot_pack_matrix(&up, scratch);
+ free(scratch);
+ if (!result) {
+ policy->packed[slot_index] = 1;
+ policy->packed_slots++;
+ }
+ return result;
+#endif
+}
+
+static void hot_repin_locked(V4HotPolicy *policy, V4ExpertStoreState *state,
+ int layer) {
+ if (!policy || policy->pin_count < 1) return;
+ uint64_t *usage = policy->usage +
+ (size_t)layer * state->experts_per_layer;
+ int *pins = policy->pins + (size_t)layer * policy->pin_count;
+ for (int rank = 0; rank < policy->pin_count; rank++) {
+ int best = -1;
+ for (int expert = 0; expert < state->experts_per_layer; expert++) {
+ int already = 0;
+ for (int prior = 0; prior < rank; prior++)
+ if (pins[prior] == expert) { already = 1; break; }
+ if (!already && usage[expert] &&
+ (best < 0 || usage[expert] > usage[best] ||
+ (usage[expert] == usage[best] && expert < best)))
+ best = expert;
+ }
+ pins[rank] = best;
+ }
+ V4ExpertSlot *slots = layer_slots(state, layer);
+ for (int i = 0; i < state->slots_per_layer; i++)
+ if (slots[i].slab && slots[i].expert >= 0 &&
+ hot_is_pinned(policy, layer, slots[i].expert))
+ slots[i].used = ++state->clock;
+ uint64_t decay_interval = policy->repin_interval * 64;
+ if (decay_interval &&
+ policy->layer_requests[layer] % decay_interval == 0)
+ for (int expert = 0; expert < state->experts_per_layer; expert++)
+ usage[expert] = (usage[expert] + 1) / 2;
+}
+
+static int lookup_hot(ColiExpertStore *store, ColiExpertKey key,
+ ColiExpertView *view) {
+ if (!store || !store->state || !view) {
+ if (view) memset(view, 0, sizeof(*view));
+ return -1;
+ }
+ V4ExpertStoreState *state = store->state;
+ V4ExpertRecord *record = get_record(state, key);
+ V4HotPolicy *policy = hot_find(store);
+ if (!record || !policy) {
+ memset(view, 0, sizeof(*view));
+ return -1;
+ }
+ pthread_mutex_lock(&state->mutex);
+ state->stats.requests++;
+ policy->usage[(size_t)key.layer * state->experts_per_layer + key.expert]++;
+ uint64_t layer_requests = ++policy->layer_requests[key.layer];
+ if (policy->repin_interval &&
+ layer_requests % policy->repin_interval == 0)
+ hot_repin_locked(policy, state, key.layer);
+
+ V4ExpertSlot *slots = layer_slots(state, key.layer);
+ V4ExpertSlot *slot = NULL;
+ for (int i = 0; i < state->slots_per_layer; i++) {
+ if (slots[i].slab && slots[i].expert == key.expert) {
+ slot = &slots[i]; slot->references++;
+ state->active_leases++;
+ slot->used = ++state->clock; state->stats.hits++;
+ if (hot_is_pinned(policy, key.layer, key.expert))
+ hot_pack_slot_locked(policy, state, record, slot);
+ memset(view, 0, sizeof(*view)); view->key = key;
+ hot_fill_view(&view->gate, record, slot, V4_W1, policy, state);
+ hot_fill_view(&view->down, record, slot, V4_W2, policy, state);
+ hot_fill_view(&view->up, record, slot, V4_W3, policy, state);
+ view->lease = slot;
+ pthread_mutex_unlock(&state->mutex); return 0;
+ }
+ }
+ for (int i = 0; i < state->slots_per_layer; i++)
+ if (!slots[i].references && !slots[i].slab) { slot = &slots[i]; break; }
+ if (!slot)
+ for (int i = 0; i < state->slots_per_layer; i++)
+ if (!slots[i].references &&
+ !hot_is_pinned(policy, key.layer, slots[i].expert) &&
+ (!slot || slots[i].used < slot->used)) slot = &slots[i];
+ if (!slot)
+ for (int i = 0; i < state->slots_per_layer; i++)
+ if (!slots[i].references && (!slot || slots[i].used < slot->used))
+ slot = &slots[i];
+ if (!slot) {
+ pthread_mutex_unlock(&state->mutex);
+ memset(view, 0, sizeof(*view));
+ return -1;
+ }
+ if (!slot->slab) {
+ slot->slab = malloc((size_t)state->record_bytes);
+ if (!slot->slab) {
+ pthread_mutex_unlock(&state->mutex);
+ memset(view, 0, sizeof(*view));
+ return -1;
+ }
+ state->stats.resident_bytes += state->record_bytes;
+ }
+ policy->packed[hot_slot_index(state, slot)] = 0;
+ slot->expert = -1; slot->references = 1;
+ state->active_leases++;
+ slot->used = ++state->clock;
+ pthread_mutex_unlock(&state->mutex);
+ int read_result = coli_st_read_at(
+ state->index, record->shard, record->scale_offset,
+ (size_t)record->scale_bytes, slot->slab);
+ if (!read_result) read_result = coli_st_read_at(
+ state->index, record->shard, record->weight_offset,
+ (size_t)record->weight_bytes, slot->slab + record->scale_bytes);
+ pthread_mutex_lock(&state->mutex);
+ if (read_result) {
+ slot->references = 0; slot->expert = -1;
+ if (state->active_leases) state->active_leases--;
+ pthread_mutex_unlock(&state->mutex);
+ memset(view, 0, sizeof(*view));
+ return -1;
+ }
+ slot->expert = key.expert; slot->used = ++state->clock;
+ state->stats.misses++; state->stats.bytes_read += record->record_bytes;
+ if (hot_is_pinned(policy, key.layer, key.expert))
+ hot_pack_slot_locked(policy, state, record, slot);
+ memset(view, 0, sizeof(*view)); view->key = key;
+ hot_fill_view(&view->gate, record, slot, V4_W1, policy, state);
+ hot_fill_view(&view->down, record, slot, V4_W2, policy, state);
+ hot_fill_view(&view->up, record, slot, V4_W3, policy, state);
+ view->lease = slot;
+ pthread_mutex_unlock(&state->mutex); return 0;
+}
+
+static void destroy_hot(ColiExpertStore *store) {
+ pthread_mutex_lock(&hot_policies_mutex);
+ V4HotPolicy **link = &hot_policies;
+ while (*link && (*link)->store != store) link = &(*link)->next;
+ V4HotPolicy *policy = *link;
+ if (policy) *link = policy->next;
+ pthread_mutex_unlock(&hot_policies_mutex);
+ if (policy) {
+ fprintf(stderr, "v4_rows16 packed_slots=%llu\n",
+ (unsigned long long)policy->packed_slots);
+ free(policy->packed); free(policy->pins);
+ free(policy->layer_requests); free(policy->usage); free(policy);
+ }
+ destroy(store);
+}
+
+#ifndef COLI_V4_ROWS16_STORE_OPEN
+#define COLI_V4_ROWS16_STORE_OPEN coli_deepseek_v4_expert_store_open
+#endif
+
+int COLI_V4_ROWS16_STORE_OPEN(
+ const ColiDeepSeekV4ExpertStoreOptions *options, ColiExpertStore **output,
+ char *error, size_t error_size) {
+ static const ColiExpertStoreOps hot_operations = {
+ lookup_hot, release, prefetch, stats, destroy_hot
+ };
+ int result = coli_deepseek_v4_expert_store_open_base(
+ options, output, error, error_size);
+ if (result) return result;
+ V4ExpertStoreState *state = (*output)->state;
+ int minimum_slots = state->experts_per_layer < 6
+ ? state->experts_per_layer : 6;
+ int maximum_pins = state->slots_per_layer - minimum_slots;
+#ifndef COLI_V4_MAX_PIN_SLOTS_PER_LAYER
+#define COLI_V4_MAX_PIN_SLOTS_PER_LAYER 4
+#endif
+ if (maximum_pins > COLI_V4_MAX_PIN_SLOTS_PER_LAYER)
+ maximum_pins = COLI_V4_MAX_PIN_SLOTS_PER_LAYER;
+ int pin_requested = options->pin_slots_per_layer;
+ /* -1 / 0 => implementation default (use maximum_pins). */
+ uint64_t requested = pin_requested > 0
+ ? (uint64_t)pin_requested
+ : (uint64_t)(maximum_pins > 0 ? maximum_pins : 0);
+ int pin_count = requested > (uint64_t)maximum_pins
+ ? maximum_pins : (int)requested;
+ if (pin_count < 0) pin_count = 0;
+ V4HotPolicy *policy = calloc(1, sizeof(*policy));
+ size_t records = (size_t)state->layers * state->experts_per_layer;
+ size_t pins = (size_t)state->layers * (pin_count ? pin_count : 1);
+ size_t slots = (size_t)state->layers * state->slots_per_layer;
+ if (policy) policy->usage = calloc(records, sizeof(*policy->usage));
+ if (policy) policy->layer_requests = calloc(
+ (size_t)state->layers, sizeof(*policy->layer_requests));
+ if (policy) policy->pins = malloc(pins * sizeof(*policy->pins));
+ if (policy) policy->packed = calloc(slots, sizeof(*policy->packed));
+ if (!policy || !policy->usage || !policy->layer_requests ||
+ !policy->pins || !policy->packed) {
+ free(policy ? policy->packed : NULL); free(policy ? policy->pins : NULL);
+ free(policy ? policy->layer_requests : NULL);
+ free(policy ? policy->usage : NULL); free(policy);
+ destroy(*output); *output = NULL;
+ return set_error(error, error_size, "out of memory creating hot policy");
+ }
+ for (size_t i = 0; i < pins; i++) policy->pins[i] = -1;
+ policy->store = *output; policy->pin_count = pin_count;
+ policy->repin_interval = options->repin_interval
+ ? options->repin_interval : (uint64_t)minimum_slots;
+ if (!policy->repin_interval) policy->repin_interval = 1;
+ pthread_mutex_lock(&hot_policies_mutex);
+ policy->next = hot_policies; hot_policies = policy;
+ pthread_mutex_unlock(&hot_policies_mutex);
+ (*output)->ops = &hot_operations;
+ fprintf(stderr,
+ "v4_hot_policy pin_slots_per_layer=%d repin_interval=%llu "
+ "mode=resident-ram rows16=hot-pins\n", pin_count,
+ (unsigned long long)policy->repin_interval);
+ return 0;
+}
+#endif /* COLI_V4_UNIT_EXPERT_STORE_HOT_ROWS16 */
+
+#ifdef COLI_V4_UNIT_EXPERT_ROWS16
+/* ######## deepseek_v4_expert_rows16.c ######## */
+#define coli_v4_expert_forward_ref coli_v4_expert_forward_v17_fallback
+/* ---- begin include deepseek_v4_expert_dual.c ---- */
+#include "deepseek_v4_internal.h"
+
+#include <stdlib.h>
+
+#include "deepseek_v4_internal.h"
+#include "native_quant.h"
+#include "native_quant_dual.h"
+
+int coli_v4_expert_forward_ref(float *output, const ColiExpertView *expert,
+ const float *input, float route_weight,
+ float swiglu_limit) {
+ if (!output || !expert || !input || swiglu_limit < 0.0f ||
+ expert->gate.rows != expert->up.rows ||
+ expert->gate.columns != expert->up.columns ||
+ expert->down.columns != expert->gate.rows ||
+ expert->down.rows != expert->gate.columns) return -1;
+ size_t intermediate = (size_t)expert->gate.rows;
+ size_t output_size = (size_t)expert->down.rows;
+ float *gate = malloc(intermediate * sizeof(*gate));
+ float *up = malloc(intermediate * sizeof(*up));
+ float *activated = malloc(intermediate * sizeof(*activated));
+ if (!gate || !up || !activated) {
+ free(activated); free(up); free(gate); return -1;
+ }
+ int result = coli_fp4_dual_matvec_ref(
+ gate, up, &expert->gate, &expert->up, input);
+ if (!result) {
+ coli_bf16_round_array(gate, intermediate);
+ coli_bf16_round_array(up, intermediate);
+ result = coli_v4_swiglu(activated, gate, up,
+ (int)intermediate, swiglu_limit);
+ }
+ if (!result) {
+ for (size_t i = 0; i < intermediate; i++)
+ activated[i] = coli_bf16_round(activated[i] * route_weight);
+ result = coli_fp4_matvec_ref(output, &expert->down, activated);
+ }
+ if (!result) coli_bf16_round_array(output, output_size);
+ free(activated); free(up); free(gate);
+ return result ? -1 : 0;
+}
+
+int coli_v4_shared_expert_forward_ref(float *output,
+ const ColiTensorView *gate_weight,
+ const ColiTensorView *down_weight,
+ const ColiTensorView *up_weight,
+ const float *input,
+ float swiglu_limit) {
+ if (!output || !gate_weight || !down_weight || !up_weight || !input ||
+ gate_weight->format != COLI_TENSOR_FP8_E4M3_BLOCK ||
+ down_weight->format != COLI_TENSOR_FP8_E4M3_BLOCK ||
+ up_weight->format != COLI_TENSOR_FP8_E4M3_BLOCK ||
+ gate_weight->rows != up_weight->rows ||
+ gate_weight->columns != up_weight->columns ||
+ down_weight->columns != gate_weight->rows ||
+ down_weight->rows != gate_weight->columns) return -1;
+ size_t intermediate = (size_t)gate_weight->rows;
+ size_t output_size = (size_t)down_weight->rows;
+ float *gate = malloc(intermediate * sizeof(*gate));
+ float *up = malloc(intermediate * sizeof(*up));
+ float *activated = malloc(intermediate * sizeof(*activated));
+ if (!gate || !up || !activated) {
+ free(activated); free(up); free(gate); return -1;
+ }
+ int result = coli_fp8_dual_matvec_ref(
+ gate, up, gate_weight, up_weight, input);
+ if (!result) {
+ coli_bf16_round_array(gate, intermediate);
+ coli_bf16_round_array(up, intermediate);
+ result = coli_v4_swiglu(activated, gate, up,
+ (int)intermediate, swiglu_limit);
+ }
+ if (!result) {
+ coli_bf16_round_array(activated, intermediate);
+ result = coli_fp8_matvec_ref(output, down_weight, activated);
+ }
+ if (!result) coli_bf16_round_array(output, output_size);
+ free(activated); free(up); free(gate);
+ return result ? -1 : 0;
+}
+/* ---- end include deepseek_v4_expert_dual.c ---- */
+
+#undef coli_v4_expert_forward_ref
+
+#include "native_quant_fp4_rows16.h"
+
+
+int coli_v4_expert_forward_ref(float *output, const ColiExpertView *expert,
+ const float *input, float route_weight,
+ float swiglu_limit) {
+#ifndef __AVX512F__
+ return coli_v4_expert_forward_v17_fallback(
+ output, expert, input, route_weight, swiglu_limit);
+#else
+ if (!expert || expert->gate.block_rows != 16 ||
+ expert->down.block_rows != 16 || expert->up.block_rows != 16)
+ return coli_v4_expert_forward_v17_fallback(
+ output, expert, input, route_weight, swiglu_limit);
+ if (!output || !input || swiglu_limit < 0.0f) return -1;
+ size_t intermediate = (size_t)expert->gate.rows;
+ size_t output_size = (size_t)expert->down.rows;
+ float *gate = malloc(intermediate * sizeof(*gate));
+ float *up = malloc(intermediate * sizeof(*up));
+ float *activated = malloc(intermediate * sizeof(*activated));
+ if (!gate || !up || !activated) {
+ free(activated); free(up); free(gate); return -1;
+ }
+ int result = coli_fp4_dual_matvec_rows16_v10(
+ gate, up, &expert->gate, &expert->up, input);
+ if (!result) {
+ coli_bf16_round_array(gate, intermediate);
+ coli_bf16_round_array(up, intermediate);
+ result = coli_v4_swiglu(activated, gate, up,
+ (int)intermediate, swiglu_limit);
+ }
+ if (!result) {
+ for (size_t i = 0; i < intermediate; i++)
+ activated[i] = coli_bf16_round(activated[i] * route_weight);
+ result = coli_fp4_matvec_rows16_v10(
+ output, &expert->down, activated);
+ }
+ if (!result) coli_bf16_round_array(output, output_size);
+ free(activated); free(up); free(gate);
+ return result ? -1 : 0;
+#endif
+}
+#endif /* COLI_V4_UNIT_EXPERT_ROWS16 */
+
+#ifdef COLI_V4_UNIT_ROUTE_BF16
+/* ######## deepseek_v4_route_bf16.c ######## */
+#include <math.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+static float route_bf16_decode(uint16_t value) {
+ uint32_t bits = (uint32_t)value << 16;
+ float output;
+ memcpy(&output, &bits, sizeof(output));
+ return output;
+}
+
+static float route_softplus(float value) {
+ return fmaxf(value, 0.0f) + log1pf(expf(-fabsf(value)));
+}
+
+int coli_v4_route_bf16(float *weights, int *indices, const float *hidden,
+ const uint16_t *gate, const float *bias,
+ const int *forced_indices, int experts, int dimension,
+ int topk, float route_scale) {
+ if (!weights || !indices || !hidden || !gate || experts < 1 ||
+ dimension < 1 || topk < 1 || topk > experts) return -1;
+ float *scores = malloc((size_t)experts * sizeof(*scores));
+ float *selection = malloc((size_t)experts * sizeof(*selection));
+ unsigned char *selected = calloc((size_t)experts, 1);
+ if (!scores || !selection || !selected) {
+ free(selected); free(selection); free(scores); return -1;
+ }
+ for (int expert = 0; expert < experts; expert++) {
+ float sum = 0.0f;
+ const uint16_t *row = gate + (size_t)expert * dimension;
+ for (int column = 0; column < dimension; column++)
+ sum += route_bf16_decode(row[column]) * hidden[column];
+ scores[expert] = sqrtf(route_softplus(sum));
+ selection[expert] = scores[expert] + (bias ? bias[expert] : 0.0f);
+ }
+ if (forced_indices) {
+ for (int rank = 0; rank < topk; rank++) {
+ if (forced_indices[rank] < 0 || forced_indices[rank] >= experts) {
+ free(selected); free(selection); free(scores); return -1;
+ }
+ indices[rank] = forced_indices[rank];
+ }
+ } else {
+ for (int rank = 0; rank < topk; rank++) {
+ int best = -1;
+ for (int expert = 0; expert < experts; expert++)
+ if (!selected[expert] &&
+ (best < 0 || selection[expert] > selection[best]))
+ best = expert;
+ indices[rank] = best;
+ selected[best] = 1;
+ }
+ }
+ float total = 0.0f;
+ for (int rank = 0; rank < topk; rank++)
+ total += scores[indices[rank]];
+ if (!(total > 0.0f)) {
+ free(selected); free(selection); free(scores); return -1;
+ }
+ for (int rank = 0; rank < topk; rank++)
+ weights[rank] = scores[indices[rank]] / total * route_scale;
+ free(selected); free(selection); free(scores);
+ return 0;
+}
+#endif /* COLI_V4_UNIT_ROUTE_BF16 */
+
+#ifdef COLI_V4_UNIT_RUNTIME
+/* ######## deepseek_v4_runtime.c / engine ######## */
+#include "deepseek_v4_internal.h"
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* Provided by LAYER_RESIDENT / DSPARK heads units. */
+void coli_v4_layer_resident_reference_free(ColiV4Engine *engine,
+ ColiDeepSeekV4LayerWeights *weights);
+void coli_v4_dspark_heads_close(ColiV4DSparkHeads *heads);
+void coli_v4_dspark_runner_close(ColiV4DSparkRunner *runner);
+
+#ifdef COLI_V4_TEST_HOOKS
+int coli_v4_test_fail_expert_store_open = 0;
+int coli_v4_test_skip_expert_store_open = 0;
+int coli_v4_test_closed_owned_index = 0;
+#endif
+
+void coli_v4_engine_attach_session(ColiV4Engine *engine) {
+ if (engine) engine->active_sessions++;
+}
+
+void coli_v4_engine_detach_session(ColiV4Engine *engine) {
+ if (!engine) return;
+ assert(engine->active_sessions > 0);
+ engine->active_sessions--;
+}
+
+void coli_v4_session_take_runner(ColiV4Session *session,
+ ColiV4DSparkRunner *runner) {
+ if (!session) return;
+ if (session->runner && session->runner != runner)
+ coli_v4_dspark_runner_close(session->runner);
+ session->runner = runner;
+}
+
+void coli_v4_session_clear_runner(ColiV4Session *session) {
+ if (!session || !session->runner) return;
+ coli_v4_dspark_runner_close(session->runner);
+ session->runner = NULL;
+}
+
+#ifdef COLI_V4_TEST_HOOKS
+ColiV4Session *coli_v4_test_session_bare_create(ColiV4Engine *engine) {
+ if (!engine) return NULL;
+ ColiV4Session *session = calloc(1, sizeof(*session));
+ if (!session) return NULL;
+ session->engine = engine;
+ coli_v4_engine_attach_session(engine);
+ return session;
+}
+
+void coli_v4_test_session_bare_destroy(ColiV4Session *session) {
+ if (!session) return;
+ coli_v4_session_clear_runner(session);
+ if (session->engine) {
+ coli_v4_engine_detach_session(session->engine);
+ session->engine = NULL;
+ }
+ free(session);
+}
+
+ColiV4DSparkRunner *coli_v4_session_peek_runner(const ColiV4Session *session) {
+ return session ? session->runner : NULL;
+}
+#endif
+
+const ColiDeepSeekV4Config *coli_v4_engine_config(const ColiV4Engine *engine) {
+ return engine ? &engine->config : NULL;
+}
+
+ColiSafetensorsIndex *coli_v4_engine_target_index(ColiV4Engine *engine) {
+ return engine ? engine->target_index : NULL;
+}
+
+ColiExpertStore *coli_v4_engine_expert_store(ColiV4Engine *engine) {
+ return engine ? engine->experts : NULL;
+}
+
+void coli_v4_engine_memory_summary(const ColiV4Engine *engine,
+ ColiV4EngineMemorySummary *summary) {
+ if (!summary) return;
+ if (!engine) {
+ memset(summary, 0, sizeof(*summary));
+ return;
+ }
+ *summary = engine->summary;
+}
+
+const char *coli_v4_engine_target_model_dir(const ColiV4Engine *engine) {
+ return engine ? engine->runtime.target_model_dir : NULL;
+}
+
+const char *coli_v4_engine_dspark_model_dir(const ColiV4Engine *engine) {
+ return engine ? engine->runtime.dspark_model_dir : NULL;
+}
+
+void coli_v4_engine_destroy(ColiV4Engine *engine) {
+ if (!engine) return;
+ assert(engine->active_sessions == 0 &&
+ "destroy engine while sessions are still alive");
+
+ /* DSpark capture (heads owned by engine; runners must not free them). */
+ if (engine->dspark_capture.heads) {
+ coli_v4_dspark_heads_close(engine->dspark_capture.heads);
+ engine->dspark_capture.heads = NULL;
+ }
+ for (int i = 0; i < COLI_V4_DSPARK_MAX_TARGETS; i++) {
+ free(engine->dspark_capture.states[i]);
+ engine->dspark_capture.states[i] = NULL;
+ }
+ free(engine->dspark_capture.staged_main_x);
+ engine->dspark_capture.staged_main_x = NULL;
+ engine->dspark_capture.capacity = 0;
+ engine->dspark_capture.staged_batch = 0;
+ engine->dspark_capture.staged_hidden = 0;
+ memset(&engine->dspark_capture.manifest, 0,
+ sizeof(engine->dspark_capture.manifest));
+
+ memset(&engine->dspark_verify, 0, sizeof(engine->dspark_verify));
+
+ for (int layer = 0; layer < COLI_V4_RESIDENT_MAX_LAYERS; layer++) {
+ if (!engine->dense_resident.ready[layer]) continue;
+ coli_v4_layer_resident_reference_free(
+ NULL, &engine->dense_resident.layers[layer]);
+ engine->dense_resident.ready[layer] = 0;
+ }
+ engine->dense_resident.index = NULL;
+ engine->dense_resident.total_bytes = 0;
+
+ for (int stage = 0; stage < COLI_V4_DSPARK_MAX_STAGES; stage++) {
+ if (!engine->dspark_resident.ready[stage]) continue;
+ coli_v4_layer_resident_reference_free(
+ NULL, &engine->dspark_resident.layers[stage]);
+ engine->dspark_resident.ready[stage] = 0;
+ }
+ engine->dspark_resident.index = NULL;
+ engine->dspark_resident.total_bytes = 0;
+
+ free(engine->head_cache.data);
+ engine->head_cache.data = NULL;
+ if (engine->owns_experts && engine->experts && engine->experts->ops &&
+ engine->experts->ops->destroy)
+ engine->experts->ops->destroy(engine->experts);
+ engine->experts = NULL;
+ if (engine->owns_index && engine->target_index) {
+#ifdef COLI_V4_TEST_HOOKS
+ coli_v4_test_closed_owned_index++;
+#endif
+ coli_st_index_close(engine->target_index);
+ }
+ engine->target_index = NULL;
+ engine->runtime.target_model_dir = NULL;
+ engine->runtime.dspark_model_dir = NULL;
+ free(engine->owned_target_model_dir);
+ engine->owned_target_model_dir = NULL;
+ free(engine->owned_dspark_model_dir);
+ engine->owned_dspark_model_dir = NULL;
+ free(engine);
+}
+
+int coli_v4_engine_open(ColiV4Engine **output,
+ const ColiV4EngineOpenOptions *options,
+ char *error, size_t error_size) {
+ if (!output || !options || !options->target_model_dir) {
+ if (error && error_size)
+ snprintf(error, error_size, "invalid V4 engine open options");
+ return -1;
+ }
+ *output = NULL;
+ ColiV4Engine *engine = calloc(1, sizeof(*engine));
+ if (!engine) {
+ if (error && error_size)
+ snprintf(error, error_size, "out of memory creating V4 engine");
+ return -1;
+ }
+
+ engine->owned_target_model_dir = strdup(options->target_model_dir);
+ engine->owned_dspark_model_dir = strdup(
+ options->dspark_model_dir ? options->dspark_model_dir
+ : options->target_model_dir);
+ if (!engine->owned_target_model_dir || !engine->owned_dspark_model_dir) {
+ if (error && error_size)
+ snprintf(error, error_size, "out of memory copying model directories");
+ goto fail;
+ }
+ engine->runtime.target_model_dir = engine->owned_target_model_dir;
+ engine->runtime.dspark_model_dir = engine->owned_dspark_model_dir;
+ engine->runtime.memory_limit_bytes = options->memory_limit_bytes;
+ engine->runtime.context_tokens =
+ options->context_tokens > 0 ? options->context_tokens : 4096;
+ engine->runtime.verify_drafts = options->verify_drafts;
+ engine->runtime.repin_interval = options->repin_interval;
+ engine->runtime.pin_slots_per_layer = options->pin_slots_per_layer;
+
+ /* Set owns_* immediately after each acquire so destroy cleans partial opens. */
+ if (coli_v4_config_load(&engine->config, engine->runtime.target_model_dir,
+ error, error_size))
+ goto fail;
+ if (coli_st_index_open(&engine->target_index,
+ engine->runtime.target_model_dir, error,
+ error_size))
+ goto fail;
+ engine->owns_index = 1;
+#ifdef COLI_V4_TEST_HOOKS
+ if (coli_v4_test_fail_expert_store_open) {
+ if (error && error_size)
+ snprintf(error, error_size, "forced expert store open failure");
+ goto fail;
+ }
+ if (coli_v4_test_skip_expert_store_open) {
+ engine->summary.dense_resident = engine->runtime.dense_resident;
+ engine->summary.dspark_resident = engine->runtime.dspark_resident;
+ engine->summary.head_resident = engine->head_cache.data != NULL;
+ engine->summary.expert_cache_bytes =
+ engine->runtime.dspark_expert_cache_bytes;
+ *output = engine;
+ return 0;
+ }
+#endif
+ if (coli_v4_expert_store_open_planned(
+ engine,
+ &(ColiDeepSeekV4ExpertStoreOptions){
+ engine->runtime.target_model_dir,
+ engine->config.num_hidden_layers,
+ engine->config.n_routed_experts,
+ 4ULL << 30,
+ engine->runtime.pin_slots_per_layer,
+ engine->runtime.repin_interval},
+ &engine->experts, error, error_size))
+ goto fail;
+ engine->owns_experts = 1;
+ engine->summary.dense_resident = engine->runtime.dense_resident;
+ engine->summary.dspark_resident = engine->runtime.dspark_resident;
+ engine->summary.head_resident = engine->head_cache.data != NULL;
+ engine->summary.expert_cache_bytes =
+ engine->runtime.dspark_expert_cache_bytes;
+ *output = engine;
+ return 0;
+
+fail:
+ coli_v4_engine_destroy(engine);
+ return -1;
+}
+#endif /* COLI_V4_UNIT_RUNTIME */
+
+#ifdef COLI_V4_UNIT_PROMPT
+/* ######## deepseek_v4_prompt.c ######## */
+#include "deepseek_v4_internal.h"
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+static const char v4_bos[] = "<|begin▁of▁sentence|>";
+static const char v4_user[] = "<|User|>";
+static const char v4_assistant[] = "<|Assistant|>";
+
+static int add_size(size_t *total, size_t value) {
+ if (SIZE_MAX - *total < value) return -1;
+ *total += value;
+ return 0;
+}
+
+int coli_v4_prompt_build(char **output, size_t *output_length,
+ const char *user_message, const char *system_message,
+ ColiDeepSeekV4PromptMode mode) {
+ if (!output || !user_message || mode < COLI_V4_PROMPT_CHAT ||
+ mode > COLI_V4_PROMPT_RAW) return -1;
+ *output = NULL;
+ if (output_length) *output_length = 0;
+ const char *system = system_message ? system_message : "";
+ if (mode == COLI_V4_PROMPT_RAW) {
+ size_t length = strlen(user_message);
+ char *copy = malloc(length + 1);
+ if (!copy) return -1;
+ memcpy(copy, user_message, length + 1);
+ *output = copy;
+ if (output_length) *output_length = length;
+ return 0;
+ }
+ const char *thinking = mode == COLI_V4_PROMPT_THINKING
+ ? "<think>" : "</think>";
+ size_t length = 0;
+ if (add_size(&length, strlen(v4_bos)) ||
+ add_size(&length, strlen(system)) ||
+ add_size(&length, strlen(v4_user)) ||
+ add_size(&length, strlen(user_message)) ||
+ add_size(&length, strlen(v4_assistant)) ||
+ add_size(&length, strlen(thinking)) || length == SIZE_MAX)
+ return -1;
+ char *prompt = malloc(length + 1);
+ if (!prompt) return -1;
+ char *at = prompt;
+#define APPEND(part) do { \
+ size_t count = strlen(part); memcpy(at, part, count); at += count; \
+} while (0)
+ APPEND(v4_bos);
+ APPEND(system);
+ APPEND(v4_user);
+ APPEND(user_message);
+ APPEND(v4_assistant);
+ APPEND(thinking);
+#undef APPEND
+ *at = '\0';
+ *output = prompt;
+ if (output_length) *output_length = length;
+ return 0;
+}
+#endif /* COLI_V4_UNIT_PROMPT */
+
+#ifdef COLI_V4_UNIT_GENERATE_STATS
+/* ######## tools/deepseek_v4_generate_stats.c ######## */
+#define COLI_V4_GENERATE_MAIN coli_v4_generate_stats_legacy_main
+#define COLI_V4_GENERATE_HELPERS_ONLY
+#define spec_print spec_print_diagnostic_legacy
+/* ---- begin include tools/deepseek_v4_generate_speculative.c ---- */
+#include <time.h>
+
+#define main coli_v4_first_token_legacy_main
+/* ---- begin include tools/deepseek_v4_first_token.c ---- */
+#include <float.h>
+#include <math.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+#include "json.h"
+#include "native_quant.h"
+#include "safetensors_index.h"
+#include "tensor_io.h"
+#include "tok.h"
+
+static int load_embedding(float *state, const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config, int token) {
+ const ColiSafetensorsTensor *embed = coli_st_find(index, "embed.weight");
+ int d = config->hidden_size, hc = config->hc_mult;
+ uint16_t *row = malloc((size_t)d * sizeof(*row));
+ if (!embed || embed->dtype != COLI_ST_BF16 || !row || token < 0 ||
+ token >= config->vocab_size ||
+ coli_st_read_at(index, embed->shard,
+ embed->offset + (uint64_t)token * d * sizeof(*row),
+ (size_t)d * sizeof(*row), row)) {
+ free(row);
+ return -1;
+ }
+ for (int copy = 0; copy < hc; copy++)
+ for (int i = 0; i < d; i++)
+ state[(size_t)copy * d + i] = coli_bf16_decode(row[i]);
+ free(row);
+ return 0;
+}
+
+static int final_hidden(float *output, const float *state,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config,
+ char *error, size_t error_size) {
+ ColiFloatTensor function = {0}, base = {0}, scale = {0}, norm = {0};
+ if (coli_tensor_load_f32(&function, index, "hc_head_fn", error, error_size) ||
+ coli_tensor_load_f32(&base, index, "hc_head_base", error, error_size) ||
+ coli_tensor_load_f32(&scale, index, "hc_head_scale", error, error_size) ||
+ coli_tensor_load_f32(&norm, index, "norm.weight", error, error_size))
+ return -1;
+ int d = config->hidden_size, hc = config->hc_mult;
+ int flattened = hc * d;
+ float square = 0.0f;
+ for (int i = 0; i < flattened; i++) square += state[i] * state[i];
+ float inverse_rms = 1.0f / sqrtf(square / flattened + config->rms_norm_eps);
+ float pre[16];
+ if (hc > 16) return -1;
+ for (int copy = 0; copy < hc; copy++) {
+ float mix = 0.0f;
+ for (int i = 0; i < flattened; i++)
+ mix += function.data[(size_t)copy * flattened + i] * state[i];
+ mix *= inverse_rms;
+ float z = mix * scale.data[0] + base.data[copy];
+ float sigmoid = z >= 0.0f
+ ? 1.0f / (1.0f + expf(-z))
+ : expf(z) / (1.0f + expf(z));
+ pre[copy] = sigmoid + config->hc_eps;
+ }
+ for (int i = 0; i < d; i++) {
+ float value = 0.0f;
+ for (int copy = 0; copy < hc; copy++)
+ value += pre[copy] * state[(size_t)copy * d + i];
+ output[i] = coli_bf16_round(value);
+ }
+ coli_v4_rmsnorm(output, output, norm.data, d, config->rms_norm_eps);
+ coli_bf16_round_array(output, (size_t)d);
+ coli_float_tensor_free(&norm);
+ coli_float_tensor_free(&scale);
+ coli_float_tensor_free(&base);
+ coli_float_tensor_free(&function);
+ return 0;
+}
+
+static int head_argmax(ColiV4Engine *engine, const float *hidden,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config,
+ int *best_token, float *best_logit) {
+ const ColiSafetensorsTensor *head = coli_st_find(index, "head.weight");
+ int d = config->hidden_size, vocab = config->vocab_size;
+ enum { ROWS = 64 };
+ uint16_t *raw = malloc((size_t)ROWS * d * sizeof(*raw));
+ float *scores = malloc((size_t)ROWS * sizeof(*scores));
+ if (!head || head->dtype != COLI_ST_BF16 || !raw || !scores) {
+ free(scores); free(raw);
+ return -1;
+ }
+ int winner = -1;
+ float maximum = -FLT_MAX;
+ for (int start = 0; start < vocab; start += ROWS) {
+ int rows = vocab - start < ROWS ? vocab - start : ROWS;
+ size_t bytes = (size_t)rows * d * sizeof(*raw);
+ if (coli_st_read_at_engine(
+ engine, index, head->shard,
+ head->offset + (uint64_t)start * d * sizeof(*raw),
+ bytes, raw)) {
+ free(scores); free(raw);
+ return -1;
+ }
+ #pragma omp parallel for
+ for (int row = 0; row < rows; row++) {
+ float sum = 0.0f;
+ const uint16_t *weight = raw + (size_t)row * d;
+ for (int i = 0; i < d; i++)
+ sum += coli_bf16_decode(weight[i]) * hidden[i];
+ scores[row] = sum;
+ }
+ for (int row = 0; row < rows; row++)
+ if (scores[row] > maximum) {
+ maximum = scores[row];
+ winner = start + row;
+ }
+ }
+ free(scores); free(raw);
+ *best_token = winner;
+ *best_logit = maximum;
+ return winner < 0 ? -1 : 0;
+}
+
+static int has_sentence_end(const char *text, int length) {
+ for (int i = 0; i < length; i++) {
+ unsigned char value = (unsigned char)text[i];
+ if (value == '.' || value == '!' || value == '?' || value == '\n') return 1;
+ if (i + 2 < length && value == 0xe3 &&
+ (unsigned char)text[i + 1] == 0x80 &&
+ (unsigned char)text[i + 2] == 0x82) return 1; /* 。 */
+ if (i + 2 < length && value == 0xef &&
+ (unsigned char)text[i + 1] == 0xbc &&
+ ((unsigned char)text[i + 2] == 0x81 ||
+ (unsigned char)text[i + 2] == 0x9f)) return 1; /* !? */
+ }
+ return 0;
+}
+
+int main(int argc, char **argv) {
+ if (argc < 3 || argc > 6) {
+ fprintf(stderr, "usage: %s MODEL_DIR INPUT_TOKEN_ID [TOKEN_COUNT]\n"
+ " %s MODEL_DIR --prompt TEXT [MAX_NEW_TOKENS] [--stop-sentence]\n",
+ argv[0], argv[0]);
+ return 2;
+ }
+ int text_mode = !strcmp(argv[2], "--prompt");
+ if (text_mode && argc < 4) return 2;
+ int stop_sentence = text_mode && argc == 6 &&
+ !strcmp(argv[5], "--stop-sentence");
+ if (text_mode && argc == 6 && !stop_sentence) return 2;
+ int input_token = text_mode ? -1 : atoi(argv[2]);
+ int token_count = text_mode ? (argc == 5 ? atoi(argv[4]) : 32)
+ : (argc == 4 ? atoi(argv[3]) : 1);
+ if (token_count < 1) return 2;
+ char error[512] = {0};
+ ColiDeepSeekV4Config config;
+ ColiSafetensorsIndex *index = NULL;
+ ColiExpertStore *experts = NULL;
+ if (coli_v4_config_load(&config, argv[1], error, sizeof(error)) ||
+ coli_st_index_open(&index, argv[1], error, sizeof(error)) ||
+ coli_deepseek_v4_expert_store_open(
+ &(ColiDeepSeekV4ExpertStoreOptions){
+ argv[1], config.num_hidden_layers, config.n_routed_experts,
+ UINT64_C(4) * 1024 * 1024 * 1024, -1, 0,
+ }, &experts, error, sizeof(error))) {
+ fprintf(stderr, "%s\n", error);
+ return 1;
+ }
+ Tok tokenizer;
+ int *prompt_ids = NULL, prompt_count = 0;
+ int *generated_ids = NULL, generated_count = 0;
+ if (text_mode) {
+ char tokenizer_path[4096];
+ snprintf(tokenizer_path, sizeof(tokenizer_path), "%s/tokenizer.json", argv[1]);
+ tok_load(&tokenizer, tokenizer_path);
+ int prompt_capacity = (int)strlen(argv[3]) + 16;
+ prompt_ids = malloc((size_t)prompt_capacity * sizeof(*prompt_ids));
+ generated_ids = malloc((size_t)token_count * sizeof(*generated_ids));
+ if (!prompt_ids || !generated_ids) return 1;
+ prompt_count = tok_encode(&tokenizer, argv[3], (int)strlen(argv[3]),
+ prompt_ids, prompt_capacity);
+ if (prompt_count < 1) {
+ fprintf(stderr, "prompt produced no tokens\n"); return 1;
+ }
+ fprintf(stderr, "prompt_tokens=%d max_new_tokens=%d eos_token=1\n",
+ prompt_count, token_count);
+ }
+ size_t state_count = (size_t)config.hc_mult * config.hidden_size;
+ float *state = malloc(state_count * sizeof(*state));
+ float *next = malloc(state_count * sizeof(*next));
+ float *hidden = malloc((size_t)config.hidden_size * sizeof(*hidden));
+ ColiDeepSeekV4WindowAttentionState **attention = calloc(
+ (size_t)config.num_hidden_layers, sizeof(*attention));
+ if (!state || !next || !hidden || !attention) return 1;
+ for (int layer = 0; layer < config.num_hidden_layers; layer++)
+ if (coli_v4_window_attention_create(&attention[layer], &config)) return 1;
+
+ int current_token = text_mode ? prompt_ids[0] : input_token;
+ int total_steps = text_mode ? prompt_count + token_count - 1 : token_count;
+ for (int position = 0; position < total_steps; position++) {
+ if (text_mode && position < prompt_count)
+ current_token = prompt_ids[position];
+ if (load_embedding(state, index, &config, current_token)) return 1;
+ for (int layer_id = 0; layer_id < config.num_hidden_layers; layer_id++) {
+ ColiDeepSeekV4LayerWeights layer;
+ if (coli_v4_layer_load(NULL, &layer, &config, index, layer_id,
+ error, sizeof(error)) ||
+ coli_v4_block_window_token_ref(
+ next, attention[layer_id], &layer, &config, experts, state,
+ current_token, position, error, sizeof(error))) {
+ fprintf(stderr, "position %d layer %d: %s\n",
+ position, layer_id, error);
+ return 1;
+ }
+ coli_v4_layer_free(NULL, &layer);
+ float *swap = state; state = next; next = swap;
+ }
+ fprintf(stderr, "position %d/%d complete (%d layers)\n", position,
+ total_steps - 1, config.num_hidden_layers);
+ if (final_hidden(hidden, state, index, &config, error, sizeof(error))) {
+ fprintf(stderr, "final hidden: %s\n", error);
+ return 1;
+ }
+ int output_token;
+ float output_logit;
+ if (head_argmax(NULL, hidden, index, &config, &output_token, &output_logit)) {
+ fprintf(stderr, "lm_head failed\n");
+ return 1;
+ }
+ if (!text_mode) {
+ printf("position=%d input_token=%d output_token=%d logit=%.9g\n",
+ position, current_token, output_token, output_logit);
+ } else if (position >= prompt_count - 1) {
+ generated_ids[generated_count++] = output_token;
+ char piece[1024];
+ int piece_length = tok_decode(&tokenizer, &output_token, 1,
+ piece, (int)sizeof(piece) - 1);
+ printf("generated=%d position=%d token=%d logit=%.9g piece=",
+ generated_count, position, output_token, output_logit);
+ fwrite(piece, 1, (size_t)piece_length, stdout);
+ fputc('\n', stdout);
+ fflush(stdout);
+ if (output_token == 1) break;
+ if (stop_sentence && has_sentence_end(piece, piece_length)) break;
+ }
+ current_token = output_token;
+ }
+ ColiExpertStoreStats stats;
+ experts->ops->stats(experts, &stats);
+ printf("summary tokens=%d expert_reads=%llu bytes=%llu\n",
+ text_mode ? generated_count : token_count,
+ (unsigned long long)stats.misses,
+ (unsigned long long)stats.bytes_read);
+ if (text_mode) {
+ size_t text_capacity = (size_t)generated_count * 256 + 1;
+ char *text = malloc(text_capacity);
+ if (!text) return 1;
+ int decode_count = generated_count;
+ if (decode_count && generated_ids[decode_count - 1] == 1) decode_count--;
+ int text_length = tok_decode(&tokenizer, generated_ids, decode_count,
+ text, (int)text_capacity - 1);
+ printf("generated_text=");
+ fwrite(text, 1, (size_t)text_length, stdout);
+ fputc('\n', stdout);
+ printf("completed_text=");
+ fwrite(argv[3], 1, strlen(argv[3]), stdout);
+ fwrite(text, 1, (size_t)text_length, stdout);
+ fputc('\n', stdout);
+ free(text);
+ }
+ for (int layer = 0; layer < config.num_hidden_layers; layer++)
+ coli_v4_window_attention_destroy(attention[layer]);
+ free(attention);
+ free(generated_ids); free(prompt_ids);
+ free(hidden); free(next); free(state);
+ experts->ops->destroy(experts);
+ coli_st_index_close(index);
+ return 0;
+}
+/* ---- end include tools/deepseek_v4_first_token.c ---- */
+
+#undef main
+
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_dspark.h"
+#include "deepseek_v4_dspark.h"
+#include "deepseek_v4_dspark.h"
+#include "deepseek_v4_dspark.h"
+#include "deepseek_v4_dspark.h"
+
+static double spec_now(void) {
+ struct timespec value;
+ clock_gettime(CLOCK_MONOTONIC, &value);
+ return (double)value.tv_sec + value.tv_nsec * 1e-9;
+}
+
+static int spec_sentence_end(const char *text, int length) {
+ for (int i = 0; i < length; i++) {
+ unsigned char c = (unsigned char)text[i];
+ if (c == '.' || c == '!' || c == '?' || c == '\n') return 1;
+ if (i + 2 < length && c == 0xe3 &&
+ (unsigned char)text[i + 1] == 0x80 &&
+ (unsigned char)text[i + 2] == 0x82) return 1;
+ if (i + 2 < length && c == 0xef &&
+ (unsigned char)text[i + 1] == 0xbc &&
+ ((unsigned char)text[i + 2] == 0x81 ||
+ (unsigned char)text[i + 2] == 0x9f)) return 1;
+ }
+ return 0;
+}
+
+#ifndef COLI_V4_GENERATE_HELPERS_ONLY
+static int spec_print(Tok *tokenizer, int token, float logit,
+ int position, int ordinal, int stop_sentence) {
+ char piece[1024];
+ int length = tok_decode(tokenizer, &token, 1, piece, sizeof(piece) - 1);
+ printf("generated=%d position=%d token=%d logit=%.9g piece=",
+ ordinal, position, token, logit);
+ fwrite(piece, 1, (size_t)length, stdout); fputc('\n', stdout); fflush(stdout);
+ return token == 1 || (stop_sentence && spec_sentence_end(piece, length));
+}
+#endif
+
+static int target_batch(ColiV4Engine *engine, float **state_ptr, float **next_ptr,
+ ColiDeepSeekV4WindowAttentionState **attention,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config,
+ ColiExpertStore *experts, const int *tokens,
+ int start, int batch, char *error, size_t error_size) {
+ if (!state_ptr || !next_ptr || !*state_ptr || !*next_ptr || !attention ||
+ !index || !config || !experts || !tokens || start < 0 || batch < 1) {
+ if (error && error_size)
+ snprintf(error, error_size, "invalid target batch arguments");
+ return -1;
+ }
+ float *state = *state_ptr, *next = *next_ptr;
+ size_t hd = (size_t)config->hc_mult * config->hidden_size;
+ for (int layer_id = 0; layer_id < config->num_hidden_layers; layer_id++) {
+ ColiDeepSeekV4LayerWeights layer;
+ if (coli_v4_layer_load(engine, &layer, config, index, layer_id,
+ error, error_size)) return -1;
+ int result = 0;
+ for (int offset = 0; !result && offset < batch; offset += 64) {
+ int chunk = batch - offset;
+ if (chunk > 64) chunk = 64;
+ result = coli_v4_block_window_batch_ref(
+ next + (size_t)offset * hd, attention[layer_id],
+ &layer, config, experts, state + (size_t)offset * hd,
+ tokens + offset, start + offset, chunk, error, error_size);
+ if (result && error && error_size && !error[0])
+ snprintf(error, error_size,
+ "target prefill failed layer=%d offset=%d batch=%d",
+ layer_id, offset, chunk);
+ }
+ if (!result)
+ coli_v4_dspark_capture_after_block(engine, &layer, config, next,
+ batch);
+ coli_v4_layer_free(engine, &layer);
+ if (result) return -1;
+ float *swap = state; state = next; next = swap;
+ }
+ *state_ptr = state; *next_ptr = next; return 0;
+}
+
+static int dspark_prefill_chunks(ColiV4DSparkRunner *runner,
+ const float *main_x, int hidden_size,
+ int start_position, int batch,
+ char *error, size_t error_size) {
+ if (!runner || !main_x || hidden_size < 1 || start_position < 0 ||
+ batch < 1) {
+ if (error && error_size)
+ snprintf(error, error_size, "invalid chunked DSpark prefill");
+ return -1;
+ }
+ for (int offset = 0; offset < batch; offset += 64) {
+ int chunk = batch - offset;
+ if (chunk > 64) chunk = 64;
+ if (coli_v4_dspark_runner_prefill(
+ runner, main_x + (size_t)offset * hidden_size,
+ start_position + offset, chunk, error, error_size))
+ return -1;
+ }
+ return 0;
+}
+
+static int target_token(ColiV4Engine *engine, float **state_ptr, float **next_ptr,
+ ColiDeepSeekV4WindowAttentionState **attention,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config,
+ ColiExpertStore *experts, int token, int position,
+ char *error, size_t error_size) {
+ float *state = *state_ptr, *next = *next_ptr;
+ if (load_embedding(state, index, config, token)) return -1;
+ for (int layer_id = 0; layer_id < config->num_hidden_layers; layer_id++) {
+ ColiDeepSeekV4LayerWeights layer;
+ if (coli_v4_layer_load(engine, &layer, config, index, layer_id,
+ error, error_size)) return -1;
+ int result = coli_v4_block_window_token_ref(
+ next, attention[layer_id], &layer, config, experts,
+ state, token, position, error, error_size);
+ if (!result)
+ coli_v4_dspark_capture_after_block(engine, &layer, config, next, 1);
+ coli_v4_layer_free(engine, &layer);
+ if (result) return -1;
+ float *swap = state; state = next; next = swap;
+ }
+ *state_ptr = state; *next_ptr = next; return 0;
+}
+
+#ifndef COLI_V4_GENERATE_HELPERS_ONLY
+static uint64_t env_u64(const char *name, uint64_t fallback) {
+ const char *text = getenv(name); char *end = NULL;
+ if (!text || !*text) return fallback;
+ unsigned long long value = strtoull(text, &end, 10);
+ return end != text && !*end && value ? (uint64_t)value : fallback;
+}
+
+static float env_float(const char *name, float fallback) {
+ const char *text = getenv(name); char *end = NULL;
+ if (!text || !*text) return fallback;
+ float value = strtof(text, &end);
+ return end != text && !*end && value > 0.0f && value <= 1.0f
+ ? value : fallback;
+}
+
+#ifndef COLI_V4_GENERATE_MAIN
+#define COLI_V4_GENERATE_MAIN main
+#endif
+
+int COLI_V4_GENERATE_MAIN(int argc, char **argv) {
+ if (argc < 4 || argc > 6) {
+ fprintf(stderr, "usage: %s TARGET_MODEL DSPARK_MODEL PROMPT "
+ "[MAX_NEW_TOKENS] [--stop-sentence]\n", argv[0]);
+ return 2;
+ }
+ int max_new = argc >= 5 ? atoi(argv[4]) : 32;
+ int stop_sentence = argc == 6 && !strcmp(argv[5], "--stop-sentence");
+ if (max_new < 1 || (argc == 6 && !stop_sentence)) return 2;
+#ifdef _WIN32
+ _putenv_s("COLI_V4_DSPARK_MODEL", argv[2]);
+#else
+ setenv("COLI_V4_DSPARK_MODEL", argv[2], 1);
+#endif
+ char error[512] = {0}, tokenizer_path[4096];
+ ColiDeepSeekV4Config config; ColiSafetensorsIndex *index = NULL;
+ ColiExpertStore *experts = NULL; ColiV4DSparkRunner *runner = NULL;
+ if (coli_v4_config_load(&config, argv[1], error, sizeof(error)) ||
+ coli_st_index_open(&index, argv[1], error, sizeof(error)) ||
+ coli_deepseek_v4_expert_store_open(
+ &(ColiDeepSeekV4ExpertStoreOptions){argv[1],
+ config.num_hidden_layers, config.n_routed_experts, 4ULL << 30},
+ &experts, error, sizeof(error))) {
+ fprintf(stderr, "%s\n", error); return 1;
+ }
+ snprintf(tokenizer_path, sizeof(tokenizer_path), "%s/tokenizer.json", argv[1]);
+ Tok tokenizer; tok_load(&tokenizer, tokenizer_path);
+ int prompt_capacity = (int)strlen(argv[3]) + 16;
+ int *prompt_ids = malloc((size_t)prompt_capacity * sizeof(int));
+ int *generated = malloc((size_t)(max_new + 64) * sizeof(int));
+ int prompt_count = tok_encode(&tokenizer, argv[3], strlen(argv[3]),
+ prompt_ids, prompt_capacity);
+ if (!prompt_ids || !generated || prompt_count < 1 || prompt_count > 64)
+ return 1;
+ size_t hd = (size_t)config.hc_mult * config.hidden_size;
+ float *state = malloc((size_t)64 * hd * sizeof(float));
+ float *next = malloc((size_t)64 * hd * sizeof(float));
+ float *hidden = malloc((size_t)config.hidden_size * sizeof(float));
+ float *main_x_batch = malloc((size_t)64 * config.hidden_size * sizeof(float));
+ ColiDeepSeekV4WindowAttentionState **attention = calloc(
+ config.num_hidden_layers, sizeof(*attention));
+ if (!state || !next || !hidden || !main_x_batch || !attention) return 1;
+ for (int layer = 0; layer < config.num_hidden_layers; layer++)
+ if (coli_v4_window_attention_create(&attention[layer], &config)) return 1;
+ for (int item = 0; item < prompt_count; item++)
+ if (load_embedding(state + (size_t)item * hd, index, &config,
+ prompt_ids[item])) return 1;
+
+ double started = spec_now();
+ if (target_batch(NULL, &state, &next, attention, index, &config, experts,
+ prompt_ids, 0, prompt_count, error, sizeof(error)) ||
+ coli_v4_dspark_capture_main_x(NULL, main_x_batch, prompt_count, &config)) {
+ fprintf(stderr, "%s\n", error); return 1;
+ }
+ if (coli_v4_dspark_runner_open(&runner, NULL, argv[2], argv[1], &config,
+ 256ULL << 20, error, sizeof(error))) {
+ fprintf(stderr, "%s\n", error); return 1;
+ }
+ if (coli_v4_dspark_runner_use_shared_heads(
+ runner, coli_v4_dspark_capture_heads(NULL)) ||
+ dspark_prefill_chunks(runner, main_x_batch, config.hidden_size,
+ 0, prompt_count, error, sizeof(error))) {
+ coli_v4_dspark_runner_close(runner);
+ fprintf(stderr, "%s\n", error); return 1;
+ }
+ const float *last = state + (size_t)(prompt_count - 1) * hd;
+ int current; float current_logit;
+ if (final_hidden(hidden, last, index, &config, error, sizeof(error)) ||
+ head_argmax(NULL, hidden, index, &config, &current, &current_logit)) return 1;
+ int generated_count = 0, last_processed = prompt_count - 1;
+ generated[generated_count++] = current;
+ int done = spec_print(&tokenizer, current, current_logit,
+ last_processed, generated_count, stop_sentence);
+ double first_at = spec_now();
+
+ ColiV4SpeculativeController controller;
+ coli_v4_speculative_controller_init(
+ &controller, env_u64("COLI_V4_DSPARK_MIN_PROPOSALS", 10),
+ env_float("COLI_V4_DSPARK_DISABLE_THRESHOLD", 0.35f));
+ int block = coli_v4_dspark_runner_verify_block_size(runner);
+ int drafts[64], verified[65]; float draft_logits[64];
+ while (!done && generated_count < max_new) {
+ if (!controller.enabled || last_processed == prompt_count - 1) {
+ int position = last_processed + 1;
+ if (target_token(NULL, &state, &next, attention, index, &config, experts,
+ current, position, error, sizeof(error)) ||
+ final_hidden(hidden, state, index, &config, error, sizeof(error)) ||
+ head_argmax(NULL, hidden, index, &config, &current, &current_logit) ||
+ coli_v4_dspark_capture_main_x(NULL, main_x_batch, 1, &config)) {
+ fprintf(stderr, "%s\n", error); return 1;
+ }
+ last_processed = position; generated[generated_count++] = current;
+ done = spec_print(&tokenizer, current, current_logit,
+ last_processed, generated_count, stop_sentence);
+ continue;
+ }
+ const float *main_x = main_x_batch;
+ if (coli_v4_dspark_runner_draft(
+ runner, main_x, current, last_processed,
+ drafts, draft_logits, error, sizeof(error))) {
+ fprintf(stderr, "%s\n", error); return 1;
+ }
+ ColiV4VerificationResult verification;
+ if (coli_v4_target_verify_greedy_batch(
+ &verification, verified, 65, NULL, attention, index, &config,
+ experts, current, drafts, block, last_processed + 1,
+ error, sizeof(error))) {
+ fprintf(stderr, "%s\n", error); return 1;
+ }
+ coli_v4_speculative_record(&controller, block,
+ verification.accepted_draft_tokens);
+ int committed = verification.accepted_draft_tokens + 1;
+ if (coli_v4_dspark_capture_main_x(NULL, main_x_batch, committed, &config))
+ return 1;
+ memmove(main_x_batch,
+ main_x_batch + (size_t)(committed - 1) * config.hidden_size,
+ (size_t)config.hidden_size * sizeof(float));
+ int base_position = last_processed + 1;
+ last_processed += committed;
+ fprintf(stderr,
+ "dspark_verify proposed=%d accepted=%d rate=%.3f enabled=%d\n",
+ block, verification.accepted_draft_tokens,
+ coli_v4_speculative_acceptance(&controller), controller.enabled);
+ for (int i = 0; i < verification.output_count &&
+ generated_count < max_new && !done; i++) {
+ current = verified[i]; current_logit = 0.0f;
+ generated[generated_count++] = current;
+ done = spec_print(&tokenizer, current, current_logit,
+ base_position + i, generated_count, stop_sentence);
+ }
+ }
+ double ended = spec_now();
+ ColiExpertStoreStats stats; experts->ops->stats(experts, &stats);
+ size_t text_capacity = (size_t)generated_count * 256 + 1;
+ char *text = malloc(text_capacity);
+ int text_count = generated_count;
+ if (text_count && generated[text_count - 1] == 1) text_count--;
+ int text_length = tok_decode(&tokenizer, generated, text_count,
+ text, text_capacity - 1);
+ printf("summary tokens=%d expert_reads=%llu bytes=%llu "
+ "dspark_rounds=%llu proposed=%llu accepted=%llu rate=%.3f enabled=%d\n",
+ generated_count, (unsigned long long)stats.misses,
+ (unsigned long long)stats.bytes_read,
+ (unsigned long long)controller.rounds,
+ (unsigned long long)controller.proposed,
+ (unsigned long long)controller.accepted,
+ coli_v4_speculative_acceptance(&controller), controller.enabled);
+ printf("generated_text="); fwrite(text, 1, text_length, stdout);
+ printf("\ntiming time_to_first_token=%.3fs after_first=%.3fs total=%.3fs\n",
+ first_at - started, ended - first_at, ended - started);
+ return 0;
+}
+#endif
+/* ---- end include tools/deepseek_v4_generate_speculative.c ---- */
+
+#undef spec_print
+#undef COLI_V4_GENERATE_HELPERS_ONLY
+#undef COLI_V4_GENERATE_MAIN
+
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+
+#ifdef COLI_V4_EXPERIMENTAL_PARALLEL_PREFIX_VERIFY
+#include <omp.h>
+
+#include "deepseek_v4_dspark.h"
+#include "deepseek_v4_dspark.h"
+
+
+typedef struct {
+ float **state_ptr;
+ float **next_ptr;
+ ColiDeepSeekV4WindowAttentionState **attention;
+ const ColiSafetensorsIndex *index;
+ const ColiDeepSeekV4Config *config;
+ ColiExpertStore *experts;
+ int token;
+ int position;
+ int threads;
+ int target_token;
+ float target_logit;
+ float *main_x;
+ double seconds;
+ int result;
+ char error[512];
+} ParallelPrefixJob;
+
+
+static void *parallel_prefix_worker(void *argument) {
+ ParallelPrefixJob *job = argument;
+ double began = spec_now();
+ int previous_threads = omp_get_max_threads();
+ omp_set_num_threads(job->threads);
+ float *hidden = malloc((size_t)job->config->hidden_size * sizeof(float));
+ if (!hidden) {
+ snprintf(job->error, sizeof(job->error),
+ "out of memory in parallel target prefix");
+ job->result = -1;
+ } else {
+ job->result = target_token(
+ NULL, job->state_ptr, job->next_ptr, job->attention, job->index,
+ job->config, job->experts, job->token, job->position,
+ job->error, sizeof(job->error));
+ if (!job->result)
+ job->result = final_hidden(
+ hidden, *job->state_ptr, job->index, job->config,
+ job->error, sizeof(job->error));
+ if (!job->result)
+ job->result = head_argmax(
+ NULL, hidden, job->index, job->config,
+ &job->target_token, &job->target_logit);
+ if (!job->result)
+ job->result = coli_v4_dspark_capture_main_x(
+ NULL, job->main_x, 1, job->config);
+ }
+ free(hidden);
+ omp_set_num_threads(previous_threads);
+ job->seconds = spec_now() - began;
+ return NULL;
+}
+#endif
+
+static ColiExpertStoreStats stats_subtract(ColiExpertStoreStats end,
+ ColiExpertStoreStats begin) {
+ ColiExpertStoreStats delta = end;
+ delta.requests -= begin.requests; delta.hits -= begin.hits;
+ delta.misses -= begin.misses; delta.prefetched -= begin.prefetched;
+ delta.prefetch_hits -= begin.prefetch_hits;
+ delta.bytes_read -= begin.bytes_read;
+ return delta;
+}
+
+static double stats_hit_rate(ColiExpertStoreStats stats) {
+ return stats.requests ? 100.0 * stats.hits / stats.requests : 0.0;
+}
+
+#ifdef COLI_V4_EXPERIMENTAL_STATE_HASH
+static uint64_t state_hash_v70(const float *values, size_t count) {
+ const unsigned char *bytes = (const unsigned char *)values;
+ uint64_t hash = UINT64_C(1469598103934665603);
+ for (size_t i = 0; i < count * sizeof(float); i++) {
+ hash ^= bytes[i];
+ hash *= UINT64_C(1099511628211);
+ }
+ return hash;
+}
+#endif
+
+typedef struct {
+ const char *model_dir;
+ const char *draft_model_dir;
+ const char *prompt;
+ const char *prompt_file;
+ const char *system_prompt;
+ const char *oracle_path;
+ const char *record_oracle_path;
+ int max_new_tokens;
+ int teacher_forcing;
+ int greedy;
+ int stop_sentence;
+ int no_dspark;
+ double memory_gib;
+ ColiDeepSeekV4PromptMode prompt_mode;
+} V4CliOptions;
+
+static void v4_cli_usage(FILE *stream, const char *program) {
+ fprintf(stream,
+ "usage: %s MODEL PROMPT [options]\n"
+ " %s MODEL --prompt-file FILE [options]\n"
+ " %s MODEL --oracle FILE [--teacher-forcing N] [--greedy N] [options]\n"
+ " --max-tokens N maximum generated tokens (default: 128)\n"
+ " --memory-gb GiB cap this process; otherwise use available RAM\n"
+ " --prompt-file PATH read UTF-8 prompt from file (avoids argv encoding issues)\n"
+ " --draft-model PATH separate DSpark checkpoint (default: MODEL)\n"
+ " --system TEXT optional system message\n"
+ " --thinking enable the official V4 thinking prefix\n"
+ " --raw-prompt bypass the default V4 chat template\n"
+ " --stop-sentence stop after the first sentence terminator\n"
+ " --no-dspark disable speculative decode (greedy target only)\n"
+ " --oracle FILE validate against an oracle JSON fixture\n"
+ " --teacher-forcing N oracle: compare top-1 on N prompt positions\n"
+ " --greedy N oracle: compare N greedy continuation tokens\n"
+ " --record-oracle FILE write greedy tokens + tf_pred to JSON\n",
+ program, program, program);
+}
+
+static char *v4_read_prompt_file(const char *path, char *error, size_t error_size) {
+ FILE *stream = fopen(path, "rb");
+ if (!stream) {
+ snprintf(error, error_size, "cannot open prompt file: %s", path);
+ return NULL;
+ }
+ if (fseek(stream, 0, SEEK_END)) {
+ fclose(stream);
+ snprintf(error, error_size, "cannot seek prompt file: %s", path);
+ return NULL;
+ }
+ long length = ftell(stream);
+ if (length < 0 || fseek(stream, 0, SEEK_SET)) {
+ fclose(stream);
+ snprintf(error, error_size, "cannot size prompt file: %s", path);
+ return NULL;
+ }
+ char *text = malloc((size_t)length + 1);
+ if (!text) {
+ fclose(stream);
+ snprintf(error, error_size, "out of memory reading prompt file");
+ return NULL;
+ }
+ size_t read = fread(text, 1, (size_t)length, stream);
+ fclose(stream);
+ if (read != (size_t)length) {
+ free(text);
+ snprintf(error, error_size, "cannot read prompt file: %s", path);
+ return NULL;
+ }
+ while (read > 0 && (text[read - 1] == '\n' || text[read - 1] == '\r'))
+ read--;
+ text[read] = 0;
+ return text;
+}
+
+static int v4_cli_positive_int(const char *text, int *output) {
+ char *end = NULL;
+ long value = strtol(text, &end, 10);
+ if (!text[0] || end == text || *end || value < 1 || value > 1048576)
+ return -1;
+ *output = (int)value;
+ return 0;
+}
+
+static int v4_cli_memory(const char *text, double *output) {
+ char *end = NULL;
+ double value = strtod(text, &end);
+ if (!text[0] || end == text || *end || value <= 0.0 || value > 1048576.0)
+ return -1;
+ *output = value;
+ return 0;
+}
+
+static int v4_cli_parse(int argc, char **argv, V4CliOptions *options) {
+ if (!options || argc < 3) return -1;
+ memset(options, 0, sizeof(*options));
+ options->model_dir = argv[1];
+ options->draft_model_dir = argv[1];
+ options->max_new_tokens = 128;
+ options->prompt_mode = COLI_V4_PROMPT_CHAT;
+ int argi = 2;
+ if (argv[2][0] != '-') {
+ options->prompt = argv[2];
+ argi = 3;
+ }
+ for (int i = argi; i < argc; i++) {
+ const char *option = argv[i];
+ if (!strcmp(option, "--max-tokens")) {
+ if (++i == argc ||
+ v4_cli_positive_int(argv[i], &options->max_new_tokens))
+ return -1;
+ } else if (!strcmp(option, "--memory-gb")) {
+ if (++i == argc || v4_cli_memory(argv[i], &options->memory_gib))
+ return -1;
+ } else if (!strcmp(option, "--prompt-file")) {
+ if (++i == argc || !argv[i][0]) return -1;
+ options->prompt_file = argv[i];
+ } else if (!strcmp(option, "--draft-model")) {
+ if (++i == argc || !argv[i][0]) return -1;
+ options->draft_model_dir = argv[i];
+ } else if (!strcmp(option, "--system")) {
+ if (++i == argc) return -1;
+ options->system_prompt = argv[i];
+ } else if (!strcmp(option, "--thinking")) {
+ if (options->prompt_mode == COLI_V4_PROMPT_RAW) return -1;
+ options->prompt_mode = COLI_V4_PROMPT_THINKING;
+ } else if (!strcmp(option, "--raw-prompt")) {
+ if (options->prompt_mode == COLI_V4_PROMPT_THINKING) return -1;
+ options->prompt_mode = COLI_V4_PROMPT_RAW;
+ } else if (!strcmp(option, "--stop-sentence")) {
+ options->stop_sentence = 1;
+ } else if (!strcmp(option, "--no-dspark")) {
+ options->no_dspark = 1;
+ } else if (!strcmp(option, "--oracle")) {
+ if (++i == argc || !argv[i][0]) return -1;
+ options->oracle_path = argv[i];
+ } else if (!strcmp(option, "--record-oracle")) {
+ if (++i == argc || !argv[i][0]) return -1;
+ options->record_oracle_path = argv[i];
+ } else if (!strcmp(option, "--teacher-forcing")) {
+ if (++i == argc ||
+ v4_cli_positive_int(argv[i], &options->teacher_forcing))
+ return -1;
+ } else if (!strcmp(option, "--greedy")) {
+ if (++i == argc ||
+ v4_cli_positive_int(argv[i], &options->greedy))
+ return -1;
+ } else {
+ return -1;
+ }
+ }
+ if (options->oracle_path) {
+ if (options->prompt || options->prompt_file ||
+ options->record_oracle_path) return -1;
+ if (!options->teacher_forcing) options->teacher_forcing = 32;
+ if (!options->greedy) options->greedy = 20;
+ options->no_dspark = 1;
+ } else if (options->prompt_file) {
+ if (options->prompt) return -1;
+ } else if (!options->prompt) {
+ return -1;
+ }
+ return 0;
+}
+
+static int *v4_oracle_read_ids(jval *root, const char *key, int *count) {
+ jval *array = json_get(root, key);
+ if (!array || array->t != J_ARR || array->len < 1) return NULL;
+ int *ids = malloc((size_t)array->len * sizeof(*ids));
+ if (!ids) return NULL;
+ for (int i = 0; i < array->len; i++) {
+ if (!array->kids[i] || array->kids[i]->t != J_NUM) {
+ free(ids);
+ return NULL;
+ }
+ ids[i] = (int)array->kids[i]->num;
+ }
+ *count = array->len;
+ return ids;
+}
+
+static int v4_oracle_write_json(const char *path, const char *source,
+ const char *model_dir, const char *prompt,
+ const int *prompt_ids, int prompt_count,
+ const int *full_ids, int full_count,
+ const int *tf_pred, int tf_count) {
+ FILE *out = fopen(path, "wb");
+ if (!out) return -1;
+ fprintf(out,
+ "{\n \"source\": \"%s\",\n \"model\": \"%s\",\n \"prompt\": ",
+ source, model_dir);
+ fputc('"', out);
+ for (const char *p = prompt ? prompt : ""; *p; p++) {
+ unsigned char c = (unsigned char)*p;
+ if (c == '\\' || c == '"') fputc('\\', out);
+ if (c == '\n') { fputs("\\n", out); continue; }
+ if (c == '\r') { fputs("\\r", out); continue; }
+ if (c == '\t') { fputs("\\t", out); continue; }
+ fputc(c, out);
+ }
+ fputs("\",\n \"comparison\": {\n"
+ " \"top1_token\": \"exact\",\n"
+ " \"logits\": \"not required for coli-self fixtures\",\n"
+ " \"dspark\": \"greedy tokens must match --no-dspark\"\n"
+ " },\n \"prompt_ids\": [", out);
+ for (int i = 0; i < prompt_count; i++)
+ fprintf(out, "%s%d", i ? ", " : "", prompt_ids[i]);
+ fputs("],\n \"full_ids\": [", out);
+ for (int i = 0; i < full_count; i++)
+ fprintf(out, "%s%d", i ? ", " : "", full_ids[i]);
+ fputs("],\n \"tf_pred\": [", out);
+ for (int i = 0; i < tf_count; i++)
+ fprintf(out, "%s%d", i ? ", " : "", tf_pred[i]);
+ fputs("]\n}\n", out);
+ fclose(out);
+ return 0;
+}
+
+static void v4_attention_free(ColiDeepSeekV4WindowAttentionState **attention,
+ int layers) {
+ if (!attention) return;
+ for (int layer = 0; layer < layers; layer++)
+ coli_v4_window_attention_destroy(attention[layer]);
+ free(attention);
+}
+
+#include <assert.h>
+
+static void session_free_buffers(ColiV4Session *session) {
+ if (!session) return;
+ free(session->text); session->text = NULL; session->text_length = 0;
+ free(session->main_x_batch); session->main_x_batch = NULL;
+ free(session->hidden); session->hidden = NULL;
+ free(session->next); session->next = NULL;
+ free(session->state); session->state = NULL;
+ free(session->generated); session->generated = NULL;
+ free(session->prompt_ids); session->prompt_ids = NULL;
+}
+
+static void session_free_attention(ColiV4Session *session) {
+ if (!session || !session->attention) return;
+ v4_attention_free(session->attention, session->config.num_hidden_layers);
+ session->attention = NULL;
+}
+
+void coli_v4_session_destroy(ColiV4Session *session) {
+ if (!session) return;
+ coli_v4_session_clear_runner(session);
+ session_free_attention(session);
+ session_free_buffers(session);
+ if (session->tokenizer_ready) {
+ tok_free(&session->tokenizer);
+ session->tokenizer_ready = 0;
+ }
+ if (session->engine) {
+ coli_v4_engine_detach_session(session->engine);
+ session->engine = NULL;
+ }
+ free(session);
+}
+
+int coli_v4_session_create(ColiV4Session **output, ColiV4Engine *engine,
+ const ColiV4SessionCreateOptions *options,
+ char *error, size_t error_size) {
+ if (!output || !engine) {
+ if (error && error_size)
+ snprintf(error, error_size, "invalid V4 session create arguments");
+ return -1;
+ }
+ *output = NULL;
+ ColiV4Session *session = calloc(1, sizeof(*session));
+ if (!session) {
+ if (error && error_size)
+ snprintf(error, error_size, "out of memory creating V4 session");
+ return -1;
+ }
+ session->engine = engine;
+ coli_v4_engine_attach_session(engine);
+ session->config = *coli_v4_engine_config(engine);
+ session->max_prompt_tokens =
+ options && options->max_prompt_tokens > 0 ? options->max_prompt_tokens
+ : 512;
+ session->max_new_tokens_cap =
+ options && options->max_new_tokens_cap > 0 ? options->max_new_tokens_cap
+ : 512;
+
+ const char *model_dir = coli_v4_engine_target_model_dir(engine);
+ char tokenizer_path[4096];
+ if (!model_dir) {
+ coli_v4_session_destroy(session);
+ if (error && error_size)
+ snprintf(error, error_size, "engine has no target model directory");
+ return -1;
+ }
+ snprintf(tokenizer_path, sizeof(tokenizer_path), "%s/tokenizer.json",
+ model_dir);
+ tok_load(&session->tokenizer, tokenizer_path);
+ session->tokenizer_ready = 1;
+
+ session->attention = calloc((size_t)session->config.num_hidden_layers,
+ sizeof(*session->attention));
+ if (!session->attention) {
+ coli_v4_session_destroy(session);
+ if (error && error_size)
+ snprintf(error, error_size, "out of memory allocating attention");
+ return -1;
+ }
+ for (int layer = 0; layer < session->config.num_hidden_layers; layer++) {
+ if (coli_v4_window_attention_create(&session->attention[layer],
+ &session->config)) {
+ coli_v4_session_destroy(session);
+ if (error && error_size)
+ snprintf(error, error_size, "cannot create attention layer %d",
+ layer);
+ return -1;
+ }
+ }
+
+ size_t hd = (size_t)session->config.hc_mult * session->config.hidden_size;
+ size_t slots = (size_t)session->max_prompt_tokens;
+ session->state = malloc(slots * hd * sizeof(float));
+ session->next = malloc(slots * hd * sizeof(float));
+ session->hidden = malloc((size_t)session->config.hidden_size * sizeof(float));
+ session->main_x_batch =
+ malloc(slots * (size_t)session->config.hidden_size * sizeof(float));
+ session->prompt_ids =
+ malloc((size_t)(session->max_prompt_tokens + 16) * sizeof(int));
+ session->generated =
+ malloc((size_t)(session->max_new_tokens_cap + 64) * sizeof(int));
+ if (!session->state || !session->next || !session->hidden ||
+ !session->main_x_batch || !session->prompt_ids || !session->generated) {
+ coli_v4_session_destroy(session);
+ if (error && error_size)
+ snprintf(error, error_size, "out of memory allocating session buffers");
+ return -1;
+ }
+ *output = session;
+ return 0;
+}
+
+int coli_v4_session_generated_text(const ColiV4Session *session,
+ char *buffer, size_t buffer_size,
+ size_t *out_length) {
+ if (!session || !buffer || buffer_size == 0) return -1;
+ size_t copy = session->text_length;
+ if (copy >= buffer_size) copy = buffer_size - 1;
+ if (session->text && copy)
+ memcpy(buffer, session->text, copy);
+ buffer[copy] = 0;
+ if (out_length) *out_length = copy;
+ return 0;
+}
+
+static int session_emit_token(ColiV4Session *session,
+ ColiV4SessionTokenFn on_token, void *user_data,
+ int token, float logit, int position, int ordinal,
+ int stop_at_sentence) {
+ if (on_token) {
+ if (on_token(user_data, token, logit, position, ordinal)) return 1;
+ } else {
+ char piece[1024];
+ int length = tok_decode(&session->tokenizer, &token, 1, piece,
+ (int)sizeof(piece) - 1);
+ if (length > 0) fwrite(piece, 1, (size_t)length, stdout);
+ fflush(stdout);
+ if (stop_at_sentence && spec_sentence_end(piece, length)) return 1;
+ }
+ return token == 1;
+}
+
+int coli_v4_session_generate(ColiV4Session *session,
+ const char *prompt, size_t prompt_length,
+ const ColiV4SessionGenerateOptions *options,
+ ColiV4SessionTokenFn on_token, void *user_data,
+ ColiV4SessionGenerateStats *stats_out,
+ char *error, size_t error_size) {
+ if (!session || !session->engine || !prompt || !options ||
+ options->max_new_tokens < 1) {
+ if (error && error_size)
+ snprintf(error, error_size, "invalid V4 session generate arguments");
+ return -1;
+ }
+ if (stats_out) memset(stats_out, 0, sizeof(*stats_out));
+ free(session->text);
+ session->text = NULL;
+ session->text_length = 0;
+ session->prompt_count = 0;
+ session->generated_count = 0;
+ coli_v4_session_clear_runner(session);
+ for (int layer = 0; layer < session->config.num_hidden_layers; layer++)
+ coli_v4_window_attention_reset(session->attention[layer]);
+
+ int max_new = options->max_new_tokens;
+ if (max_new > session->max_new_tokens_cap)
+ max_new = session->max_new_tokens_cap;
+ int prompt_capacity = session->max_prompt_tokens + 16;
+ int prompt_count = tok_encode(&session->tokenizer, prompt, prompt_length,
+ session->prompt_ids, prompt_capacity);
+ if (prompt_count < 1 || prompt_count > session->max_prompt_tokens) {
+ if (error && error_size)
+ snprintf(error, error_size,
+ "V4 prompt must encode to between 1 and %d tokens",
+ session->max_prompt_tokens);
+ return -1;
+ }
+ session->prompt_count = prompt_count;
+
+ ColiV4Engine *engine = session->engine;
+ const ColiDeepSeekV4Config *config = &session->config;
+ ColiSafetensorsIndex *index = coli_v4_engine_target_index(engine);
+ ColiExpertStore *experts = coli_v4_engine_expert_store(engine);
+ float *state = session->state;
+ float *next = session->next;
+ float *hidden = session->hidden;
+ float *main_x_batch = session->main_x_batch;
+ ColiDeepSeekV4WindowAttentionState **attention = session->attention;
+ int *generated = session->generated;
+ size_t hd = (size_t)config->hc_mult * config->hidden_size;
+
+ for (int item = 0; item < prompt_count; item++)
+ if (load_embedding(state + (size_t)item * hd, index, config,
+ session->prompt_ids[item])) {
+ if (error && error_size)
+ snprintf(error, error_size, "cannot load embedding");
+ return -1;
+ }
+
+ double setup_done = spec_now();
+ if (target_batch(engine, &state, &next, attention, index, config, experts,
+ session->prompt_ids, 0, prompt_count, error, error_size))
+ return -1;
+ session->state = state;
+ session->next = next;
+ if (coli_v4_dspark_capture_main_x(engine, main_x_batch, prompt_count, config)) {
+ if (error && error_size)
+ snprintf(error, error_size, "dspark capture main_x failed");
+ return -1;
+ }
+
+ ColiV4DSparkRunner *runner = NULL;
+ if (!options->no_dspark) {
+ const char *draft = coli_v4_engine_dspark_model_dir(engine);
+ const char *target = coli_v4_engine_target_model_dir(engine);
+ if (coli_v4_dspark_runner_open(&runner, session->engine, draft, target,
+ config, 256ULL << 20, error, error_size))
+ return -1;
+ coli_v4_session_take_runner(session, runner);
+ if (coli_v4_dspark_runner_use_shared_heads(
+ runner, coli_v4_dspark_capture_heads(engine)))
+ return -1;
+ if (dspark_prefill_chunks(runner, main_x_batch, config->hidden_size,
+ 0, prompt_count, error, error_size))
+ return -1;
+ }
+
+ const float *last = state + (size_t)(prompt_count - 1) * hd;
+ int current = 0;
+ float current_logit = 0.0f;
+ if (final_hidden(hidden, last, index, config, error, error_size) ||
+ head_argmax(engine, hidden, index, config, &current, &current_logit))
+ return -1;
+ int generated_count = 0;
+ int last_processed = prompt_count - 1;
+ generated[generated_count++] = current;
+ int done = session_emit_token(session, on_token, user_data, current,
+ current_logit, last_processed,
+ generated_count, options->stop_at_sentence);
+ double first_at = spec_now();
+
+ ColiV4SpeculativeController controller;
+ coli_v4_speculative_controller_init(&controller, 10, 0.35f);
+ if (options->no_dspark) controller.enabled = 0;
+ int block = runner ? coli_v4_dspark_runner_verify_block_size(runner) : 0;
+ int drafts[64], verified[65];
+ float draft_logits[64];
+
+ while (!done && generated_count < max_new) {
+ if (!controller.enabled || last_processed == prompt_count - 1) {
+ int position = last_processed + 1;
+ if (target_token(engine, &state, &next, attention, index, config, experts,
+ current, position, error, error_size))
+ return -1;
+ session->state = state;
+ session->next = next;
+ if (final_hidden(hidden, state, index, config, error, error_size) ||
+ head_argmax(engine, hidden, index, config, &current, &current_logit))
+ return -1;
+ if (coli_v4_dspark_capture_main_x(engine, main_x_batch, 1, config)) {
+ if (error && error_size)
+ snprintf(error, error_size, "dspark capture failed");
+ return -1;
+ }
+ last_processed = position;
+ generated[generated_count++] = current;
+ done = session_emit_token(session, on_token, user_data, current,
+ current_logit, last_processed,
+ generated_count,
+ options->stop_at_sentence);
+ continue;
+ }
+ ColiV4VerificationResult verification;
+ if (coli_v4_dspark_runner_draft(runner, main_x_batch, current,
+ last_processed, drafts, draft_logits,
+ error, error_size))
+ return -1;
+ if (coli_v4_target_verify_greedy_batch(
+ &verification, verified, 65, engine, attention, index, config,
+ experts, current, drafts, block, last_processed + 1, error,
+ error_size))
+ return -1;
+ coli_v4_speculative_record(&controller, block,
+ verification.accepted_draft_tokens);
+ int committed = verification.accepted_draft_tokens + 1;
+ if (coli_v4_dspark_capture_main_x(engine, main_x_batch, committed, config)) {
+ if (error && error_size)
+ snprintf(error, error_size, "dspark capture failed");
+ return -1;
+ }
+ memmove(main_x_batch,
+ main_x_batch + (size_t)(committed - 1) * config->hidden_size,
+ (size_t)config->hidden_size * sizeof(float));
+ int base_position = last_processed + 1;
+ last_processed += committed;
+ for (int i = 0; i < verification.output_count &&
+ generated_count < max_new && !done; i++) {
+ current = verified[i];
+ current_logit = 0.0f;
+ generated[generated_count++] = current;
+ done = session_emit_token(session, on_token, user_data, current,
+ current_logit, base_position + i,
+ generated_count,
+ options->stop_at_sentence);
+ }
+ }
+ double ended = spec_now();
+ session->state = state;
+ session->next = next;
+ session->generated_count = generated_count;
+
+ size_t text_capacity = (size_t)generated_count * 256 + 1;
+ session->text = malloc(text_capacity);
+ if (session->text) {
+ int text_count = generated_count;
+ if (text_count && generated[text_count - 1] == 1) text_count--;
+ session->text_length = tok_decode(&session->tokenizer, generated,
+ text_count, session->text,
+ (int)text_capacity - 1);
+ }
+ if (!on_token) {
+ fputc('\n', stdout);
+ fflush(stdout);
+ }
+ if (stats_out) {
+ stats_out->prompt_tokens = prompt_count;
+ stats_out->generated_tokens = generated_count;
+ stats_out->eos_stopped = done && generated_count > 0 &&
+ generated[generated_count - 1] == 1;
+ stats_out->dspark_rounds = controller.rounds;
+ stats_out->dspark_proposed = controller.proposed;
+ stats_out->dspark_accepted = controller.accepted;
+ stats_out->dspark_acceptance =
+ coli_v4_speculative_acceptance(&controller);
+ stats_out->time_to_first_token_sec = first_at - setup_done;
+ stats_out->decode_sec = ended - first_at;
+ }
+ return 0;
+}
+
+static int v4_oracle_teacher_forcing(
+ const int *full_ids, int full_count, const int *expected, int expect_count,
+ ColiDeepSeekV4WindowAttentionState **attention,
+ const ColiSafetensorsIndex *index, const ColiDeepSeekV4Config *config,
+ ColiExpertStore *experts, char *error, size_t error_size,
+ int *matched_out) {
+ size_t hd = (size_t)config->hc_mult * config->hidden_size;
+ float *state = malloc((size_t)full_count * hd * sizeof(float));
+ float *next = malloc((size_t)full_count * hd * sizeof(float));
+ float *hidden = malloc((size_t)config->hidden_size * sizeof(float));
+ if (!state || !next || !hidden) {
+ free(state); free(next); free(hidden);
+ return -1;
+ }
+ for (int item = 0; item < full_count; item++)
+ if (load_embedding(state + (size_t)item * hd, index, config,
+ full_ids[item])) {
+ free(state); free(next); free(hidden);
+ return -1;
+ }
+ if (target_batch(NULL, &state, &next, attention, index, config, experts,
+ full_ids, 0, full_count, error, error_size)) {
+ free(state); free(next); free(hidden);
+ return -1;
+ }
+ int limit = expect_count < full_count ? expect_count : full_count;
+ int matched = 0;
+ for (int pos = 0; pos < limit; pos++) {
+ int pred = -1;
+ float logit = 0.0f;
+ if (final_hidden(hidden, state + (size_t)pos * hd, index, config,
+ error, error_size) ||
+ head_argmax(NULL, hidden, index, config, &pred, &logit)) {
+ free(state); free(next); free(hidden);
+ return -1;
+ }
+ if (pred == expected[pos]) matched++;
+ else
+ fprintf(stderr,
+ "[ORACLE] TF mismatch pos=%d expected=%d got=%d logit=%.6g\n",
+ pos, expected[pos], pred, logit);
+ }
+ free(state); free(next); free(hidden);
+ *matched_out = matched;
+ return 0;
+}
+
+static int v4_oracle_greedy_from_prompt(
+ const int *prompt_ids, int prompt_count, int *generated, int max_new,
+ ColiDeepSeekV4WindowAttentionState **attention,
+ const ColiSafetensorsIndex *index, const ColiDeepSeekV4Config *config,
+ ColiExpertStore *experts, char *error, size_t error_size) {
+ size_t hd = (size_t)config->hc_mult * config->hidden_size;
+ float *state = malloc((size_t)prompt_count * hd * sizeof(float));
+ float *next = malloc((size_t)prompt_count * hd * sizeof(float));
+ float *hidden = malloc((size_t)config->hidden_size * sizeof(float));
+ if (!state || !next || !hidden) {
+ free(state); free(next); free(hidden);
+ return -1;
+ }
+ for (int item = 0; item < prompt_count; item++)
+ if (load_embedding(state + (size_t)item * hd, index, config,
+ prompt_ids[item])) {
+ free(state); free(next); free(hidden);
+ return -1;
+ }
+ if (target_batch(NULL, &state, &next, attention, index, config, experts,
+ prompt_ids, 0, prompt_count, error, error_size)) {
+ free(state); free(next); free(hidden);
+ return -1;
+ }
+ int current = -1;
+ float logit = 0.0f;
+ if (final_hidden(hidden, state + (size_t)(prompt_count - 1) * hd,
+ index, config, error, error_size) ||
+ head_argmax(NULL, hidden, index, config, &current, &logit)) {
+ free(state); free(next); free(hidden);
+ return -1;
+ }
+ int count = 0;
+ generated[count++] = current;
+ int position = prompt_count;
+ while (count < max_new && current != 1) {
+ if (target_token(NULL, &state, &next, attention, index, config, experts,
+ current, position, error, error_size) ||
+ final_hidden(hidden, state, index, config, error, error_size) ||
+ head_argmax(NULL, hidden, index, config, &current, &logit)) {
+ free(state); free(next); free(hidden);
+ return -1;
+ }
+ generated[count++] = current;
+ position++;
+ }
+ free(state); free(next); free(hidden);
+ return count;
+}
+
+static int spec_print(Tok *tokenizer, int token, float logit,
+ int position, int ordinal, int stop_sentence) {
+ (void)logit; (void)position; (void)ordinal;
+ if (token == 1) return 1;
+ char piece[1024];
+ int length = tok_decode(tokenizer, &token, 1, piece, sizeof(piece) - 1);
+ if (length > 0) fwrite(piece, 1, (size_t)length, stdout);
+ fflush(stdout);
+ return stop_sentence && spec_sentence_end(piece, length);
+}
+
+static void v4_generate_cleanup(
+ ColiV4Session *session,
+ char *prompt_storage,
+ ColiV4Engine *engine,
+ ColiV4DSparkRunner *runner,
+ ColiDeepSeekV4WindowAttentionState **attention,
+ int layers,
+ int *prompt_ids,
+ int *generated,
+ float *state,
+ float *next,
+ float *hidden,
+ float *main_x_batch,
+ char *text,
+ int *full_ids,
+ int *tf_pred,
+ float *tf_state,
+ float *tf_next,
+ float *tf_hidden)
+{
+ /* Session owns attention/runner/buffers on the normal path. Clear any
+ * aliases into the session before destroying it to avoid double-free. */
+ if (session) {
+ prompt_ids = NULL;
+ generated = NULL;
+ attention = NULL;
+ runner = NULL;
+ state = NULL;
+ next = NULL;
+ hidden = NULL;
+ main_x_batch = NULL;
+ text = NULL;
+ coli_v4_session_destroy(session);
+ }
+ free(tf_hidden);
+ free(tf_next);
+ free(tf_state);
+ free(tf_pred);
+ free(full_ids);
+ free(text);
+ free(main_x_batch);
+ free(hidden);
+ free(next);
+ free(state);
+ free(generated);
+ free(prompt_ids);
+ v4_attention_free(attention, layers);
+ if (runner) coli_v4_dspark_runner_close(runner);
+ coli_v4_engine_destroy(engine);
+ free(prompt_storage);
+}
+
+#ifndef COLI_V4_SKIP_GENERATE_MAIN
+int main(int argc, char **argv) {
+ double process_started = spec_now();
+ int result = 1;
+ V4CliOptions cli;
+ if (v4_cli_parse(argc, argv, &cli)) {
+ v4_cli_usage(stderr, argc ? argv[0] : "deepseek-v4");
+ return 2;
+ }
+ int max_new = cli.max_new_tokens;
+ int stop_sentence = cli.stop_sentence;
+
+ char error[512] = {0}, tokenizer_path[4096];
+ char *prompt_storage = NULL;
+ ColiV4Engine *engine = NULL;
+ ColiV4Session *session = NULL;
+ Tok tokenizer;
+ memset(&tokenizer, 0, sizeof(tokenizer));
+ ColiDeepSeekV4Config config;
+ memset(&config, 0, sizeof(config));
+ ColiSafetensorsIndex *index = NULL;
+ ColiExpertStore *experts = NULL;
+ ColiV4DSparkRunner *runner = NULL;
+ ColiDeepSeekV4WindowAttentionState **attention = NULL;
+ int *prompt_ids = NULL;
+ int *generated = NULL;
+ float *state = NULL, *next = NULL, *hidden = NULL, *main_x_batch = NULL;
+ char *text = NULL;
+ int *full_ids = NULL, *tf_pred = NULL;
+ float *tf_state = NULL, *tf_next = NULL, *tf_hidden = NULL;
+ int layers = 0;
+ if (cli.prompt_file) {
+ prompt_storage = v4_read_prompt_file(cli.prompt_file, error, sizeof(error));
+ if (!prompt_storage) {
+ fprintf(stderr, "%s\n", error);
+ goto cleanup;
+ }
+ cli.prompt = prompt_storage;
+ }
+ {
+ ColiV4EngineOpenOptions open_opts = {
+ .target_model_dir = cli.model_dir,
+ .dspark_model_dir = cli.draft_model_dir,
+ .pin_slots_per_layer = -1,
+ };
+ if (cli.memory_gib > 0.0)
+ open_opts.memory_limit_bytes =
+ (uint64_t)(cli.memory_gib * 1073741824.0);
+ if (coli_v4_engine_open(&engine, &open_opts, error, sizeof(error))) {
+ fprintf(stderr, "%s\n", error);
+ goto cleanup;
+ }
+ }
+ config = *coli_v4_engine_config(engine);
+ index = coli_v4_engine_target_index(engine);
+ experts = coli_v4_engine_expert_store(engine);
+ layers = config.num_hidden_layers;
+ snprintf(tokenizer_path, sizeof(tokenizer_path), "%s/tokenizer.json",
+ cli.model_dir);
+ tok_load(&tokenizer, tokenizer_path);
+
+ if (cli.oracle_path) {
+ FILE *oracle_file = fopen(cli.oracle_path, "rb");
+ if (!oracle_file) { perror(cli.oracle_path); goto cleanup; }
+ fseek(oracle_file, 0, SEEK_END);
+ long oracle_bytes = ftell(oracle_file);
+ fseek(oracle_file, 0, SEEK_SET);
+ char *oracle_text = malloc((size_t)oracle_bytes + 1);
+ if (!oracle_text ||
+ fread(oracle_text, 1, (size_t)oracle_bytes, oracle_file) !=
+ (size_t)oracle_bytes) {
+ fclose(oracle_file);
+ free(oracle_text);
+ goto cleanup;
+ }
+ oracle_text[oracle_bytes] = 0;
+ fclose(oracle_file);
+ char *arena = NULL;
+ jval *root = json_parse(oracle_text, &arena);
+ free(oracle_text);
+ int prompt_count = 0, full_count = 0, tf_count = 0;
+ prompt_ids = v4_oracle_read_ids(root, "prompt_ids", &prompt_count);
+ full_ids = v4_oracle_read_ids(root, "full_ids", &full_count);
+ tf_pred = v4_oracle_read_ids(root, "tf_pred", &tf_count);
+ json_free(root);
+ free(arena);
+ if (!prompt_ids || !full_ids || !tf_pred ||
+ prompt_count < 1 || full_count <= prompt_count ||
+ tf_count < 1) {
+ fprintf(stderr, "invalid oracle fixture: %s\n", cli.oracle_path);
+ goto cleanup;
+ }
+ attention = calloc((size_t)config.num_hidden_layers, sizeof(*attention));
+ if (!attention) goto cleanup;
+ for (int layer = 0; layer < config.num_hidden_layers; layer++)
+ if (coli_v4_window_attention_create(&attention[layer], &config))
+ goto cleanup;
+
+ int tf_limit = cli.teacher_forcing;
+ if (tf_limit > tf_count) tf_limit = tf_count;
+ if (tf_limit > full_count) tf_limit = full_count;
+ int tf_matched = 0;
+ if (v4_oracle_teacher_forcing(full_ids, full_count, tf_pred, tf_limit,
+ attention, index, &config, experts,
+ error, sizeof(error), &tf_matched)) {
+ fprintf(stderr, "%s\n", error);
+ goto cleanup;
+ }
+ printf("PREFILL (teacher-forcing) C vs oracle: %d/%d positions\n",
+ tf_matched, tf_limit);
+
+ for (int layer = 0; layer < config.num_hidden_layers; layer++)
+ coli_v4_window_attention_reset(attention[layer]);
+ int greedy_limit = cli.greedy;
+ generated = malloc((size_t)(greedy_limit + 8) * sizeof(int));
+ int got = v4_oracle_greedy_from_prompt(
+ prompt_ids, prompt_count, generated, greedy_limit, attention,
+ index, &config, experts, error, sizeof(error));
+ if (got < 0) {
+ fprintf(stderr, "%s\n", error);
+ goto cleanup;
+ }
+ int greedy_matched = 0;
+ int continue_count = full_count - prompt_count;
+ int expected_count = continue_count < greedy_limit
+ ? continue_count : greedy_limit;
+ int compare = got < expected_count ? got : expected_count;
+ for (int i = 0; i < compare; i++) {
+ int expected = full_ids[prompt_count + i];
+ if (generated[i] == expected) greedy_matched++;
+ else
+ fprintf(stderr,
+ "[ORACLE] greedy mismatch i=%d expected=%d got=%d\n",
+ i, expected, generated[i]);
+ }
+ int greedy_exact_length = got == expected_count;
+ if (!greedy_exact_length)
+ fprintf(stderr,
+ "[ORACLE] greedy length mismatch expected=%d got=%d "
+ "reference=%d requested=%d\n",
+ expected_count, got, continue_count, greedy_limit);
+ printf("GREEDY C vs oracle: %d/%d tokens\n",
+ greedy_matched, expected_count);
+ free(full_ids); free(tf_pred);
+ full_ids = NULL; tf_pred = NULL;
+ (void)process_started;
+ result = (tf_matched == tf_limit && greedy_exact_length &&
+ greedy_matched == expected_count) ? 0 : 1;
+ goto cleanup;
+ }
+
+ char *prompt = NULL;
+ size_t prompt_length = 0;
+ if (coli_v4_prompt_build(&prompt, &prompt_length, cli.prompt,
+ cli.system_prompt, cli.prompt_mode) ||
+ prompt_length > INT_MAX - 16) {
+ free(prompt);
+ fprintf(stderr, "cannot build DeepSeek V4 prompt\n");
+ goto cleanup;
+ }
+ fprintf(stderr, "v4_cli mode=%s memory=%s draft_model=%s no_dspark=%d\n",
+ cli.prompt_mode == COLI_V4_PROMPT_RAW ? "raw" :
+ cli.prompt_mode == COLI_V4_PROMPT_THINKING ? "thinking" : "chat",
+ cli.memory_gib > 0.0 ? "limited" : "auto",
+ cli.draft_model_dir, cli.no_dspark);
+
+ ColiV4SessionCreateOptions session_opts = {
+ .max_prompt_tokens = 512,
+ .max_new_tokens_cap = max_new > 512 ? max_new : 512,
+ };
+ if (coli_v4_session_create(&session, engine, &session_opts, error,
+ sizeof(error))) {
+ free(prompt);
+ fprintf(stderr, "%s\n", error);
+ goto cleanup;
+ }
+ ColiV4SessionGenerateOptions gen_opts = {
+ .max_new_tokens = max_new,
+ .stop_at_sentence = stop_sentence,
+ .no_dspark = cli.no_dspark,
+ };
+ ColiV4SessionGenerateStats gen_stats;
+ memset(&gen_stats, 0, sizeof(gen_stats));
+ if (coli_v4_session_generate(session, prompt, prompt_length, &gen_opts,
+ NULL, NULL, &gen_stats, error,
+ sizeof(error))) {
+ free(prompt);
+ fprintf(stderr, "%s\n", error);
+ goto cleanup;
+ }
+ free(prompt);
+ prompt = NULL;
+
+ char out_text[65536];
+ size_t out_len = 0;
+ coli_v4_session_generated_text(session, out_text, sizeof(out_text),
+ &out_len);
+ ColiExpertStoreStats stats_end = {0};
+ experts->ops->stats(experts, &stats_end);
+ fprintf(stderr, "v4_tokens prompt=%d generated=%d total=%d "
+ "expert_requests=%llu hits=%llu misses=%llu hit_rate=%.3f "
+ "bytes=%llu speculative_rounds=%llu proposed=%llu accepted=%llu "
+ "acceptance=%.3f enabled=%d\n",
+ gen_stats.prompt_tokens, gen_stats.generated_tokens,
+ gen_stats.prompt_tokens + gen_stats.generated_tokens,
+ (unsigned long long)stats_end.requests,
+ (unsigned long long)stats_end.hits,
+ (unsigned long long)stats_end.misses, stats_hit_rate(stats_end),
+ (unsigned long long)stats_end.bytes_read,
+ (unsigned long long)gen_stats.dspark_rounds,
+ (unsigned long long)gen_stats.dspark_proposed,
+ (unsigned long long)gen_stats.dspark_accepted,
+ gen_stats.dspark_acceptance,
+ !cli.no_dspark);
+ fprintf(stderr, "generated_text=");
+ if (out_len) fwrite(out_text, 1, out_len, stderr);
+ fprintf(stderr, "\ntiming time_to_first_token=%.3fs after_first=%.3fs\n",
+ gen_stats.time_to_first_token_sec, gen_stats.decode_sec);
+
+ /* Alias session buffers for optional record-oracle path.
+ * cleanup must destroy the session and must not free these aliases. */
+ prompt_ids = session->prompt_ids;
+ generated = session->generated;
+ attention = session->attention;
+ layers = session->config.num_hidden_layers;
+ int prompt_count = session->prompt_count;
+ int generated_count = session->generated_count;
+
+ if (cli.record_oracle_path) {
+ int full_count = prompt_count + generated_count;
+ full_ids = malloc((size_t)full_count * sizeof(int));
+ tf_pred = malloc((size_t)full_count * sizeof(int));
+ if (!full_ids || !tf_pred) goto cleanup;
+ memcpy(full_ids, prompt_ids, (size_t)prompt_count * sizeof(int));
+ memcpy(full_ids + prompt_count, generated,
+ (size_t)generated_count * sizeof(int));
+ for (int layer = 0; layer < config.num_hidden_layers; layer++)
+ coli_v4_window_attention_reset(attention[layer]);
+ /* Rebuild tf_pred for the fixture (argmax at each position). */
+ size_t hd_tf = (size_t)config.hc_mult * config.hidden_size;
+ tf_state = malloc((size_t)full_count * hd_tf * sizeof(float));
+ tf_next = malloc((size_t)full_count * hd_tf * sizeof(float));
+ tf_hidden = malloc((size_t)config.hidden_size * sizeof(float));
+ if (!tf_state || !tf_next || !tf_hidden) goto cleanup;
+ for (int item = 0; item < full_count; item++)
+ if (load_embedding(tf_state + (size_t)item * hd_tf, index, &config,
+ full_ids[item])) goto cleanup;
+ if (target_batch(engine, &tf_state, &tf_next, attention, index, &config,
+ experts, full_ids, 0, full_count, error, sizeof(error))) {
+ fprintf(stderr, "%s\n", error); goto cleanup;
+ }
+ for (int pos = 0; pos < full_count; pos++) {
+ float logit = 0.0f;
+ if (final_hidden(tf_hidden, tf_state + (size_t)pos * hd_tf,
+ index, &config, error, sizeof(error)) ||
+ head_argmax(engine, tf_hidden, index, &config, &tf_pred[pos],
+ &logit))
+ goto cleanup;
+ }
+ free(tf_state); free(tf_next); free(tf_hidden);
+ tf_state = NULL; tf_next = NULL; tf_hidden = NULL;
+ /* Chat-template prompt tokens need not be model-greedy; only score
+ * the continuation window that record actually generated. */
+ int tf_matched = 0, tf_total = generated_count;
+ for (int i = 0; i < generated_count; i++) {
+ int pos = prompt_count - 1 + i;
+ if (pos >= 0 && pos < full_count - 1 &&
+ tf_pred[pos] == full_ids[pos + 1])
+ tf_matched++;
+ }
+ if (v4_oracle_write_json(cli.record_oracle_path, "coli-self",
+ cli.model_dir, cli.prompt,
+ prompt_ids, prompt_count,
+ full_ids, full_count,
+ tf_pred, full_count)) {
+ fprintf(stderr, "cannot write oracle %s\n", cli.record_oracle_path);
+ goto cleanup;
+ }
+ fprintf(stderr,
+ "wrote oracle %s (source=coli-self, "
+ "continuation_self_check=%d/%d)\n",
+ cli.record_oracle_path, tf_matched, tf_total);
+ free(full_ids); free(tf_pred);
+ full_ids = NULL; tf_pred = NULL;
+ }
+ result = 0;
+cleanup:
+ v4_generate_cleanup(session, prompt_storage, engine, runner, attention,
+ layers, prompt_ids, generated, state, next, hidden,
+ main_x_batch, text, full_ids, tf_pred, tf_state,
+ tf_next, tf_hidden);
+ return result;
+}
+#endif /* !COLI_V4_SKIP_GENERATE_MAIN */
+
+#endif /* COLI_V4_UNIT_GENERATE_STATS */
+
+#ifdef COLI_V4_UNIT_KV_CACHE
+/* ######## deepseek_v4_kv_cache.c ######## */
+#include "deepseek_v4_internal.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+struct ColiDeepSeekV4KVCache {
+ int window_size;
+ int compression_ratio;
+ int head_dimension;
+ int compressed_capacity;
+ float *values;
+};
+
+int coli_v4_kv_cache_create(ColiDeepSeekV4KVCache **output,
+ int window_size, int compression_ratio,
+ int head_dimension, int max_context) {
+ if (!output || window_size < 1 || compression_ratio < 1 ||
+ head_dimension < 1 || max_context < 1)
+ return -1;
+ *output = NULL;
+ ColiDeepSeekV4KVCache *cache = calloc(1, sizeof(*cache));
+ if (!cache) return -1;
+ cache->window_size = window_size;
+ cache->compression_ratio = compression_ratio;
+ cache->head_dimension = head_dimension;
+ cache->compressed_capacity = max_context / compression_ratio;
+ if (cache->compressed_capacity < 1) cache->compressed_capacity = 1;
+ size_t count = (size_t)(window_size + cache->compressed_capacity) * head_dimension;
+ cache->values = calloc(count, sizeof(*cache->values));
+ if (!cache->values) {
+ free(cache);
+ return -1;
+ }
+ *output = cache;
+ return 0;
+}
+
+void coli_v4_kv_cache_reset(ColiDeepSeekV4KVCache *cache) {
+ if (!cache) return;
+ size_t count = (size_t)(cache->window_size + cache->compressed_capacity) *
+ cache->head_dimension;
+ memset(cache->values, 0, count * sizeof(*cache->values));
+}
+
+void coli_v4_kv_cache_destroy(ColiDeepSeekV4KVCache *cache) {
+ if (!cache) return;
+ free(cache->values);
+ free(cache);
+}
+
+int coli_v4_kv_cache_put_window(ColiDeepSeekV4KVCache *cache,
+ int position, const float *kv) {
+ if (!cache || !kv || position < 0) return -1;
+ int slot = position % cache->window_size;
+ memcpy(cache->values + (size_t)slot * cache->head_dimension, kv,
+ (size_t)cache->head_dimension * sizeof(*kv));
+ return slot;
+}
+
+int coli_v4_kv_cache_put_compressed(ColiDeepSeekV4KVCache *cache,
+ int position, const float *kv) {
+ if (!cache || !kv || position < 0 ||
+ (position + 1) % cache->compression_ratio != 0)
+ return -1;
+ int slot = (position + 1) / cache->compression_ratio - 1;
+ if (slot < 0 || slot >= cache->compressed_capacity) return -1;
+ int combined = cache->window_size + slot;
+ memcpy(cache->values + (size_t)combined * cache->head_dimension, kv,
+ (size_t)cache->head_dimension * sizeof(*kv));
+ return combined;
+}
+
+int coli_v4_kv_cache_indices(const ColiDeepSeekV4KVCache *cache,
+ int position, int *indices, size_t capacity) {
+ if (!cache || !indices || position < 0) return -1;
+ int compressed = (position + 1) / cache->compression_ratio;
+ if (compressed > cache->compressed_capacity) return -1;
+ size_t required = (size_t)cache->window_size + compressed;
+ if (capacity < required) return -1;
+ if (position < cache->window_size - 1) {
+ for (int i = 0; i < cache->window_size; i++)
+ indices[i] = i <= position ? i : -1;
+ } else {
+ int oldest = (position + 1) % cache->window_size;
+ for (int i = 0; i < cache->window_size; i++)
+ indices[i] = (oldest + i) % cache->window_size;
+ }
+ for (int i = 0; i < compressed; i++)
+ indices[cache->window_size + i] = cache->window_size + i;
+ return (int)required;
+}
+
+const float *coli_v4_kv_cache_values(const ColiDeepSeekV4KVCache *cache) {
+ return cache ? cache->values : NULL;
+}
+
+int coli_v4_kv_cache_value_count(const ColiDeepSeekV4KVCache *cache) {
+ return cache ? cache->window_size + cache->compressed_capacity : 0;
+}
+#endif /* COLI_V4_UNIT_KV_CACHE */
+
+#ifdef COLI_V4_UNIT_ATTENTION_CACHE
+/* ######## deepseek_v4_attention_cache.c ######## */
+#include "deepseek_v4_internal.h"
+
+#include <stdlib.h>
+
+#include "deepseek_v4_internal.h"
+#include "deepseek_v4_internal.h"
+
+struct ColiDeepSeekV4AttentionCache {
+ ColiDeepSeekV4KVCache *kv;
+ int window_size;
+ int compression_ratio;
+ int head_dimension;
+ int compressed_capacity;
+};
+
+int coli_v4_attention_cache_create(ColiDeepSeekV4AttentionCache **output,
+ int window_size, int compression_ratio,
+ int head_dimension, int max_context) {
+ if (!output) return -1;
+ *output = NULL;
+ ColiDeepSeekV4AttentionCache *cache = calloc(1, sizeof(*cache));
+ if (!cache) return -1;
+ cache->window_size = window_size;
+ cache->compression_ratio = compression_ratio;
+ cache->head_dimension = head_dimension;
+ cache->compressed_capacity = max_context / compression_ratio;
+ if (cache->compressed_capacity < 1) cache->compressed_capacity = 1;
+ if (coli_v4_kv_cache_create(&cache->kv, window_size, compression_ratio,
+ head_dimension, max_context) != 0) {
+ free(cache);
+ return -1;
+ }
+ *output = cache;
+ return 0;
+}
+
+void coli_v4_attention_cache_reset(ColiDeepSeekV4AttentionCache *cache) {
+ if (cache) coli_v4_kv_cache_reset(cache->kv);
+}
+
+void coli_v4_attention_cache_destroy(ColiDeepSeekV4AttentionCache *cache) {
+ if (!cache) return;
+ coli_v4_kv_cache_destroy(cache->kv);
+ free(cache);
+}
+
+int coli_v4_attention_cache_step(ColiDeepSeekV4AttentionCache *cache,
+ float *output, const float *query,
+ const float *window_kv,
+ const float *compressed_kv,
+ const float *sinks, int heads,
+ int position, float softmax_scale) {
+ if (!cache || !output || !query || !window_kv || !sinks || heads < 1 ||
+ position < 0 || position / cache->compression_ratio >= cache->compressed_capacity)
+ return -1;
+ int boundary = (position + 1) % cache->compression_ratio == 0;
+ if (boundary != (compressed_kv != NULL)) return -1;
+ if (coli_v4_kv_cache_put_window(cache->kv, position, window_kv) < 0)
+ return -1;
+ if (compressed_kv &&
+ coli_v4_kv_cache_put_compressed(cache->kv, position, compressed_kv) < 0)
+ return -1;
+ size_t capacity = (size_t)cache->window_size + cache->compressed_capacity;
+ int *indices = malloc(capacity * sizeof(*indices));
+ if (!indices) return -1;
+ int topk = coli_v4_kv_cache_indices(cache->kv, position, indices, capacity);
+ int result = topk < 0 ? -1 : coli_v4_sparse_attention_ref(
+ output, query, coli_v4_kv_cache_values(cache->kv), sinks, indices,
+ heads, cache->head_dimension, coli_v4_kv_cache_value_count(cache->kv),
+ topk, softmax_scale);
+ free(indices);
+ return result;
+}
+#endif /* COLI_V4_UNIT_ATTENTION_CACHE */
+
+#ifdef COLI_V4_UNIT_EXPERT
+/* ######## deepseek_v4_expert.c ######## */
+#include "deepseek_v4_internal.h"
+
+#include <stdlib.h>
+
+#include "deepseek_v4_internal.h"
+#include "native_quant.h"
+
+int coli_v4_expert_forward_ref(float *output, const ColiExpertView *expert,
+ const float *input, float route_weight,
+ float swiglu_limit) {
+ if (!output || !expert || !input || swiglu_limit < 0.0f ||
+ expert->gate.rows != expert->up.rows ||
+ expert->gate.columns != expert->up.columns ||
+ expert->down.columns != expert->gate.rows ||
+ expert->down.rows != expert->gate.columns)
+ return -1;
+ size_t intermediate = (size_t)expert->gate.rows;
+ size_t output_size = (size_t)expert->down.rows;
+ float *gate = malloc(intermediate * sizeof(*gate));
+ float *up = malloc(intermediate * sizeof(*up));
+ float *activated = malloc(intermediate * sizeof(*activated));
+ if (!gate || !up || !activated) {
+ free(gate);
+ free(up);
+ free(activated);
+ return -1;
+ }
+ int result = coli_fp4_matvec_ref(gate, &expert->gate, input) ||
+ coli_fp4_matvec_ref(up, &expert->up, input);
+ if (!result) {
+ coli_bf16_round_array(gate, intermediate);
+ coli_bf16_round_array(up, intermediate);
+ result = coli_v4_swiglu(activated, gate, up,
+ (int)intermediate, swiglu_limit);
+ }
+ if (!result) {
+ for (size_t index = 0; index < intermediate; index++)
+ activated[index] = coli_bf16_round(
+ activated[index] * route_weight);
+ result = coli_fp4_matvec_ref(output, &expert->down, activated);
+ }
+ if (!result) coli_bf16_round_array(output, output_size);
+ free(activated);
+ free(up);
+ free(gate);
+ return result ? -1 : 0;
+}
+
+int coli_v4_shared_expert_forward_ref(float *output,
+ const ColiTensorView *gate_weight,
+ const ColiTensorView *down_weight,
+ const ColiTensorView *up_weight,
+ const float *input,
+ float swiglu_limit) {
+ if (!output || !gate_weight || !down_weight || !up_weight || !input ||
+ gate_weight->format != COLI_TENSOR_FP8_E4M3_BLOCK ||
+ down_weight->format != COLI_TENSOR_FP8_E4M3_BLOCK ||
+ up_weight->format != COLI_TENSOR_FP8_E4M3_BLOCK ||
+ gate_weight->rows != up_weight->rows ||
+ gate_weight->columns != up_weight->columns ||
+ down_weight->columns != gate_weight->rows ||
+ down_weight->rows != gate_weight->columns)
+ return -1;
+ size_t intermediate = (size_t)gate_weight->rows;
+ size_t output_size = (size_t)down_weight->rows;
+ float *gate = malloc(intermediate * sizeof(*gate));
+ float *up = malloc(intermediate * sizeof(*up));
+ float *activated = malloc(intermediate * sizeof(*activated));
+ if (!gate || !up || !activated) {
+ free(activated); free(up); free(gate);
+ return -1;
+ }
+ int result = coli_fp8_matvec_ref(gate, gate_weight, input) ||
+ coli_fp8_matvec_ref(up, up_weight, input);
+ if (!result) {
+ coli_bf16_round_array(gate, intermediate);
+ coli_bf16_round_array(up, intermediate);
+ result = coli_v4_swiglu(activated, gate, up,
+ (int)intermediate, swiglu_limit);
+ }
+ if (!result) {
+ coli_bf16_round_array(activated, intermediate);
+ result = coli_fp8_matvec_ref(output, down_weight, activated);
+ }
+ if (!result) coli_bf16_round_array(output, output_size);
+ free(activated); free(up); free(gate);
+ return result ? -1 : 0;
+}
+#endif /* COLI_V4_UNIT_EXPERT */
+
+#ifdef COLI_V4_UNIT_EXPERT_STORE
+/* ######## deepseek_v4_expert_store.c ######## */
+#define _GNU_SOURCE
+#include "deepseek_v4_internal.h"
+
+#include <assert.h>
+#include <pthread.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "safetensors_index.h"
+
+#ifdef COLI_V4_EXPERIMENTAL_PREFETCH_BATCH
+int coli_st_prefetch_many(
+ const ColiSafetensorsIndex *index, const int *shards,
+ const uint64_t *offsets, const size_t *lengths, size_t count);
+#endif
+
+enum { V4_W1 = 0, V4_W2 = 1, V4_W3 = 2, V4_MATRIX_COUNT = 3 };
+
+typedef struct {
+ const ColiSafetensorsTensor *weight[V4_MATRIX_COUNT];
+ const ColiSafetensorsTensor *scale[V4_MATRIX_COUNT];
+ int shard;
+ uint64_t scale_offset;
+ uint64_t scale_bytes;
+ uint64_t weight_offset;
+ uint64_t weight_bytes;
+ uint64_t record_bytes;
+} V4ExpertRecord;
+
+typedef struct {
+ int expert;
+ unsigned references;
+ uint64_t used;
+ unsigned char *slab;
+} V4ExpertSlot;
+
+typedef struct {
+ ColiSafetensorsIndex *index;
+ int layers;
+ int experts_per_layer;
+ int slots_per_layer;
+ uint64_t record_bytes;
+ V4ExpertRecord *records;
+ V4ExpertSlot *slots;
+ uint64_t clock;
+ unsigned active_leases;
+ ColiExpertStoreStats stats;
+ pthread_mutex_t mutex;
+} V4ExpertStoreState;
+
+static int set_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list args;
+ va_start(args, format);
+ vsnprintf(error, size, format, args);
+ va_end(args);
+ }
+ return -1;
+}
+
+static int compare_tensors(const void *left, const void *right) {
+ const ColiSafetensorsTensor *const *a = left;
+ const ColiSafetensorsTensor *const *b = right;
+ return ((*a)->offset > (*b)->offset) - ((*a)->offset < (*b)->offset);
+}
+
+static int contiguous_group(const ColiSafetensorsTensor *const input[3],
+ int *shard, uint64_t *offset, uint64_t *bytes) {
+ const ColiSafetensorsTensor *parts[3] = {input[0], input[1], input[2]};
+ qsort(parts, 3, sizeof(parts[0]), compare_tensors);
+ if (parts[0]->shard != parts[1]->shard || parts[1]->shard != parts[2]->shard ||
+ parts[0]->offset + parts[0]->nbytes != parts[1]->offset ||
+ parts[1]->offset + parts[1]->nbytes != parts[2]->offset)
+ return -1;
+ *shard = parts[0]->shard;
+ *offset = parts[0]->offset;
+ *bytes = parts[2]->offset + parts[2]->nbytes - parts[0]->offset;
+ return 0;
+}
+
+static int validate_matrix(const ColiSafetensorsTensor *weight,
+ const ColiSafetensorsTensor *scale) {
+ if (!weight || !scale || weight->dtype != COLI_ST_I8 ||
+ scale->dtype != COLI_ST_F8_E8M0 || weight->rank != 2 || scale->rank != 2 ||
+ weight->shape[0] != scale->shape[0] || weight->shape[1] <= 0 ||
+ scale->shape[1] <= 0)
+ return -1;
+ int64_t logical_columns = weight->shape[1] * 2;
+ return scale->shape[1] * 32 == logical_columns ? 0 : -1;
+}
+
+static int build_record(V4ExpertStoreState *state, int layer, int expert,
+ V4ExpertRecord *record, char *error, size_t error_size) {
+ static const char *matrix_names[V4_MATRIX_COUNT] = {"w1", "w2", "w3"};
+ char name[160];
+ memset(record, 0, sizeof(*record));
+ for (int matrix = 0; matrix < V4_MATRIX_COUNT; matrix++) {
+ snprintf(name, sizeof(name), "layers.%d.ffn.experts.%d.%s.weight",
+ layer, expert, matrix_names[matrix]);
+ record->weight[matrix] = coli_st_find(state->index, name);
+ snprintf(name, sizeof(name), "layers.%d.ffn.experts.%d.%s.scale",
+ layer, expert, matrix_names[matrix]);
+ record->scale[matrix] = coli_st_find(state->index, name);
+ if (validate_matrix(record->weight[matrix], record->scale[matrix]) != 0)
+ return set_error(error, error_size,
+ "invalid native FP4 expert matrix: layer=%d expert=%d %s",
+ layer, expert, matrix_names[matrix]);
+ }
+ int scale_shard = -1, weight_shard = -1;
+ if (contiguous_group(record->scale, &scale_shard, &record->scale_offset,
+ &record->scale_bytes) != 0 ||
+ contiguous_group(record->weight, &weight_shard, &record->weight_offset,
+ &record->weight_bytes) != 0 || scale_shard != weight_shard)
+ return set_error(error, error_size,
+ "expert is not two contiguous ranges: layer=%d expert=%d",
+ layer, expert);
+ record->shard = scale_shard;
+ record->record_bytes = record->scale_bytes + record->weight_bytes;
+ return 0;
+}
+
+static V4ExpertRecord *get_record(V4ExpertStoreState *state, ColiExpertKey key) {
+ if (key.layer < 0 || key.layer >= state->layers || key.expert < 0 ||
+ key.expert >= state->experts_per_layer)
+ return NULL;
+ return &state->records[(size_t)key.layer * state->experts_per_layer + key.expert];
+}
+
+static V4ExpertSlot *layer_slots(V4ExpertStoreState *state, int layer) {
+ return state->slots + (size_t)layer * state->slots_per_layer;
+}
+
+static void fill_tensor_view(ColiTensorView *view,
+ const V4ExpertRecord *record,
+ const V4ExpertSlot *slot, int matrix) {
+ const ColiSafetensorsTensor *weight = record->weight[matrix];
+ const ColiSafetensorsTensor *scale = record->scale[matrix];
+ memset(view, 0, sizeof(*view));
+ view->format = COLI_TENSOR_FP4_NATIVE_BLOCK;
+ view->scale_format = COLI_SCALE_UE8M0;
+ view->data = slot->slab + record->scale_bytes +
+ (weight->offset - record->weight_offset);
+ view->scales = slot->slab + (scale->offset - record->scale_offset);
+ view->data_bytes = (size_t)weight->nbytes;
+ view->scale_bytes = (size_t)scale->nbytes;
+ view->rows = weight->shape[0];
+ view->columns = weight->shape[1] * 2;
+ view->block_rows = 1;
+ view->block_columns = 32;
+}
+
+static int lookup(ColiExpertStore *store, ColiExpertKey key,
+ ColiExpertView *view) {
+ if (!store || !store->state || !view) {
+ if (view) memset(view, 0, sizeof(*view));
+ return -1;
+ }
+ V4ExpertStoreState *state = store->state;
+ V4ExpertRecord *record = get_record(state, key);
+ if (!record) {
+ memset(view, 0, sizeof(*view));
+ return -1;
+ }
+ pthread_mutex_lock(&state->mutex);
+ state->stats.requests++;
+ V4ExpertSlot *slots = layer_slots(state, key.layer);
+ V4ExpertSlot *slot = NULL;
+ for (int i = 0; i < state->slots_per_layer; i++) {
+ if (slots[i].slab && slots[i].expert == key.expert) {
+ slot = &slots[i];
+ state->stats.hits++;
+ break;
+ }
+ }
+ if (!slot) {
+ for (int i = 0; i < state->slots_per_layer; i++) {
+ if (!slots[i].references && (!slot || !slots[i].slab ||
+ (slot->slab && slots[i].used < slot->used)))
+ slot = &slots[i];
+ }
+ if (!slot) {
+ pthread_mutex_unlock(&state->mutex);
+ memset(view, 0, sizeof(*view));
+ return -1;
+ }
+ if (!slot->slab) {
+ slot->slab = malloc((size_t)state->record_bytes);
+ if (!slot->slab) {
+ pthread_mutex_unlock(&state->mutex);
+ memset(view, 0, sizeof(*view));
+ return -1;
+ }
+ state->stats.resident_bytes += state->record_bytes;
+ }
+ /* A short read must never expose a partially overwritten old slot. */
+ slot->expert = -1;
+ if (coli_st_read_at(state->index, record->shard, record->scale_offset,
+ (size_t)record->scale_bytes, slot->slab) != 0 ||
+ coli_st_read_at(state->index, record->shard, record->weight_offset,
+ (size_t)record->weight_bytes,
+ slot->slab + record->scale_bytes) != 0) {
+ pthread_mutex_unlock(&state->mutex);
+ memset(view, 0, sizeof(*view));
+ return -1;
+ }
+ slot->expert = key.expert;
+ state->stats.misses++;
+ state->stats.bytes_read += record->record_bytes;
+ }
+ slot->references++;
+ state->active_leases++;
+ slot->used = ++state->clock;
+ memset(view, 0, sizeof(*view));
+ view->key = key;
+ fill_tensor_view(&view->gate, record, slot, V4_W1);
+ fill_tensor_view(&view->down, record, slot, V4_W2);
+ fill_tensor_view(&view->up, record, slot, V4_W3);
+ view->lease = slot;
+ pthread_mutex_unlock(&state->mutex);
+ return 0;
+}
+
+static void release(ColiExpertStore *store, ColiExpertView *view) {
+ if (!store || !store->state || !view || !view->lease) {
+ if (view) memset(view, 0, sizeof(*view));
+ return;
+ }
+ V4ExpertStoreState *state = store->state;
+ V4ExpertSlot *slot = view->lease;
+ pthread_mutex_lock(&state->mutex);
+ if (slot->references) slot->references--;
+ if (state->active_leases) state->active_leases--;
+ pthread_mutex_unlock(&state->mutex);
+ memset(view, 0, sizeof(*view));
+}
+
+static int prefetch(ColiExpertStore *store, const ColiExpertKey *keys,
+ size_t count) {
+ if (!store || !store->state || (!keys && count)) return 0;
+ V4ExpertStoreState *state = store->state;
+ int accepted = 0;
+#ifdef COLI_V4_EXPERIMENTAL_PREFETCH_BATCH
+ size_t capacity = count * 2, ranges = 0;
+ int *shards = malloc(capacity * sizeof(*shards));
+ uint64_t *offsets = malloc(capacity * sizeof(*offsets));
+ size_t *lengths = malloc(capacity * sizeof(*lengths));
+ int candidates = 0;
+ if ((!shards || !offsets || !lengths) && capacity) {
+ free(lengths); free(offsets); free(shards); return 0;
+ }
+ pthread_mutex_lock(&state->mutex);
+ for (size_t i = 0; i < count; i++) {
+ V4ExpertRecord *record = get_record(state, keys[i]);
+ if (!record) continue;
+ int resident = 0;
+ V4ExpertSlot *slots = layer_slots(state, keys[i].layer);
+ for (int slot = 0; slot < state->slots_per_layer; slot++)
+ if (slots[slot].slab && slots[slot].expert == keys[i].expert) {
+ resident = 1; break;
+ }
+ if (resident) continue;
+ shards[ranges] = record->shard;
+ offsets[ranges] = record->scale_offset;
+ lengths[ranges++] = (size_t)record->scale_bytes;
+ shards[ranges] = record->shard;
+ offsets[ranges] = record->weight_offset;
+ lengths[ranges++] = (size_t)record->weight_bytes;
+ candidates++;
+ }
+ pthread_mutex_unlock(&state->mutex);
+ if (candidates && !coli_st_prefetch_many(
+ state->index, shards, offsets, lengths, ranges))
+ accepted = candidates;
+ free(lengths); free(offsets); free(shards);
+#else
+ for (size_t i = 0; i < count; i++) {
+ V4ExpertRecord *record = get_record(state, keys[i]);
+ if (!record) continue;
+ if (coli_st_prefetch_at(state->index, record->shard, record->scale_offset,
+ (size_t)record->scale_bytes) == 0 &&
+ coli_st_prefetch_at(state->index, record->shard, record->weight_offset,
+ (size_t)record->weight_bytes) == 0)
+ accepted++;
+ }
+#endif
+ pthread_mutex_lock(&state->mutex);
+ state->stats.prefetched += (uint64_t)accepted;
+ pthread_mutex_unlock(&state->mutex);
+ return accepted;
+}
+
+static void stats(const ColiExpertStore *store, ColiExpertStoreStats *output) {
+ if (!store || !store->state || !output) return;
+ V4ExpertStoreState *state = store->state;
+ pthread_mutex_lock(&state->mutex);
+ *output = state->stats;
+ pthread_mutex_unlock(&state->mutex);
+}
+
+static void destroy(ColiExpertStore *store) {
+ if (!store) return;
+ V4ExpertStoreState *state = store->state;
+ if (state) {
+ assert(state->active_leases == 0 && "destroy with active expert leases");
+ for (int i = 0; i < state->layers * state->slots_per_layer; i++)
+ free(state->slots[i].slab);
+ pthread_mutex_destroy(&state->mutex);
+ coli_st_index_close(state->index);
+ free(state->records);
+ free(state->slots);
+ free(state);
+ }
+ free(store);
+}
+
+int coli_deepseek_v4_expert_store_open(
+ const ColiDeepSeekV4ExpertStoreOptions *options, ColiExpertStore **output,
+ char *error, size_t error_size) {
+ static const ColiExpertStoreOps operations = {
+ lookup, release, prefetch, stats, destroy
+ };
+ if (!options || !output || !options->model_dir || options->layers < 1 ||
+ options->experts_per_layer < 1 || !options->cache_bytes)
+ return set_error(error, error_size, "invalid DeepSeek-V4 ExpertStore options");
+ *output = NULL;
+ ColiExpertStore *store = calloc(1, sizeof(*store));
+ V4ExpertStoreState *state = calloc(1, sizeof(*state));
+ if (!store || !state) {
+ free(store);
+ free(state);
+ return set_error(error, error_size, "out of memory creating ExpertStore");
+ }
+ pthread_mutex_init(&state->mutex, NULL);
+ state->layers = options->layers;
+ state->experts_per_layer = options->experts_per_layer;
+ if (coli_st_index_open(&state->index, options->model_dir, error, error_size) != 0)
+ goto fail;
+ size_t record_count = (size_t)state->layers * state->experts_per_layer;
+ state->records = calloc(record_count, sizeof(*state->records));
+ if (!state->records) {
+ set_error(error, error_size, "out of memory creating expert manifest");
+ goto fail;
+ }
+ for (int layer = 0; layer < state->layers; layer++) {
+ for (int expert = 0; expert < state->experts_per_layer; expert++) {
+ V4ExpertRecord *record = &state->records[
+ (size_t)layer * state->experts_per_layer + expert];
+ if (build_record(state, layer, expert, record, error, error_size) != 0)
+ goto fail;
+ if (!state->record_bytes) state->record_bytes = record->record_bytes;
+ if (record->record_bytes != state->record_bytes) {
+ set_error(error, error_size, "non-uniform expert size at layer=%d expert=%d",
+ layer, expert);
+ goto fail;
+ }
+ }
+ }
+ state->slots_per_layer = (int)(options->cache_bytes /
+ ((uint64_t)state->layers * state->record_bytes));
+ int minimum_slots = state->experts_per_layer < 6
+ ? state->experts_per_layer : 6;
+ if (state->slots_per_layer < minimum_slots) {
+ set_error(error, error_size,
+ "cache budget cannot hold %d active experts per layer "
+ "(need %llu bytes)", minimum_slots,
+ (unsigned long long)((uint64_t)state->layers * minimum_slots *
+ state->record_bytes));
+ goto fail;
+ }
+ if (state->slots_per_layer > state->experts_per_layer)
+ state->slots_per_layer = state->experts_per_layer;
+ state->slots = calloc((size_t)state->layers * state->slots_per_layer,
+ sizeof(*state->slots));
+ if (!state->slots) {
+ set_error(error, error_size, "out of memory creating expert cache slots");
+ goto fail;
+ }
+ for (int i = 0; i < state->layers * state->slots_per_layer; i++)
+ state->slots[i].expert = -1;
+ state->stats.capacity_bytes = (uint64_t)state->layers *
+ state->slots_per_layer * state->record_bytes;
+ store->ops = &operations;
+ store->state = state;
+ *output = store;
+ return 0;
+
+fail:
+ if (state->slots) free(state->slots);
+ free(state->records);
+ coli_st_index_close(state->index);
+ pthread_mutex_destroy(&state->mutex);
+ free(state);
+ free(store);
+ return -1;
+}
+#endif /* COLI_V4_UNIT_EXPERT_STORE */
+
+#ifdef COLI_V4_UNIT_LAYER
+/* ######## deepseek_v4_layer.c ######## */
+#include "deepseek_v4_internal.h"
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static int set_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list arguments;
+ va_start(arguments, format);
+ vsnprintf(error, size, format, arguments);
+ va_end(arguments);
+ }
+ return -1;
+}
+
+static int add_spec(ColiDeepSeekV4LayerPlan *plan, ColiSafetensorsDType dtype,
+ int rank, const int64_t *shape, const char *suffix,
+ char *error, size_t error_size) {
+ if (plan->tensor_count >= COLI_V4_MAX_LAYER_TENSORS)
+ return set_error(error, error_size, "too many tensors in layer %d", plan->layer);
+ ColiDeepSeekV4TensorSpec *spec = &plan->tensors[plan->tensor_count++];
+ int written = snprintf(spec->name, sizeof(spec->name), "layers.%d.%s",
+ plan->layer, suffix);
+ if (written < 0 || (size_t)written >= sizeof(spec->name))
+ return set_error(error, error_size, "tensor name is too long: %s", suffix);
+ spec->dtype = dtype;
+ spec->rank = rank;
+ memcpy(spec->shape, shape, (size_t)rank * sizeof(*shape));
+ return 0;
+}
+
+static int add_1d(ColiDeepSeekV4LayerPlan *plan, ColiSafetensorsDType dtype,
+ int64_t d0, const char *name, char *error, size_t size) {
+ int64_t shape[] = {d0};
+ return add_spec(plan, dtype, 1, shape, name, error, size);
+}
+
+static int add_2d(ColiDeepSeekV4LayerPlan *plan, ColiSafetensorsDType dtype,
+ int64_t d0, int64_t d1, const char *name,
+ char *error, size_t size) {
+ int64_t shape[] = {d0, d1};
+ return add_spec(plan, dtype, 2, shape, name, error, size);
+}
+
+static int add_fp8(ColiDeepSeekV4LayerPlan *plan, int64_t rows, int64_t columns,
+ const char *prefix, char *error, size_t size) {
+ char name[128];
+ snprintf(name, sizeof(name), "%s.weight", prefix);
+ if (add_2d(plan, COLI_ST_F8_E4M3, rows, columns, name, error, size) != 0) return -1;
+ snprintf(name, sizeof(name), "%s.scale", prefix);
+ return add_2d(plan, COLI_ST_F8_E8M0, (rows + 127) / 128,
+ (columns + 127) / 128, name, error, size);
+}
+
+#define ADD(call) do { if ((call) != 0) return -1; } while (0)
+
+int coli_v4_layer_plan(ColiDeepSeekV4LayerPlan *plan,
+ const ColiDeepSeekV4Config *config, int layer,
+ char *error, size_t error_size) {
+ if (!plan || !config || layer < 0 || layer >= config->num_hidden_layers ||
+ layer >= config->compress_ratio_count)
+ return set_error(error, error_size, "invalid DeepSeek-V4 layer plan arguments");
+ memset(plan, 0, sizeof(*plan));
+ plan->layer = layer;
+ plan->compression_ratio = config->compress_ratios[layer];
+ plan->uses_hash_router = layer < config->num_hash_layers;
+ plan->has_compressor = plan->compression_ratio != 0;
+ plan->has_indexer = plan->compression_ratio == 4;
+
+ const int64_t hidden = config->hidden_size;
+ const int64_t heads = config->num_attention_heads;
+ const int64_t head_dim = config->head_dim;
+ const int64_t q_rank = config->q_lora_rank;
+ const int64_t o_width = (int64_t)config->o_groups * config->o_lora_rank;
+ const int64_t experts = config->n_routed_experts;
+ const int64_t moe = config->moe_intermediate_size;
+ const int64_t hc = config->hc_mult;
+ const int64_t hc_params = (2 + hc) * hc;
+
+ ADD(add_1d(plan, COLI_ST_F32, heads, "attn.attn_sink", error, error_size));
+ ADD(add_1d(plan, COLI_ST_BF16, head_dim, "attn.kv_norm.weight", error, error_size));
+ ADD(add_1d(plan, COLI_ST_BF16, q_rank, "attn.q_norm.weight", error, error_size));
+ ADD(add_fp8(plan, head_dim, hidden, "attn.wkv", error, error_size));
+ ADD(add_fp8(plan, o_width, hidden, "attn.wo_a", error, error_size));
+ ADD(add_fp8(plan, hidden, o_width, "attn.wo_b", error, error_size));
+ ADD(add_fp8(plan, q_rank, hidden, "attn.wq_a", error, error_size));
+ ADD(add_fp8(plan, heads * head_dim, q_rank, "attn.wq_b", error, error_size));
+ ADD(add_1d(plan, COLI_ST_BF16, hidden, "attn_norm.weight", error, error_size));
+
+ if (plan->has_compressor) {
+ int64_t ratio = plan->compression_ratio;
+ int64_t coff = ratio == 4 ? 2 : 1;
+ ADD(add_2d(plan, COLI_ST_F32, ratio, coff * head_dim,
+ "attn.compressor.ape", error, error_size));
+ ADD(add_1d(plan, COLI_ST_BF16, head_dim,
+ "attn.compressor.norm.weight", error, error_size));
+ ADD(add_2d(plan, COLI_ST_BF16, coff * head_dim, hidden,
+ "attn.compressor.wgate.weight", error, error_size));
+ ADD(add_2d(plan, COLI_ST_BF16, coff * head_dim, hidden,
+ "attn.compressor.wkv.weight", error, error_size));
+ }
+ if (plan->has_indexer) {
+ int64_t ih = config->index_head_dim;
+ int64_t in = config->index_n_heads;
+ ADD(add_2d(plan, COLI_ST_F32, 4, 2 * ih,
+ "attn.indexer.compressor.ape", error, error_size));
+ ADD(add_1d(plan, COLI_ST_BF16, ih,
+ "attn.indexer.compressor.norm.weight", error, error_size));
+ ADD(add_2d(plan, COLI_ST_BF16, 2 * ih, hidden,
+ "attn.indexer.compressor.wgate.weight", error, error_size));
+ ADD(add_2d(plan, COLI_ST_BF16, 2 * ih, hidden,
+ "attn.indexer.compressor.wkv.weight", error, error_size));
+ ADD(add_2d(plan, COLI_ST_BF16, in, hidden,
+ "attn.indexer.weights_proj.weight", error, error_size));
+ ADD(add_fp8(plan, in * ih, q_rank, "attn.indexer.wq_b", error, error_size));
+ }
+
+ ADD(add_2d(plan, COLI_ST_BF16, experts, hidden,
+ "ffn.gate.weight", error, error_size));
+ if (plan->uses_hash_router)
+ ADD(add_2d(plan, COLI_ST_I64, config->vocab_size,
+ config->num_experts_per_tok, "ffn.gate.tid2eid", error, error_size));
+ else
+ ADD(add_1d(plan, COLI_ST_F32, experts, "ffn.gate.bias", error, error_size));
+ ADD(add_fp8(plan, moe, hidden, "ffn.shared_experts.w1", error, error_size));
+ ADD(add_fp8(plan, hidden, moe, "ffn.shared_experts.w2", error, error_size));
+ ADD(add_fp8(plan, moe, hidden, "ffn.shared_experts.w3", error, error_size));
+ ADD(add_1d(plan, COLI_ST_BF16, hidden, "ffn_norm.weight", error, error_size));
+
+ ADD(add_1d(plan, COLI_ST_F32, hc_params, "hc_attn_base", error, error_size));
+ ADD(add_2d(plan, COLI_ST_F32, hc_params, hc * hidden,
+ "hc_attn_fn", error, error_size));
+ ADD(add_1d(plan, COLI_ST_F32, 3, "hc_attn_scale", error, error_size));
+ ADD(add_1d(plan, COLI_ST_F32, hc_params, "hc_ffn_base", error, error_size));
+ ADD(add_2d(plan, COLI_ST_F32, hc_params, hc * hidden,
+ "hc_ffn_fn", error, error_size));
+ ADD(add_1d(plan, COLI_ST_F32, 3, "hc_ffn_scale", error, error_size));
+ return 0;
+}
+
+int coli_v4_layer_validate(const ColiDeepSeekV4LayerPlan *plan,
+ const ColiSafetensorsIndex *index,
+ ColiDeepSeekV4LayerStats *stats,
+ char *error, size_t error_size) {
+ if (!plan || !index)
+ return set_error(error, error_size, "invalid DeepSeek-V4 layer validation arguments");
+ ColiDeepSeekV4LayerStats local = {0};
+ for (size_t i = 0; i < plan->tensor_count; i++) {
+ const ColiDeepSeekV4TensorSpec *spec = &plan->tensors[i];
+ const ColiSafetensorsTensor *tensor = coli_st_find(index, spec->name);
+ if (!tensor)
+ return set_error(error, error_size, "missing tensor: %s", spec->name);
+ if (tensor->dtype != spec->dtype || tensor->rank != spec->rank)
+ return set_error(error, error_size, "dtype/rank mismatch: %s", spec->name);
+ for (int dimension = 0; dimension < spec->rank; dimension++)
+ if (tensor->shape[dimension] != spec->shape[dimension])
+ return set_error(error, error_size, "shape mismatch: %s", spec->name);
+ local.tensor_count++;
+ local.total_bytes += tensor->nbytes;
+ switch (tensor->dtype) {
+ case COLI_ST_BF16: local.bf16_bytes += tensor->nbytes; break;
+ case COLI_ST_F32: local.f32_bytes += tensor->nbytes; break;
+ case COLI_ST_F8_E4M3: local.fp8_weight_bytes += tensor->nbytes; break;
+ case COLI_ST_F8_E8M0: local.fp8_scale_bytes += tensor->nbytes; break;
+ case COLI_ST_I64: local.i64_bytes += tensor->nbytes; break;
+ default: break;
+ }
+ }
+ if (stats) *stats = local;
+ return 0;
+}
+
+void coli_v4_layer_free(ColiV4Engine *engine,
+ ColiDeepSeekV4LayerWeights *weights) {
+ (void)engine;
+ if (!weights) return;
+ for (size_t i = 0; i < weights->plan.tensor_count; i++) free(weights->data[i]);
+ memset(weights, 0, sizeof(*weights));
+}
+
+int coli_v4_layer_load(ColiV4Engine *engine,
+ ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const ColiSafetensorsIndex *index, int layer,
+ char *error, size_t error_size) {
+ (void)engine;
+ if (!weights) return set_error(error, error_size, "missing layer weights output");
+ memset(weights, 0, sizeof(*weights));
+ if (coli_v4_layer_plan(&weights->plan, config, layer, error, error_size) != 0 ||
+ coli_v4_layer_validate(&weights->plan, index, &weights->stats,
+ error, error_size) != 0)
+ return -1;
+ for (size_t i = 0; i < weights->plan.tensor_count; i++) {
+ const ColiDeepSeekV4TensorSpec *spec = &weights->plan.tensors[i];
+ const ColiSafetensorsTensor *tensor = coli_st_find(index, spec->name);
+ weights->data[i] = malloc((size_t)tensor->nbytes);
+ if (!weights->data[i]) {
+ coli_v4_layer_free(NULL, weights);
+ return set_error(error, error_size, "out of memory loading: %s", spec->name);
+ }
+ if (coli_st_read_tensor(index, tensor, weights->data[i]) != 0) {
+ coli_v4_layer_free(NULL, weights);
+ return set_error(error, error_size, "cannot read tensor: %s", spec->name);
+ }
+ }
+ return 0;
+}
+
+const void *coli_v4_layer_data(const ColiDeepSeekV4LayerWeights *weights,
+ const char *name,
+ const ColiDeepSeekV4TensorSpec **spec) {
+ if (spec) *spec = NULL;
+ if (!weights || !name) return NULL;
+ for (size_t i = 0; i < weights->plan.tensor_count; i++) {
+ if (strcmp(weights->plan.tensors[i].name, name) == 0) {
+ if (spec) *spec = &weights->plan.tensors[i];
+ return weights->data[i];
+ }
+ }
+ return NULL;
+}
+#endif /* COLI_V4_UNIT_LAYER */
+
+#ifdef COLI_V4_UNIT_CONFIG
+/* ######## deepseek_v4_config.c ######## */
+#include "deepseek_v4_internal.h"
+
+#include <float.h>
+#include <limits.h>
+#include <math.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "json.h"
+
+static int set_error(char *error, size_t size, const char *format, ...) {
+ if (error && size) {
+ va_list arguments;
+ va_start(arguments, format);
+ vsnprintf(error, size, format, arguments);
+ va_end(arguments);
+ }
+ return -1;
+}
+
+static int json_int_value(const jval *value, int *output) {
+ if (!value || value->t != J_NUM || !isfinite(value->num) ||
+ floor(value->num) != value->num ||
+ value->num < (double)INT_MIN || value->num > (double)INT_MAX)
+ return -1;
+ *output = (int)value->num;
+ return 0;
+}
+
+static int required_int(jval *root, const char *name, int *output,
+ char *error, size_t error_size) {
+ if (json_int_value(json_get(root, name), output) != 0)
+ return set_error(error, error_size, "invalid integer config field: %s", name);
+ return 0;
+}
+
+static int required_float(jval *root, const char *name, float *output,
+ char *error, size_t error_size) {
+ jval *value = json_get(root, name);
+ if (!value || value->t != J_NUM || !isfinite(value->num) ||
+ fabs(value->num) > (double)FLT_MAX)
+ return set_error(error, error_size, "invalid numeric config field: %s", name);
+ *output = (float)value->num;
+ return 0;
+}
+
+static int require_string(jval *root, const char *name, const char *expected,
+ char *error, size_t error_size) {
+ jval *value = json_get(root, name);
+ if (!value || value->t != J_STR || strcmp(value->str, expected))
+ return set_error(error, error_size, "unsupported %s (expected %s)",
+ name, expected);
+ return 0;
+}
+
+int coli_v4_config_parse(ColiDeepSeekV4Config *config, const char *json,
+ char *error, size_t error_size) {
+ if (!config || !json)
+ return set_error(error, error_size, "invalid DeepSeek-V4 config arguments");
+ memset(config, 0, sizeof(*config));
+ char *arena = NULL;
+ jval *root = json_parse(json, &arena);
+ if (!root || root->t != J_OBJ) {
+ json_free(root);
+ free(arena);
+ return set_error(error, error_size, "DeepSeek-V4 config is not an object");
+ }
+ int failed =
+ require_string(root, "model_type", "deepseek_v4", error, error_size) ||
+ require_string(root, "expert_dtype", "fp4", error, error_size) ||
+ require_string(root, "scoring_func", "sqrtsoftplus", error, error_size) ||
+ require_string(root, "topk_method", "noaux_tc", error, error_size) ||
+ required_int(root, "hidden_size", &config->hidden_size, error, error_size) ||
+ required_int(root, "num_hidden_layers", &config->num_hidden_layers, error, error_size) ||
+ required_int(root, "num_attention_heads", &config->num_attention_heads, error, error_size) ||
+ required_int(root, "head_dim", &config->head_dim, error, error_size) ||
+ required_int(root, "q_lora_rank", &config->q_lora_rank, error, error_size) ||
+ required_int(root, "qk_rope_head_dim", &config->qk_rope_head_dim, error, error_size) ||
+ required_int(root, "o_groups", &config->o_groups, error, error_size) ||
+ required_int(root, "o_lora_rank", &config->o_lora_rank, error, error_size) ||
+ required_int(root, "sliding_window", &config->sliding_window, error, error_size) ||
+ required_int(root, "index_n_heads", &config->index_n_heads, error, error_size) ||
+ required_int(root, "index_head_dim", &config->index_head_dim, error, error_size) ||
+ required_int(root, "index_topk", &config->index_topk, error, error_size) ||
+ required_int(root, "n_routed_experts", &config->n_routed_experts, error, error_size) ||
+ required_int(root, "num_experts_per_tok", &config->num_experts_per_tok, error, error_size) ||
+ required_int(root, "n_shared_experts", &config->n_shared_experts, error, error_size) ||
+ required_int(root, "moe_intermediate_size", &config->moe_intermediate_size, error, error_size) ||
+ required_int(root, "num_hash_layers", &config->num_hash_layers, error, error_size) ||
+ required_int(root, "num_nextn_predict_layers", &config->num_nextn_predict_layers, error, error_size) ||
+ required_int(root, "hc_mult", &config->hc_mult, error, error_size) ||
+ required_int(root, "hc_sinkhorn_iters", &config->hc_sinkhorn_iters, error, error_size) ||
+ required_int(root, "vocab_size", &config->vocab_size, error, error_size) ||
+ required_int(root, "max_position_embeddings", &config->max_position_embeddings, error, error_size) ||
+ required_float(root, "rms_norm_eps", &config->rms_norm_eps, error, error_size) ||
+ required_float(root, "hc_eps", &config->hc_eps, error, error_size) ||
+ required_float(root, "routed_scaling_factor", &config->routed_scaling_factor, error, error_size) ||
+ required_float(root, "swiglu_limit", &config->swiglu_limit, error, error_size) ||
+ required_float(root, "rope_theta", &config->rope_theta, error, error_size) ||
+ required_float(root, "compress_rope_theta", &config->compress_rope_theta, error, error_size);
+ if (failed) {
+ json_free(root);
+ free(arena);
+ return -1;
+ }
+ jval *rope = json_get(root, "rope_scaling");
+ if (!rope || rope->t != J_OBJ ||
+ required_int(rope, "original_max_position_embeddings",
+ &config->original_max_position_embeddings, error, error_size) ||
+ required_int(rope, "beta_fast", &config->rope_beta_fast, error, error_size) ||
+ required_int(rope, "beta_slow", &config->rope_beta_slow, error, error_size) ||
+ required_float(rope, "factor", &config->rope_factor, error, error_size)) {
+ json_free(root);
+ free(arena);
+ return -1;
+ }
+ jval *ratios = json_get(root, "compress_ratios");
+ if (!ratios || ratios->t != J_ARR || ratios->len < 1 ||
+ ratios->len > COLI_V4_MAX_LAYERS) {
+ json_free(root);
+ free(arena);
+ return set_error(error, error_size, "invalid compress_ratios");
+ }
+ config->compress_ratio_count = ratios->len;
+ for (int index = 0; index < ratios->len; index++) {
+ if (json_int_value(ratios->kids[index],
+ &config->compress_ratios[index]) != 0) {
+ json_free(root);
+ free(arena);
+ return set_error(error, error_size, "invalid compress ratio");
+ }
+ }
+ jval *quantization = json_get(root, "quantization_config");
+ if (!quantization || quantization->t != J_OBJ ||
+ require_string(quantization, "fmt", "e4m3", error, error_size) ||
+ require_string(quantization, "scale_fmt", "ue8m0", error, error_size)) {
+ json_free(root);
+ free(arena);
+ return -1;
+ }
+ if (config->hidden_size < 1 || config->num_hidden_layers < 1 ||
+ config->num_attention_heads < 1 || config->n_routed_experts < 1 ||
+ config->num_experts_per_tok < 1 ||
+ config->num_experts_per_tok > config->n_routed_experts ||
+ config->n_shared_experts != 1 || config->hc_mult < 1 ||
+ config->compress_ratio_count != config->num_hidden_layers +
+ config->num_nextn_predict_layers) {
+ json_free(root);
+ free(arena);
+ return set_error(error, error_size, "inconsistent DeepSeek-V4 config dimensions");
+ }
+ json_free(root);
+ free(arena);
+ return 0;
+}
+
+int coli_v4_config_load(ColiDeepSeekV4Config *config, const char *model_dir,
+ char *error, size_t error_size) {
+ if (!config || !model_dir)
+ return set_error(error, error_size, "invalid DeepSeek-V4 config path");
+ size_t path_length = strlen(model_dir) + sizeof("/config.json");
+ char *path = malloc(path_length);
+ if (!path) return set_error(error, error_size, "out of memory building config path");
+ snprintf(path, path_length, "%s/config.json", model_dir);
+ FILE *stream = fopen(path, "rb");
+ if (!stream) {
+ int result = set_error(error, error_size, "cannot open %s", path);
+ free(path);
+ return result;
+ }
+ fseek(stream, 0, SEEK_END);
+ long length = ftell(stream);
+ rewind(stream);
+ if (length < 1) {
+ fclose(stream);
+ free(path);
+ return set_error(error, error_size, "empty config: %s", model_dir);
+ }
+ char *text = malloc((size_t)length + 1);
+ if (!text || fread(text, 1, (size_t)length, stream) != (size_t)length) {
+ free(text);
+ fclose(stream);
+ free(path);
+ return set_error(error, error_size, "cannot read config: %s", model_dir);
+ }
+ text[length] = 0;
+ fclose(stream);
+ int result = coli_v4_config_parse(config, text, error, error_size);
+ free(text);
+ free(path);
+ return result;
+}
+#endif /* COLI_V4_UNIT_CONFIG */
diff --git a/c/deepseek_v4.h b/c/deepseek_v4.h
new file mode 100644
index 00000000..8fae70f4
--- /dev/null
+++ b/c/deepseek_v4.h
@@ -0,0 +1,164 @@
+#ifndef COLIBRI_DEEPSEEK_V4_H
+#define COLIBRI_DEEPSEEK_V4_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* ---- config ---- */
+
+#define COLI_V4_MAX_LAYERS 128
+
+typedef struct {
+ int hidden_size;
+ int num_hidden_layers;
+ int num_attention_heads;
+ int head_dim;
+ int q_lora_rank;
+ int qk_rope_head_dim;
+ int o_groups;
+ int o_lora_rank;
+ int sliding_window;
+ int index_n_heads;
+ int index_head_dim;
+ int index_topk;
+ int n_routed_experts;
+ int num_experts_per_tok;
+ int n_shared_experts;
+ int moe_intermediate_size;
+ int num_hash_layers;
+ int num_nextn_predict_layers;
+ int hc_mult;
+ int hc_sinkhorn_iters;
+ int vocab_size;
+ int max_position_embeddings;
+ int original_max_position_embeddings;
+ int compress_ratio_count;
+ int compress_ratios[COLI_V4_MAX_LAYERS];
+ float rms_norm_eps;
+ float hc_eps;
+ float routed_scaling_factor;
+ float swiglu_limit;
+ float rope_theta;
+ float rope_factor;
+ float compress_rope_theta;
+ int rope_beta_fast;
+ int rope_beta_slow;
+} ColiDeepSeekV4Config;
+
+int coli_v4_config_parse(ColiDeepSeekV4Config *config, const char *json,
+ char *error, size_t error_size);
+int coli_v4_config_load(ColiDeepSeekV4Config *config, const char *model_dir,
+ char *error, size_t error_size);
+
+/* ---- prompt ---- */
+
+typedef enum {
+ COLI_V4_PROMPT_CHAT,
+ COLI_V4_PROMPT_THINKING,
+ COLI_V4_PROMPT_RAW,
+} ColiDeepSeekV4PromptMode;
+
+int coli_v4_prompt_build(char **output, size_t *output_length,
+ const char *user_message, const char *system_message,
+ ColiDeepSeekV4PromptMode mode);
+
+/* ---- experimental public engine API ---- */
+/*
+ * The API is scoped to the DeepSeek V4 engine and may change while
+ * the implementation remains experimental.
+ */
+
+typedef struct ColiV4Engine ColiV4Engine;
+
+typedef struct {
+ /* Copied by coli_v4_engine_open; caller strings need not outlive the engine. */
+ const char *target_model_dir; /* required */
+ const char *dspark_model_dir; /* NULL => same as target */
+ uint64_t memory_limit_bytes; /* 0 => use OS available memory */
+ int context_tokens; /* 0 => 4096 */
+ int verify_drafts; /* 0 => runner default */
+ int pin_slots_per_layer; /* -1 => auto */
+ uint64_t repin_interval; /* 0 => auto */
+} ColiV4EngineOpenOptions;
+
+typedef struct {
+ uint64_t projected_bytes;
+ uint64_t expert_cache_bytes;
+ int slots_per_layer;
+ int dense_resident;
+ int dspark_resident;
+ int head_resident;
+} ColiV4EngineMemorySummary;
+
+int coli_v4_engine_open(ColiV4Engine **engine,
+ const ColiV4EngineOpenOptions *options,
+ char *error, size_t error_size);
+/* Undefined if any ColiV4Session created from this engine is still alive.
+ * Destroy every session before destroying the engine. */
+void coli_v4_engine_destroy(ColiV4Engine *engine);
+
+const ColiDeepSeekV4Config *coli_v4_engine_config(const ColiV4Engine *engine);
+void coli_v4_engine_memory_summary(const ColiV4Engine *engine,
+ ColiV4EngineMemorySummary *summary);
+
+const char *coli_v4_engine_target_model_dir(const ColiV4Engine *engine);
+const char *coli_v4_engine_dspark_model_dir(const ColiV4Engine *engine);
+
+/* ---- experimental public session API ----
+ * Session borrows the engine; destroy all sessions before coli_v4_engine_destroy.
+ */
+
+typedef struct ColiV4Session ColiV4Session;
+
+typedef struct {
+ int max_prompt_tokens; /* 0 => 512 */
+ int max_new_tokens_cap; /* 0 => 512 */
+} ColiV4SessionCreateOptions;
+
+typedef struct {
+ int max_new_tokens; /* required; clamped by session cap */
+ int stop_at_sentence;
+ int no_dspark;
+} ColiV4SessionGenerateOptions;
+
+typedef struct {
+ int prompt_tokens;
+ int generated_tokens;
+ int eos_stopped;
+ uint64_t dspark_rounds;
+ uint64_t dspark_proposed;
+ uint64_t dspark_accepted;
+ float dspark_acceptance;
+ double time_to_first_token_sec;
+ double decode_sec;
+} ColiV4SessionGenerateStats;
+
+/* Return non-zero to stop generation. */
+typedef int (*ColiV4SessionTokenFn)(void *user_data, int token, float logit,
+ int position, int ordinal);
+
+int coli_v4_session_create(ColiV4Session **session, ColiV4Engine *engine,
+ const ColiV4SessionCreateOptions *options,
+ char *error, size_t error_size);
+void coli_v4_session_destroy(ColiV4Session *session);
+
+int coli_v4_session_generate(ColiV4Session *session,
+ const char *prompt, size_t prompt_length,
+ const ColiV4SessionGenerateOptions *options,
+ ColiV4SessionTokenFn on_token, void *user_data,
+ ColiV4SessionGenerateStats *stats,
+ char *error, size_t error_size);
+
+int coli_v4_session_generated_text(const ColiV4Session *session,
+ char *buffer, size_t buffer_size,
+ size_t *out_length);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* COLIBRI_DEEPSEEK_V4_H */
diff --git a/c/deepseek_v4_dspark.h b/c/deepseek_v4_dspark.h
new file mode 100644
index 00000000..452aed84
--- /dev/null
+++ b/c/deepseek_v4_dspark.h
@@ -0,0 +1,358 @@
+#ifndef COLIBRI_DEEPSEEK_V4_DSPARK_H
+#define COLIBRI_DEEPSEEK_V4_DSPARK_H
+
+#include "deepseek_v4_internal.h"
+
+#include "tensor.h"
+#include "expert_store.h"
+#include "native_quant.h"
+#include "native_quant_batch.h"
+#include "native_quant_dual.h"
+#include "native_quant_fp4_rows16.h"
+#include "safetensors_index.h"
+#include "tensor_io.h"
+
+/* ==== begin deepseek_v4_dspark.h ==== */
+
+#include <stddef.h>
+#include <stdint.h>
+
+/* via deepseek_v4.h: deepseek_v4_config.h */
+/* via deepseek_v4.h: deepseek_v4_layer.h */
+
+/* COLI_V4_DSPARK_MAX_* and ColiDeepSeekV4DSparkManifest: deepseek_v4_internal.h */
+
+int coli_v4_dspark_inspect(const char *model_dir,
+ const ColiDeepSeekV4Config *config,
+ ColiDeepSeekV4DSparkManifest *manifest,
+ char *error, size_t error_size);
+int coli_v4_dspark_layer_plan(ColiDeepSeekV4LayerPlan *plan,
+ const ColiDeepSeekV4Config *config, int stage,
+ char *error, size_t error_size);
+/* ==== end deepseek_v4_dspark.h ==== */
+
+/* ==== begin deepseek_v4_dspark_memory.h ==== */
+
+#include <stddef.h>
+#include <stdint.h>
+
+/* via deepseek_v4.h: deepseek_v4_config.h */
+
+typedef struct {
+ uint64_t resident_heads_bytes;
+ uint64_t streamed_stage_bytes;
+ uint64_t minimum_expert_cache_bytes;
+ uint64_t working_bytes;
+ uint64_t incremental_reserve_bytes;
+ uint64_t expert_record_bytes;
+ int stages;
+ int expert_slots_per_stage;
+} ColiV4DSparkMemoryPlan;
+
+int coli_v4_dspark_memory_plan(const char *model_dir,
+ const ColiDeepSeekV4Config *config,
+ ColiV4DSparkMemoryPlan *plan,
+ char *error, size_t error_size);
+/* ==== end deepseek_v4_dspark_memory.h ==== */
+
+/* ==== begin deepseek_v4_dspark_runtime.h ==== */
+
+/* amalgamated: deepseek_v4_dspark.h */
+/* via deepseek_v4.h: deepseek_v4_expert_store.h */
+
+int coli_v4_dspark_layer_load(ColiV4Engine *engine,
+ ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const ColiSafetensorsIndex *index, int stage,
+ char *error, size_t error_size);
+void coli_v4_dspark_layer_release(ColiV4Engine *engine,
+ ColiDeepSeekV4LayerWeights *weights);
+int coli_deepseek_v4_dspark_expert_store_open(
+ const ColiDeepSeekV4ExpertStoreOptions *options,
+ ColiExpertStore **store, char *error, size_t error_size);
+/* ==== end deepseek_v4_dspark_runtime.h ==== */
+
+/* ==== begin deepseek_v4_dspark_runtime_resident.h ==== */
+
+/* via deepseek_v4.h: deepseek_v4_layer.h */
+/* coli_v4_dspark_layer_release declared above with engine parameter */
+
+/* ==== end deepseek_v4_dspark_runtime_resident.h ==== */
+
+/* ==== begin deepseek_v4_dspark_attention.h ==== */
+
+#include <stddef.h>
+
+/* via deepseek_v4.h: deepseek_v4_config.h */
+/* via deepseek_v4.h: deepseek_v4_layer.h */
+
+typedef struct ColiV4DSparkAttentionState ColiV4DSparkAttentionState;
+
+/*
+ * Record an absolute token position in a sparse fixed-size sliding window.
+ * Positions may jump when one speculative verification commits several
+ * tokens, so occupancy cannot be inferred from the number of writes.
+ */
+static inline int coli_v4_dspark_position_window_put(
+ int *positions, int window_size, int *valid, int position) {
+ if (!positions || !valid || window_size < 1 || position < 0) return -1;
+ int64_t minimum = (int64_t)position - window_size + 1;
+ for (int slot = 0; slot < window_size; slot++)
+ if (positions[slot] >= 0 && (int64_t)positions[slot] < minimum)
+ positions[slot] = -1;
+ int slot = position % window_size;
+ positions[slot] = position;
+ int occupied = 0;
+ for (int index = 0; index < window_size; index++)
+ if (positions[index] >= 0) occupied++;
+ *valid = occupied;
+ return slot;
+}
+
+int coli_v4_dspark_attention_create(ColiV4DSparkAttentionState **state,
+ const ColiDeepSeekV4Config *config);
+void coli_v4_dspark_attention_reset(ColiV4DSparkAttentionState *state);
+void coli_v4_dspark_attention_destroy(ColiV4DSparkAttentionState *state);
+int coli_v4_dspark_attention_context_count(
+ const ColiV4DSparkAttentionState *state);
+int coli_v4_dspark_attention_precompute_context(
+ ColiV4DSparkAttentionState *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const float *main_x, int start_position, int batch,
+ char *error, size_t error_size);
+/* ==== end deepseek_v4_dspark_attention.h ==== */
+
+/* ==== begin deepseek_v4_dspark_attention_block.h ==== */
+
+/* amalgamated: deepseek_v4_dspark_attention.h */
+
+int coli_v4_dspark_attention_block(
+ float *outputs, ColiV4DSparkAttentionState *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, const float *inputs,
+ int query_start_position, int batch, char *error, size_t error_size);
+/* ==== end deepseek_v4_dspark_attention_block.h ==== */
+
+/* ==== begin deepseek_v4_dspark_block.h ==== */
+
+/* amalgamated: deepseek_v4_dspark_attention.h */
+/* via deepseek_v4.h: deepseek_v4_expert_store.h */
+
+int coli_v4_dspark_block(
+ float *outputs_hc, ColiV4DSparkAttentionState *attention,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, ColiExpertStore *experts,
+ const float *inputs_hc, const int *tokens,
+ int query_start_position, int batch,
+ char *error, size_t error_size);
+/* ==== end deepseek_v4_dspark_block.h ==== */
+
+/* ==== begin deepseek_v4_dspark_final.h ==== */
+
+/* via deepseek_v4.h: deepseek_v4_config.h */
+/* amalgamated: deepseek_v4_dspark.h */
+
+typedef struct ColiV4DSparkFinal ColiV4DSparkFinal;
+
+int coli_v4_dspark_final_open(ColiV4DSparkFinal **output,
+ const char *model_dir,
+ const ColiDeepSeekV4Config *config,
+ const ColiDeepSeekV4DSparkManifest *manifest,
+ char *error, size_t error_size);
+void coli_v4_dspark_final_close(ColiV4DSparkFinal *final);
+int coli_v4_dspark_final_hidden(ColiV4DSparkFinal *final,
+ float *outputs, const float *states_hc,
+ int batch);
+/* ==== end deepseek_v4_dspark_final.h ==== */
+
+/* ==== begin deepseek_v4_dspark_heads.h ==== */
+
+#include <stddef.h>
+
+/* via deepseek_v4.h: deepseek_v4_config.h */
+/* amalgamated: deepseek_v4_dspark.h */
+#include "safetensors_index.h"
+
+typedef struct ColiV4DSparkHeads ColiV4DSparkHeads;
+
+int coli_v4_dspark_heads_open(ColiV4DSparkHeads **output,
+ const char *model_dir,
+ const ColiDeepSeekV4Config *config,
+ const ColiDeepSeekV4DSparkManifest *manifest,
+ char *error, size_t error_size);
+void coli_v4_dspark_heads_close(ColiV4DSparkHeads *heads);
+int coli_v4_dspark_combine_hidden(ColiV4DSparkHeads *heads, float *output,
+ const float *target_hidden_hc);
+int coli_v4_dspark_markov_bias(ColiV4DSparkHeads *heads, float *bias,
+ int previous_token);
+int coli_v4_dspark_biased_argmax(
+ ColiV4Engine *engine, ColiV4DSparkHeads *heads,
+ const ColiSafetensorsIndex *target_index, const float *hidden,
+ int previous_token, int *best_token, float *best_logit);
+int coli_v4_dspark_biased_argmax_batch(
+ ColiV4Engine *engine, ColiV4DSparkHeads *heads,
+ const ColiSafetensorsIndex *target_index, const float *hidden_batch,
+ int previous_token, int *best_tokens, float *best_logits, int batch);
+/* ==== end deepseek_v4_dspark_heads.h ==== */
+
+/* ==== begin deepseek_v4_dspark_runner.h ==== */
+
+#include <stddef.h>
+#include <stdint.h>
+
+/* via deepseek_v4.h: deepseek_v4_config.h */
+
+typedef struct ColiV4DSparkRunner ColiV4DSparkRunner;
+
+int coli_v4_dspark_runner_open(ColiV4DSparkRunner **output,
+ ColiV4Engine *engine,
+ const char *dspark_model_dir,
+ const char *target_model_dir,
+ const ColiDeepSeekV4Config *config,
+ uint64_t expert_cache_bytes,
+ char *error, size_t error_size);
+void coli_v4_dspark_runner_close(ColiV4DSparkRunner *runner);
+int coli_v4_dspark_runner_prefill(ColiV4DSparkRunner *runner,
+ const float *main_x,
+ int start_position, int batch,
+ char *error, size_t error_size);
+int coli_v4_dspark_runner_draft(ColiV4DSparkRunner *runner,
+ const float *main_x, int anchor_token,
+ int position, int *draft_tokens,
+ float *draft_logits,
+ char *error, size_t error_size);
+int coli_v4_dspark_runner_block_size(const ColiV4DSparkRunner *runner);
+/* Verify-window draft count (min of manifest.block_size and runtime.verify_drafts). */
+int coli_v4_dspark_runner_verify_block_size(const ColiV4DSparkRunner *runner);
+uint64_t coli_v4_dspark_runner_loaded_stage_peak(
+ const ColiV4DSparkRunner *runner);
+/* ==== end deepseek_v4_dspark_runner.h ==== */
+
+/* ==== begin deepseek_v4_dspark_runner_shared.h ==== */
+
+/* amalgamated: deepseek_v4_dspark_heads.h */
+/* amalgamated: deepseek_v4_dspark_runner.h */
+
+int coli_v4_dspark_runner_use_shared_heads(ColiV4DSparkRunner *runner,
+ ColiV4DSparkHeads *heads);
+/* ==== end deepseek_v4_dspark_runner_shared.h ==== */
+
+/* ==== begin deepseek_v4_dspark_capture.h ==== */
+
+/* via deepseek_v4.h: deepseek_v4_config.h */
+/* amalgamated: deepseek_v4_dspark_heads.h */
+
+int coli_v4_dspark_capture_main_x(ColiV4Engine *engine, float *outputs,
+ int batch,
+ const ColiDeepSeekV4Config *config);
+ColiV4DSparkHeads *coli_v4_dspark_capture_heads(ColiV4Engine *engine);
+int coli_v4_dspark_capture_stage_main_x(ColiV4Engine *engine,
+ const float *values, int batch,
+ int hidden_size);
+void coli_v4_dspark_capture_after_block(
+ ColiV4Engine *engine, const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, const float *outputs, int batch);
+/* ==== end deepseek_v4_dspark_capture.h ==== */
+
+/* ==== begin deepseek_v4_speculative.h ==== */
+
+#include <stdint.h>
+
+typedef struct {
+ uint64_t rounds;
+ uint64_t proposed;
+ uint64_t accepted;
+ uint64_t minimum_proposals;
+ float disable_threshold;
+ int enabled;
+} ColiV4SpeculativeController;
+
+typedef struct {
+ int accepted_draft_tokens;
+ int output_count;
+ int mismatch_index;
+} ColiV4VerificationResult;
+
+void coli_v4_speculative_controller_init(ColiV4SpeculativeController *state,
+ uint64_t minimum_proposals,
+ float disable_threshold);
+void coli_v4_speculative_record(ColiV4SpeculativeController *state,
+ int proposed, int accepted);
+float coli_v4_speculative_acceptance(const ColiV4SpeculativeController *state);
+
+int coli_v4_verify_greedy(ColiV4VerificationResult *result,
+ int *output_tokens, int output_capacity,
+ const int *draft_tokens, int draft_count,
+ const int *target_tokens, int target_count);
+/* ==== end deepseek_v4_speculative.h ==== */
+
+/* ==== begin deepseek_v4_target_verify.h ==== */
+
+/* via deepseek_v4.h: deepseek_v4_attention.h */
+/* via deepseek_v4.h: deepseek_v4_config.h */
+/* via deepseek_v4.h: deepseek_v4_expert_store.h */
+/* amalgamated: deepseek_v4_speculative.h */
+
+int coli_v4_target_verify_greedy_batch(
+ ColiV4VerificationResult *verification,
+ int *output_tokens, int output_capacity,
+ ColiV4Engine *engine,
+ ColiDeepSeekV4WindowAttentionState **attention,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config, ColiExpertStore *experts,
+ int anchor_token, const int *draft_tokens, int draft_count,
+ int start_position, char *error, size_t error_size);
+/* ==== end deepseek_v4_target_verify.h ==== */
+
+/* ==== begin deepseek_v4_target_verify_prefix.h ==== */
+
+/* amalgamated: deepseek_v4_target_verify.h */
+
+/* The target has already processed the anchor and draft_tokens[0] matched its
+ * prediction. Verify drafts 1..3 with the same 2+2 schedule as the prior
+ * verifier, while preserving the prefix main_x row for the next DSpark round. */
+int coli_v4_target_verify_after_prefix_v69(
+ ColiV4VerificationResult *verification,
+ int *output_tokens, int output_capacity,
+ ColiV4Engine *engine,
+ ColiDeepSeekV4WindowAttentionState **attention,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config, ColiExpertStore *experts,
+ const int *draft_tokens, int draft_count, int prefix_position,
+ const float *prefix_main_x, char *error, size_t error_size);
+/* ==== end deepseek_v4_target_verify_prefix.h ==== */
+
+/* ==== begin deepseek_v4_target_attention_commit.h ==== */
+
+/* via deepseek_v4.h: deepseek_v4_attention.h */
+/* via deepseek_v4.h: deepseek_v4_config.h */
+/* via deepseek_v4.h: deepseek_v4_layer.h */
+
+/* Advance only a target layer's causal attention/cache for already-computed
+ * layer inputs. Used after speculative rollback to avoid replaying HC post and
+ * the complete MoE branch. */
+int coli_v4_target_attention_commit_batch(
+ ColiDeepSeekV4WindowAttentionState *attention,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const float *inputs_hc, int start_position, int batch,
+ char *error, size_t error_size);
+/* ==== end deepseek_v4_target_attention_commit.h ==== */
+
+/* ==== begin deepseek_v4_target_head_batch.h ==== */
+
+/* via deepseek_v4.h: deepseek_v4_config.h */
+#include "safetensors_index.h"
+
+int coli_v4_target_load_embeddings(float *states_hc,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config,
+ const int *tokens, int batch);
+int coli_v4_target_head_argmax_batch(
+ const float *states_hc, const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config, int batch,
+ int *tokens, float *logits, char *error, size_t error_size);
+/* ==== end deepseek_v4_target_head_batch.h ==== */
+
+#endif /* COLIBRI_DEEPSEEK_V4_DSPARK_H */
diff --git a/c/deepseek_v4_internal.h b/c/deepseek_v4_internal.h
new file mode 100644
index 00000000..69df5cf1
--- /dev/null
+++ b/c/deepseek_v4_internal.h
@@ -0,0 +1,682 @@
+#ifndef COLIBRI_DEEPSEEK_V4_INTERNAL_H
+#define COLIBRI_DEEPSEEK_V4_INTERNAL_H
+
+/*
+ * Internal DeepSeek-V4 API. Not a stability commitment.
+ * External callers should use deepseek_v4.h (engine / config / prompt).
+ */
+#include "deepseek_v4.h"
+
+#include "tensor.h"
+#include "expert_store.h"
+#include "native_quant.h"
+#include "native_quant_batch.h"
+#include "native_quant_dual.h"
+#include "native_quant_fp4_rows16.h"
+#include "safetensors_index.h"
+#include "tensor_io.h"
+
+typedef struct ColiV4Engine ColiV4Engine;
+
+/* ==== begin deepseek_v4_math.h ==== */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int coli_v4_hc_split_sinkhorn(float *pre, float *post, float *comb,
+ const float *mixes, const float scale[3],
+ const float *base, int hc, int iterations,
+ float eps);
+
+int coli_v4_hc_pre(float *output, float *post, float *comb,
+ const float *input, const float *hc_fn,
+ const float scale[3], const float *base,
+ int hc, int dimension, int iterations,
+ float norm_eps, float hc_eps);
+
+int coli_v4_hc_post(float *output, const float *branch,
+ const float *residual, const float *post,
+ const float *comb, int hc, int dimension);
+
+int coli_v4_rmsnorm(float *output, const float *input, const float *weight,
+ int dimension, float eps);
+
+int coli_v4_rope_precompute(float *cosines, float *sines,
+ int dimension, int sequence_length,
+ int original_sequence_length, float base,
+ float factor, int beta_fast, int beta_slow);
+
+int coli_v4_rope_apply(float *vectors, int vector_count, int dimension,
+ const float *cosines, const float *sines, int inverse);
+
+int coli_v4_route(float *weights, int *indices, const float *hidden,
+ const float *gate, const float *bias,
+ const int *forced_indices, int experts, int dimension,
+ int topk, float route_scale);
+
+int coli_v4_swiglu(float *output, const float *gate, const float *up,
+ int dimension, float limit);
+
+#ifdef __cplusplus
+}
+#endif
+/* ==== end deepseek_v4_math.h ==== */
+
+/* ==== begin deepseek_v4_layer.h ==== */
+
+#include <stddef.h>
+#include <stdint.h>
+
+/* amalgamated: deepseek_v4_config.h */
+#include "safetensors_index.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define COLI_V4_MAX_LAYER_TENSORS 48
+#define COLI_V4_MAX_TENSOR_NAME 160
+
+typedef struct {
+ char name[COLI_V4_MAX_TENSOR_NAME];
+ ColiSafetensorsDType dtype;
+ int rank;
+ int64_t shape[COLI_ST_MAX_RANK];
+} ColiDeepSeekV4TensorSpec;
+
+typedef struct {
+ int layer;
+ int compression_ratio;
+ int uses_hash_router;
+ int has_compressor;
+ int has_indexer;
+ size_t tensor_count;
+ ColiDeepSeekV4TensorSpec tensors[COLI_V4_MAX_LAYER_TENSORS];
+} ColiDeepSeekV4LayerPlan;
+
+typedef struct {
+ size_t tensor_count;
+ uint64_t total_bytes;
+ uint64_t bf16_bytes;
+ uint64_t f32_bytes;
+ uint64_t fp8_weight_bytes;
+ uint64_t fp8_scale_bytes;
+ uint64_t i64_bytes;
+} ColiDeepSeekV4LayerStats;
+
+typedef struct {
+ ColiDeepSeekV4LayerPlan plan;
+ ColiDeepSeekV4LayerStats stats;
+ void *data[COLI_V4_MAX_LAYER_TENSORS];
+} ColiDeepSeekV4LayerWeights;
+
+int coli_v4_layer_plan(ColiDeepSeekV4LayerPlan *plan,
+ const ColiDeepSeekV4Config *config, int layer,
+ char *error, size_t error_size);
+int coli_v4_layer_validate(const ColiDeepSeekV4LayerPlan *plan,
+ const ColiSafetensorsIndex *index,
+ ColiDeepSeekV4LayerStats *stats,
+ char *error, size_t error_size);
+int coli_v4_layer_load(ColiV4Engine *engine,
+ ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const ColiSafetensorsIndex *index, int layer,
+ char *error, size_t error_size);
+void coli_v4_layer_free(ColiV4Engine *engine,
+ ColiDeepSeekV4LayerWeights *weights);
+const void *coli_v4_layer_data(const ColiDeepSeekV4LayerWeights *weights,
+ const char *name,
+ const ColiDeepSeekV4TensorSpec **spec);
+
+#ifdef __cplusplus
+}
+#endif
+/* ==== end deepseek_v4_layer.h ==== */
+
+/* ==== begin deepseek_v4_sparse_attention.h ==== */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int coli_v4_sparse_attention_ref(float *output, const float *queries,
+ const float *kv, const float *sinks,
+ const int *indices, int heads,
+ int head_dimension, int kv_count, int topk,
+ float softmax_scale);
+
+#ifdef __cplusplus
+}
+#endif
+/* ==== end deepseek_v4_sparse_attention.h ==== */
+
+/* ==== begin deepseek_v4_kv_cache.h ==== */
+
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct ColiDeepSeekV4KVCache ColiDeepSeekV4KVCache;
+
+int coli_v4_kv_cache_create(ColiDeepSeekV4KVCache **cache,
+ int window_size, int compression_ratio,
+ int head_dimension, int max_context);
+void coli_v4_kv_cache_reset(ColiDeepSeekV4KVCache *cache);
+void coli_v4_kv_cache_destroy(ColiDeepSeekV4KVCache *cache);
+int coli_v4_kv_cache_put_window(ColiDeepSeekV4KVCache *cache,
+ int position, const float *kv);
+int coli_v4_kv_cache_put_compressed(ColiDeepSeekV4KVCache *cache,
+ int position, const float *kv);
+int coli_v4_kv_cache_indices(const ColiDeepSeekV4KVCache *cache,
+ int position, int *indices, size_t capacity);
+const float *coli_v4_kv_cache_values(const ColiDeepSeekV4KVCache *cache);
+int coli_v4_kv_cache_value_count(const ColiDeepSeekV4KVCache *cache);
+
+#ifdef __cplusplus
+}
+#endif
+/* ==== end deepseek_v4_kv_cache.h ==== */
+
+/* ==== begin deepseek_v4_attention_cache.h ==== */
+
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct ColiDeepSeekV4AttentionCache ColiDeepSeekV4AttentionCache;
+
+int coli_v4_attention_cache_create(ColiDeepSeekV4AttentionCache **cache,
+ int window_size, int compression_ratio,
+ int head_dimension, int max_context);
+void coli_v4_attention_cache_reset(ColiDeepSeekV4AttentionCache *cache);
+void coli_v4_attention_cache_destroy(ColiDeepSeekV4AttentionCache *cache);
+
+/* query is [heads, head_dimension]. window_kv and compressed_kv have one
+ * head_dimension vector each. compressed_kv is required at ratio boundaries. */
+int coli_v4_attention_cache_step(ColiDeepSeekV4AttentionCache *cache,
+ float *output, const float *query,
+ const float *window_kv,
+ const float *compressed_kv,
+ const float *sinks, int heads,
+ int position, float softmax_scale);
+
+#ifdef __cplusplus
+}
+#endif
+/* ==== end deepseek_v4_attention_cache.h ==== */
+
+/* ==== begin deepseek_v4_attention.h ==== */
+
+#include <stddef.h>
+
+/* amalgamated: deepseek_v4_config.h */
+/* amalgamated: deepseek_v4_layer.h */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct ColiDeepSeekV4WindowAttentionState
+ ColiDeepSeekV4WindowAttentionState;
+
+int coli_v4_window_attention_create(ColiDeepSeekV4WindowAttentionState **state,
+ const ColiDeepSeekV4Config *config);
+void coli_v4_window_attention_reset(ColiDeepSeekV4WindowAttentionState *state);
+void coli_v4_window_attention_destroy(ColiDeepSeekV4WindowAttentionState *state);
+
+/* Correctness-first single-KV attention. Compressed layers may use this at
+ * position zero, before any compressed KV/indexer candidate exists. */
+int coli_v4_attention_token_ref(float *output,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const float *input, int position,
+ char *error, size_t error_size);
+int coli_v4_attention_window_token_ref(
+ float *output, ColiDeepSeekV4WindowAttentionState *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, const float *input, int position,
+ char *error, size_t error_size);
+
+#ifdef __cplusplus
+}
+#endif
+/* ==== end deepseek_v4_attention.h ==== */
+
+/* ==== begin deepseek_v4_attention_batch.h ==== */
+
+/* amalgamated: deepseek_v4_attention.h */
+
+int coli_v4_attention_window_batch_ref(
+ float *outputs, ColiDeepSeekV4WindowAttentionState *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, const float *inputs,
+ int start_position, int batch, char *error, size_t error_size);
+/* ==== end deepseek_v4_attention_batch.h ==== */
+
+/* ==== begin deepseek_v4_attention_transaction.h ==== */
+
+/* amalgamated: deepseek_v4_attention.h */
+
+typedef struct ColiV4AttentionSnapshot ColiV4AttentionSnapshot;
+
+int coli_v4_attention_snapshot_create(
+ const ColiDeepSeekV4WindowAttentionState *state,
+ ColiV4AttentionSnapshot **output);
+int coli_v4_attention_snapshot_restore(
+ ColiDeepSeekV4WindowAttentionState *state,
+ const ColiV4AttentionSnapshot *snapshot);
+void coli_v4_attention_snapshot_destroy(ColiV4AttentionSnapshot *snapshot);
+/* ==== end deepseek_v4_attention_transaction.h ==== */
+
+/* ==== begin deepseek_v4_compressor.h ==== */
+
+#include <stddef.h>
+
+/* amalgamated: deepseek_v4_config.h */
+/* amalgamated: deepseek_v4_layer.h */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct ColiDeepSeekV4CompressorState ColiDeepSeekV4CompressorState;
+
+typedef struct {
+ const char *prefix;
+ int head_dimension;
+ int rotate_fp4;
+} ColiDeepSeekV4CompressorOptions;
+
+int coli_v4_compressor_create(ColiDeepSeekV4CompressorState **state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ char *error, size_t error_size);
+int coli_v4_compressor_create_with_options(
+ ColiDeepSeekV4CompressorState **state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ const ColiDeepSeekV4CompressorOptions *options,
+ char *error, size_t error_size);
+void coli_v4_compressor_reset(ColiDeepSeekV4CompressorState *state);
+int coli_v4_compressor_bind_weights(ColiDeepSeekV4CompressorState *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ char *error, size_t error_size);
+void coli_v4_compressor_destroy(ColiDeepSeekV4CompressorState *state);
+
+/* Processes one decode token. produced is set to one only when a complete
+ * compression window emits a KV vector. output may be NULL on other steps. */
+int coli_v4_compressor_step(ColiDeepSeekV4CompressorState *state,
+ float *output, int *produced,
+ const float *input, int position,
+ char *error, size_t error_size);
+
+#ifdef __cplusplus
+}
+#endif
+/* ==== end deepseek_v4_compressor.h ==== */
+
+/* ==== begin deepseek_v4_compressor_snapshot.h ==== */
+
+/* amalgamated: deepseek_v4_compressor.h */
+
+typedef struct ColiV4CompressorSnapshot ColiV4CompressorSnapshot;
+
+int coli_v4_compressor_snapshot_create(
+ const ColiDeepSeekV4CompressorState *state,
+ ColiV4CompressorSnapshot **output);
+int coli_v4_compressor_snapshot_restore(
+ ColiDeepSeekV4CompressorState *state,
+ const ColiV4CompressorSnapshot *snapshot);
+void coli_v4_compressor_snapshot_destroy(ColiV4CompressorSnapshot *snapshot);
+/* ==== end deepseek_v4_compressor_snapshot.h ==== */
+
+/* ==== begin deepseek_v4_indexer.h ==== */
+
+#include <stddef.h>
+
+/* amalgamated: deepseek_v4_config.h */
+/* amalgamated: deepseek_v4_layer.h */
+
+typedef struct ColiDeepSeekV4Indexer ColiDeepSeekV4Indexer;
+
+int coli_v4_indexer_create(ColiDeepSeekV4Indexer **state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ int max_context, char *error, size_t error_size);
+int coli_v4_indexer_bind_weights(ColiDeepSeekV4Indexer *state,
+ const ColiDeepSeekV4LayerWeights *weights,
+ char *error, size_t error_size);
+void coli_v4_indexer_reset(ColiDeepSeekV4Indexer *state);
+void coli_v4_indexer_destroy(ColiDeepSeekV4Indexer *state);
+
+/* Updates the overlap compressor, then returns compressed-cache ordinals in
+ * descending index score order. query_rank is the normalized q_lora vector. */
+int coli_v4_indexer_step(ColiDeepSeekV4Indexer *state, int *indices,
+ int index_capacity, const float *query_rank,
+ const float *input, int position,
+ char *error, size_t error_size);
+const float *coli_v4_indexer_compressed_values(
+ const ColiDeepSeekV4Indexer *state);
+int coli_v4_indexer_compressed_count(const ColiDeepSeekV4Indexer *state);
+/* ==== end deepseek_v4_indexer.h ==== */
+
+/* ==== begin deepseek_v4_indexer_snapshot.h ==== */
+
+/* amalgamated: deepseek_v4_indexer.h */
+
+typedef struct ColiV4IndexerSnapshot ColiV4IndexerSnapshot;
+
+int coli_v4_indexer_snapshot_create(const ColiDeepSeekV4Indexer *state,
+ ColiV4IndexerSnapshot **output);
+int coli_v4_indexer_snapshot_restore(ColiDeepSeekV4Indexer *state,
+ const ColiV4IndexerSnapshot *snapshot);
+void coli_v4_indexer_snapshot_destroy(ColiV4IndexerSnapshot *snapshot);
+/* ==== end deepseek_v4_indexer_snapshot.h ==== */
+
+/* ==== begin deepseek_v4_expert.h ==== */
+
+#include "expert_store.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int coli_v4_expert_forward_ref(float *output, const ColiExpertView *expert,
+ const float *input, float route_weight,
+ float swiglu_limit);
+
+int coli_v4_shared_expert_forward_ref(float *output,
+ const ColiTensorView *gate,
+ const ColiTensorView *down,
+ const ColiTensorView *up,
+ const float *input,
+ float swiglu_limit);
+
+#ifdef __cplusplus
+}
+#endif
+/* ==== end deepseek_v4_expert.h ==== */
+
+/* ==== begin deepseek_v4_expert_store.h ==== */
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "expert_store.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+ const char *model_dir;
+ int layers;
+ int experts_per_layer;
+ uint64_t cache_bytes;
+ /* Optional hot-pin policy (-1 / 0 => implementation default). */
+ int pin_slots_per_layer;
+ uint64_t repin_interval;
+} ColiDeepSeekV4ExpertStoreOptions;
+
+int coli_deepseek_v4_expert_store_open(
+ const ColiDeepSeekV4ExpertStoreOptions *options,
+ ColiExpertStore **store,
+ char *error,
+ size_t error_size);
+
+#ifdef __cplusplus
+}
+#endif
+/* ==== end deepseek_v4_expert_store.h ==== */
+
+/* ==== begin deepseek_v4_block.h ==== */
+
+#include <stddef.h>
+
+/* amalgamated: deepseek_v4_config.h */
+/* amalgamated: deepseek_v4_attention.h */
+/* amalgamated: deepseek_v4_layer.h */
+#include "expert_store.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int coli_v4_block_token_ref(float *output_hc,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config,
+ ColiExpertStore *experts,
+ const float *input_hc, int token, int position,
+ char *error, size_t error_size);
+int coli_v4_block_window_token_ref(
+ float *output_hc, ColiDeepSeekV4WindowAttentionState *attention,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, ColiExpertStore *experts,
+ const float *input_hc, int token, int position,
+ char *error, size_t error_size);
+
+#ifdef __cplusplus
+}
+#endif
+/* ==== end deepseek_v4_block.h ==== */
+
+/* ==== begin deepseek_v4_block_batch.h ==== */
+
+/* amalgamated: deepseek_v4_attention.h */
+/* amalgamated: deepseek_v4_config.h */
+/* amalgamated: deepseek_v4_layer.h */
+#include "expert_store.h"
+
+int coli_v4_block_window_batch_ref(
+ float *outputs_hc, ColiDeepSeekV4WindowAttentionState *attention,
+ const ColiDeepSeekV4LayerWeights *weights,
+ const ColiDeepSeekV4Config *config, ColiExpertStore *experts,
+ const float *inputs_hc, const int *tokens, int start_position, int batch,
+ char *error, size_t error_size);
+/* ==== end deepseek_v4_block_batch.h ==== */
+
+/* ==== begin deepseek_v4_resource_plan.h ==== */
+
+#include <stddef.h>
+#include <stdint.h>
+
+typedef struct {
+ uint64_t available_bytes;
+ uint64_t user_limit_bytes;
+ uint64_t maximum_layer_bytes;
+ uint64_t runtime_other_bytes;
+ uint64_t expert_record_bytes;
+ int sparse_layers;
+ int routed_topk;
+ int experts_per_layer;
+} ColiDeepSeekV4ResourceInputs;
+
+typedef struct {
+ uint64_t os_available_bytes;
+ uint64_t planner_available_bytes;
+ uint64_t system_reserve_bytes;
+ uint64_t runtime_reserve_bytes;
+ uint64_t minimum_expert_bytes;
+ uint64_t expert_cache_bytes;
+ uint64_t projected_bytes;
+ int slots_per_layer;
+} ColiDeepSeekV4ResourcePlan;
+
+uint64_t coli_v4_os_available_memory(void);
+int coli_v4_resource_plan_compute(
+ ColiDeepSeekV4ResourcePlan *plan,
+ const ColiDeepSeekV4ResourceInputs *inputs,
+ char *error, size_t error_size);
+/* ==== end deepseek_v4_resource_plan.h ==== */
+
+/* ==== begin deepseek_v4_head_cache.h ==== */
+
+#include <stddef.h>
+#include <stdint.h>
+
+int coli_v4_head_cache_probe(const char *model_dir, uint64_t *bytes,
+ char *error, size_t error_size);
+int coli_v4_head_cache_load(ColiV4Engine *engine, const char *model_dir,
+ char *error, size_t error_size);
+uint64_t coli_v4_head_cache_bytes(const ColiV4Engine *engine);
+const void *coli_v4_head_cache_data(const ColiV4Engine *engine,
+ int shard, uint64_t offset, size_t length);
+/* ==== end deepseek_v4_head_cache.h ==== */
+
+
+/* Runtime options live on ColiV4Engine. */
+typedef struct {
+ const char *target_model_dir;
+ const char *dspark_model_dir;
+ uint64_t memory_limit_bytes;
+ int context_tokens;
+ int dense_resident;
+ int dspark_resident;
+ uint64_t dspark_expert_cache_bytes;
+ int verify_drafts;
+ int pin_slots_per_layer;
+ uint64_t repin_interval;
+} ColiDeepSeekV4RuntimeOptions;
+
+/* Shared with deepseek_v4_dspark.h (guarded there to avoid double typedef). */
+#ifndef COLI_V4_DSPARK_MAX_STAGES
+#define COLI_V4_DSPARK_MAX_STAGES 8
+#endif
+#ifndef COLI_V4_DSPARK_MAX_TARGETS
+#define COLI_V4_DSPARK_MAX_TARGETS 8
+#endif
+
+#ifndef COLIBRI_DEEPSEEK_V4_DSPARK_MANIFEST_DEFINED
+#define COLIBRI_DEEPSEEK_V4_DSPARK_MANIFEST_DEFINED
+typedef struct {
+ int stage_count;
+ int block_size;
+ int noise_token_id;
+ int markov_rank;
+ int target_count;
+ int target_layer_ids[COLI_V4_DSPARK_MAX_TARGETS];
+ uint64_t common_stage_bytes[COLI_V4_DSPARK_MAX_STAGES];
+ uint64_t special_bytes;
+} ColiDeepSeekV4DSparkManifest;
+#endif
+
+typedef struct ColiV4DSparkHeads ColiV4DSparkHeads;
+
+enum { COLI_V4_RESIDENT_MAX_LAYERS = 128 };
+
+struct ColiV4Engine {
+ ColiDeepSeekV4Config config;
+ ColiDeepSeekV4RuntimeOptions runtime;
+ ColiSafetensorsIndex *target_index;
+ ColiExpertStore *experts;
+ ColiV4EngineMemorySummary summary;
+ struct {
+ unsigned char *data;
+ uint64_t bytes;
+ uint64_t offset;
+ int shard;
+ } head_cache;
+ struct {
+ ColiV4DSparkHeads *heads;
+ ColiDeepSeekV4DSparkManifest manifest;
+ float *states[COLI_V4_DSPARK_MAX_TARGETS];
+ size_t capacity;
+ float *staged_main_x;
+ int staged_batch;
+ int staged_hidden;
+ } dspark_capture;
+ struct {
+ int available;
+ int limit;
+ int head_slot;
+ int tokens[64];
+ float logits[64];
+ } dspark_verify;
+ struct {
+ ColiDeepSeekV4LayerWeights layers[COLI_V4_RESIDENT_MAX_LAYERS];
+ unsigned char ready[COLI_V4_RESIDENT_MAX_LAYERS];
+ const ColiSafetensorsIndex *index;
+ uint64_t total_bytes;
+ } dense_resident;
+ struct {
+ ColiDeepSeekV4LayerWeights layers[COLI_V4_DSPARK_MAX_STAGES];
+ unsigned char ready[COLI_V4_DSPARK_MAX_STAGES];
+ const ColiSafetensorsIndex *index;
+ uint64_t total_bytes;
+ } dspark_resident;
+ char *owned_target_model_dir;
+ char *owned_dspark_model_dir;
+ int owns_experts;
+ int owns_index;
+ int active_sessions; /* sessions created against this engine */
+};
+
+typedef struct ColiV4DSparkRunner ColiV4DSparkRunner;
+
+/* Session ownership helpers shared by production session code and tests. */
+void coli_v4_engine_attach_session(ColiV4Engine *engine);
+void coli_v4_engine_detach_session(ColiV4Engine *engine);
+void coli_v4_session_take_runner(ColiV4Session *session,
+ ColiV4DSparkRunner *runner);
+void coli_v4_session_clear_runner(ColiV4Session *session);
+
+#include "tok.h"
+
+struct ColiV4Session {
+ ColiV4Engine *engine;
+ ColiDeepSeekV4Config config;
+ ColiDeepSeekV4WindowAttentionState **attention;
+ float *state;
+ float *next;
+ float *hidden;
+ float *main_x_batch;
+ int *prompt_ids;
+ int *generated;
+ int max_prompt_tokens;
+ int max_new_tokens_cap;
+ int prompt_count;
+ int generated_count;
+ ColiV4DSparkRunner *runner;
+ Tok tokenizer;
+ int tokenizer_ready;
+ char *text;
+ int text_length;
+};
+
+/* RAM-tiered expert open used by coli_v4_engine_open (replaces ld --wrap). */
+int coli_v4_expert_store_open_planned(
+ ColiV4Engine *engine,
+ const ColiDeepSeekV4ExpertStoreOptions *options,
+ ColiExpertStore **store,
+ char *error,
+ size_t error_size);
+
+/* Internal accessors — not part of the experimental public API. */
+ColiSafetensorsIndex *coli_v4_engine_target_index(ColiV4Engine *engine);
+ColiExpertStore *coli_v4_engine_expert_store(ColiV4Engine *engine);
+
+/* Head-cache aware safetensors read (engine NULL => plain coli_st_read_at). */
+int coli_st_read_at_engine(ColiV4Engine *engine,
+ const ColiSafetensorsIndex *index, int shard,
+ uint64_t offset, size_t length, void *destination);
+
+#ifdef COLI_V4_TEST_HOOKS
+/*
+ * Fault-injection / counters for ownership tests only.
+ * Compile ownership objects with -DCOLI_V4_TEST_HOOKS; production objects omit this.
+ */
+extern int coli_v4_test_fail_expert_store_open;
+extern int coli_v4_test_skip_expert_store_open;
+extern int coli_v4_test_closed_owned_index;
+
+ColiV4Session *coli_v4_test_session_bare_create(ColiV4Engine *engine);
+void coli_v4_test_session_bare_destroy(ColiV4Session *session);
+ColiV4DSparkRunner *coli_v4_session_peek_runner(const ColiV4Session *session);
+#endif /* COLI_V4_TEST_HOOKS */
+
+#endif /* COLIBRI_DEEPSEEK_V4_INTERNAL_H */
diff --git a/c/tok.h b/c/tok.h
index 48ebba90..bff8730a 100644
--- a/c/tok.h
+++ b/c/tok.h
@@ -8,6 +8,7 @@
* tok_load(&T, "tokenizer.json");
* int n = tok_encode(&T, text, len, out_ids, max);
* int m = tok_decode(&T, ids, n, out_buf, max);
+ * tok_free(&T);
*/
#ifndef TOK_H
#define TOK_H
@@ -49,6 +50,7 @@ typedef struct {
* id_added, che copre anche <think>/<tool_call> ("special"
* false), i quali sono testo vero e vanno renderizzati. */
Special *sp; int nsp; /* added tokens, ordinati per lunghezza decrescente */
+ jval *json_root; /* owns borrowed vocab/special strings */
uint32_t byte2cp[256]; int byte2cp_len[256]; char byte2str[256][3];
int16_t cp2byte[1024];
int o200k; /* pre_tokenizer regex family: 0 = cl100k (GLM), 1 = o200k (Inkling) */
@@ -101,11 +103,35 @@ static char *tk_read_file(const char *path, long *out_n){
}
static int cmp_sp_len(const void *a, const void *b){ return ((const Special*)b)->len - ((const Special*)a)->len; }
+static void hm_free(hmap *m, int free_keys){
+ if(!m || !m->e) return;
+ if(free_keys){
+ for(int i=0;i<m->cap;i++)
+ if(m->e[i].used) free((void *)m->e[i].k);
+ }
+ free(m->e);
+ m->e=NULL; m->cap=0;
+}
+
+static void tok_free(Tok *T){
+ if(!T) return;
+ hm_free(&T->merges,1);
+ hm_free(&T->vocab,0);
+ free(T->sp);
+ free(T->id2str);
+ free(T->id_added);
+ free(T->id_special);
+ json_free(T->json_root);
+ memset(T,0,sizeof(*T));
+}
+
static void tok_load(Tok *T, const char *path){
memset(T,0,sizeof(*T));
tk_build_bytemap(T);
long fn; char *buf=tk_read_file(path,&fn);
char *arena=NULL; jval *root=json_parse(buf,&arena);
+ free(buf);
+ (void)arena;
jval *model=json_get(root,"model");
jval *vocab=json_get(model,"vocab");
jval *merges=json_get(model,"merges");
@@ -143,16 +169,24 @@ static void tok_load(Tok *T, const char *path){
hm_put(&T->vocab, k, (int)strlen(k), id);
T->id2str[id]=(char*)k;
}
- /* merges: "left\0right" -> rank=i */
+ /* merges: pair arrays and "left right" strings both become
+ * "left\0right" -> rank=i. */
int mc=1; while(mc < merges->len*2) mc<<=1;
hm_init(&T->merges, mc);
for(int i=0;i<merges->len;i++){
jval *pr=merges->kids[i];
- if(!pr||pr->t!=J_ARR||pr->len<2||!pr->kids[0]||!pr->kids[1]||
- pr->kids[0]->t!=J_STR||pr->kids[1]->t!=J_STR){
+ const char *l=NULL, *r=NULL; int ll=0, rl=0;
+ if(pr && pr->t==J_ARR && pr->len==2 && pr->kids[0] && pr->kids[1] &&
+ pr->kids[0]->t==J_STR && pr->kids[1]->t==J_STR){
+ l=pr->kids[0]->str; r=pr->kids[1]->str;
+ ll=(int)strlen(l); rl=(int)strlen(r);
+ }else if(pr && pr->t==J_STR && pr->str){
+ const char *separator=strchr(pr->str,' ');
+ if(!separator || separator==pr->str || !separator[1]){
+ fprintf(stderr,"tokenizer.json: malformed merge entry %d\n",i); exit(1); }
+ l=pr->str; ll=(int)(separator-l); r=separator+1; rl=(int)strlen(r);
+ }else{
fprintf(stderr,"tokenizer.json: malformed merge entry %d\n",i); exit(1); }
- const char *l=pr->kids[0]->str, *r=pr->kids[1]->str;
- int ll=(int)strlen(l), rl=(int)strlen(r);
char *key=malloc(ll+1+rl); memcpy(key,l,ll); key[ll]=0; memcpy(key+ll+1,r,rl);
hm_put(&T->merges, key, ll+1+rl, i);
}
@@ -183,8 +217,7 @@ static void tok_load(Tok *T, const char *path){
if(rx&&rx->t==J_STR&&strstr(rx->str,"\\p{Lu}")) T->o200k=1;
}
}
- /* arena/buf restano allocati: le stringhe (j_dup) sono malloc indipendenti e ci servono vive */
- (void)arena;
+ T->json_root=root;
}
/* ---------- BPE su un pezzo: byte grezzi [a,b) -> id appesi a out ---------- */