Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
deepseek_v4: AVX2 kernels, direct expert I/O, warm pins, verified drafting
JustVuggb7117bfd7cf2
diff --git a/c/Makefile.deepseek-v4 b/c/Makefile.deepseek-v4index 7aee0a92..0aa3a5ad 100644--- a/c/Makefile.deepseek-v4+++ b/c/Makefile.deepseek-v4@@ -28,7 +28,7 @@ endififneq ($(IS_WIN),)CC = gccARCH ?= x86-64-v3-CFLAGS = -D_FILE_OFFSET_BITS=64 -O3 -march=$(ARCH) -fopenmp \+CFLAGS = -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -O3 -march=$(ARCH) -fopenmp \-include pthread.h -Wall -Wextra -Wno-unused-parameter \-Wno-misleading-indentation -Wno-unused-function \-DCOLI_V4_MAX_PIN_SLOTS_PER_LAYER=16 \@@ -38,7 +38,7 @@ V4_BINARY := deepseek_v4.exeelseCC = gccARCH ?= native-CFLAGS = -O3 -march=$(ARCH) -fopenmp -pthread -include pthread.h \+CFLAGS = -D_GNU_SOURCE -O3 -march=$(ARCH) -fopenmp -pthread -include pthread.h \-Wall -Wextra -Wno-unused-parameter \-Wno-misleading-indentation -Wno-unused-function \-DCOLI_V4_MAX_PIN_SLOTS_PER_LAYER=16 \@@ -47,6 +47,12 @@ LDFLAGS = -lm -fopenmp -pthreadV4_BINARY := deepseek_v4endif+LTO ?= 1+ifeq ($(LTO),1)+CFLAGS += -flto+LDFLAGS += -flto+endif+include Makefile.deepseek-v4.unitsTARGET_OBJS = $(addsuffix .o,$(V4_TARGET_UNITS))@@ -60,7 +66,7 @@ $(V4_BINARY): $(V4_OBJS)$(CC) $(V4_OBJS) -o $@ $(LDFLAGS)$(TARGET_OBJS) $(TEST_UNIT_OBJS): %.o: deepseek_v4.c deepseek_v4.h \- deepseek_v4_internal.h st.h json.h compat.h tensor.h quant.h \+ deepseek_v4_internal.h deepseek_v4_dspark.inc st.h json.h compat.h tensor.h quant.h \native_quant.h native_quant_batch.h native_quant_dual.h \native_quant_fp4_rows16.h$(CC) $(CFLAGS) -D$* -c deepseek_v4.c -o $@diff --git a/c/coli b/c/coliindex 0d6d8f88..6f6942a9 100755--- a/c/coli+++ b/c/coli@@ -267,11 +267,50 @@ def env_for_engine(a, arch):if arch == "glm":return env_for(a)env = os.environ.copy()+ # OMP_NUM_THREADS for the sister engines too.+ #+ # #805 set this from physical cores "on every platform" -- but only inside+ # env_for(), which this function calls only for glm. inkling, kimi_k3,+ # olmoe and deepseek_v4 all took the branch below and got libgomp's+ # default, nproc, i.e. LOGICAL cores. On any SMT host that is a 2x+ # over-subscription of a memory-bound int4 GEMV, which is the collapse+ # measured in #718 (2.3x on Zen3) and again here: an i7-1355U reports 6+ # physical and 12 logical, and DeepSeek V4 was running the 12.+ #+ # Same defect shape as the one #805 fixed, one level out: a mechanism that+ # reaches one engine and not its siblings.+ #+ # setdefault, so an explicit OMP_NUM_THREADS still wins -- which is also+ # how to A/B it: OMP_NUM_THREADS=12 reproduces the old behaviour.+ if not env.get("COLI_NO_OMP_TUNE"):+ from resource_plan import physical_cpu_count+ env.setdefault("OMP_NUM_THREADS", str(physical_cpu_count()))+ if arch == "deepseek_v4":+ env.setdefault("OMP_WAIT_POLICY", "active")+ env.setdefault("GOMP_SPINCOUNT", "200000")+ env.setdefault("OMP_DYNAMIC", "FALSE")+ if sys.platform != "win32":+ env.setdefault("OMP_PROC_BIND", "close")+ env.setdefault("OMP_PLACES", "cores")if a.ngen: env["NGEN"] = str(a.ngen)if a.temp is not None: env["COLI_TEMP"] = str(a.temp)if arch == "deepseek_v4":if a.ram: env["RAM_GB"] = str(a.ram)if a.ctx: env["CTX"] = str(a.ctx)+ # Speculation stays opt-in. Real multi-turn chat measured only 1/15+ # accepted prompt-lookup candidates; recurrent-state replay dominated+ # the visible decode. V4_DRAFT=5 remains available for repeated code.+ env.setdefault("V4_DRAFT", "0")+ # Full three-stage DSpark remains opt-in. In a real multi-turn chat it+ # accepted only 10/24 candidates and rejected-suffix replay turned a+ # 14-token answer into 495 seconds. V4_MTP=1 still enables A/B runs;+ # exact zero-I/O prompt lookup above remains enabled by default.+ env.setdefault("V4_MTP", "0")+ env.setdefault("V4_MTP_DRAFT", "3")+ env.setdefault("V4_MTP_GB", "0.45")+ env.setdefault("V4_MTP_MISS", "96")+ env.setdefault("V4_MTP_MIN", "3")+ env.setdefault("V4_MTP_CONF", "0.55")return envdef cuda_binary():diff --git a/c/deepseek_v4.c b/c/deepseek_v4.cindex 31cb57ec..7f740c2d 100644--- a/c/deepseek_v4.c+++ b/c/deepseek_v4.c@@ -1,3 +1,6 @@+#ifndef _GNU_SOURCE+#define _GNU_SOURCE+#endif/* Amalgamated deepseek_v4.c — GLM-style source; compile with -DCOLI_V4_UNIT_* per object *//* Umbrella API: deepseek_v4.h (included by units) */@@ -103,6 +106,71 @@ int coli_st_read_at(const ColiSafetensorsIndex *index, int shard,return 0;}+int coli_st_streaming_direct_available(const ColiSafetensorsIndex *index,+ int shard) {+ if (!index || shard < 0 || shard >= index->nfd) return 0;+ const char *setting = getenv("COLI_V4_DIRECT");+ if (setting && atoi(setting) == 0) return 0;+ return index->dfds[shard] >= 0;+}++int coli_st_read_at_streaming(const ColiSafetensorsIndex *index, int shard,+ uint64_t offset, size_t length,+ void *destination) {+ if (!index || !destination || shard < 0 || shard >= index->nfd ||+ index->sizes[shard] < 0 || offset > (uint64_t)index->sizes[shard] ||+ length > (uint64_t)index->sizes[shard] - offset)+ return -1;+ if (!length) return 0;+ if (!coli_st_streaming_direct_available(index, shard))+ return coli_st_read_at(index, shard, offset, length, destination);++ const uint64_t alignment = 4096;+ uint64_t base = offset & ~(alignment - 1);+ uint64_t pad = offset - base;+ if ((uint64_t)length > UINT64_MAX - pad) return -1;+ uint64_t needed = pad + (uint64_t)length;+ if (needed > UINT64_MAX - (alignment - 1) ||+ needed + alignment - 1 > SIZE_MAX) return -1;+ size_t allocation_bytes = (size_t)((needed + alignment - 1) &+ ~(alignment - 1));+ unsigned char *bounce = NULL;+ if (posix_memalign((void **)&bounce, (size_t)alignment,+ allocation_bytes) != 0)+ return coli_st_read_at(index, shard, offset, length, destination);++ uint64_t file_bytes = (uint64_t)index->sizes[shard];+ uint64_t available = file_bytes - base;+ uint64_t direct_bytes = allocation_bytes;+ if (direct_bytes > available)+ direct_bytes = available & ~(alignment - 1);+ size_t done = 0;+ int failed = 0;+ while ((uint64_t)done < direct_bytes) {+ ssize_t count = pread(index->dfds[shard], bounce + done,+ (size_t)(direct_bytes - done),+ (off_t)(base + done));+ if (count < 0 && errno == EINTR) continue;+ if (count <= 0) { failed = 1; break; }+ done += (size_t)count;+ }+ if (failed) {+ free(bounce);+ return coli_st_read_at(index, shard, offset, length, destination);+ }+ while ((uint64_t)done < needed) {+ ssize_t count = pread(index->fds[shard], bounce + done,+ (size_t)(needed - done),+ (off_t)(base + done));+ if (count < 0 && errno == EINTR) continue;+ if (count <= 0) { free(bounce); return -1; }+ done += (size_t)count;+ }+ memcpy(destination, bounce + pad, length);+ free(bounce);+ return 0;+}+int coli_st_read_tensor(const ColiSafetensorsIndex *index,const ColiSafetensorsTensor *tensor, void *destination) {int shard = coli_st_tensor_shard(index, tensor);@@ -408,6 +476,33 @@ void coli_v4_layer_free(ColiV4Engine *engine,memset(weights, 0, sizeof(*weights));}+/* AVX2 consumes eight output rows at once. Interleave those rows by column+ * once when a dense layer becomes resident, instead of gathering eight+ * distant cache lines in every matvec. */+static int v4_fp8_pack_rows8_inplace(unsigned char *data,+ int64_t rows, int64_t columns) {+#ifndef __AVX2__+ (void)data; (void)rows; (void)columns;+ return 0;+#else+ if (!data || rows < 8 || rows % 8 || columns < 128 || columns % 128)+ return 0;+ size_t tile_bytes = (size_t)8 * (size_t)columns;+ unsigned char *scratch = malloc(tile_bytes);+ if (!scratch) return -1;+ for (int64_t tile = 0; tile < rows / 8; tile++) {+ unsigned char *target = data + (size_t)tile * tile_bytes;+ memcpy(scratch, target, tile_bytes);+ for (int64_t column = 0; column < columns; column++)+ for (int lane = 0; lane < 8; lane++)+ target[(size_t)column * 8 + lane] =+ scratch[(size_t)lane * columns + column];+ }+ free(scratch);+ return 1;+#endif+}+int coli_v4_layer_load(ColiV4Engine *engine,ColiDeepSeekV4LayerWeights *weights,const ColiDeepSeekV4Config *config,@@ -438,6 +533,17 @@ int coli_v4_layer_load(ColiV4Engine *engine,coli_v4_layer_free(NULL, weights);return set_error(error, error_size, "cannot read tensor: %s", spec->name);}+ if (spec->dtype == COLI_ST_F8_E4M3 && spec->rank == 2) {+ int packed = v4_fp8_pack_rows8_inplace(+ weights->data[i], spec->shape[0], spec->shape[1]);+ if (packed < 0) {+ coli_v4_layer_free(NULL, weights);+ return set_error(error, error_size,+ "out of memory packing FP8 rows8: %s",+ spec->name);+ }+ weights->plan.tensors[i].packed_rows8 = packed > 0;+ }}return 0;}@@ -848,6 +954,11 @@ static int build_runtime_plan(ColiV4Engine *engine,sizeof(float) * 2;uint64_t scratch = 512 * MIB;uint64_t runtime_other = context_bytes(&config, context) + hidden + scratch;+ if (UINT64_MAX - runtime_other < runtime->dspark_reserve_bytes) {+ snprintf(error, error_size, "V4 DSpark reserve overflow");+ return -1;+ }+ runtime_other += runtime->dspark_reserve_bytes;uint64_t available = coli_v4_os_available_memory();if (!available) {snprintf(error, error_size, "cannot determine OS available memory");@@ -1404,7 +1515,8 @@ static int fp8_view(ColiTensorView *view,COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_F32, data, scales,(size_t)(weight_spec->shape[0] * weight_spec->shape[1]),(size_t)(scale_spec->shape[0] * scale_spec->shape[1]) * sizeof(float),- weight_spec->shape[0], weight_spec->shape[1], 128, 128+ weight_spec->shape[0], weight_spec->shape[1],+ weight_spec->packed_rows8 ? 8 : 128, 128};return 0;}@@ -1801,7 +1913,8 @@ static int fp8_view(ColiTensorView *view,COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_F32, data, scales,(size_t)(weight_spec->shape[0] * weight_spec->shape[1]),(size_t)(scale_spec->shape[0] * scale_spec->shape[1]) * sizeof(float),- weight_spec->shape[0], weight_spec->shape[1], 128, 128+ weight_spec->shape[0], weight_spec->shape[1],+ weight_spec->packed_rows8 ? 8 : 128, 128};return 0;}@@ -2611,7 +2724,7 @@ static int fp8_view(ColiTensorView *view,COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_F32, data, scales,(size_t)(ws->shape[0] * ws->shape[1]),(size_t)(ss->shape[0] * ss->shape[1]) * sizeof(float),- ws->shape[0], ws->shape[1], 128, 128+ ws->shape[0], ws->shape[1], ws->packed_rows8 ? 8 : 128, 128};return 0;}@@ -2930,7 +3043,7 @@ static int fp8_view(ColiTensorView *view,COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_F32, data, scales,(size_t)(ws->shape[0] * ws->shape[1]),(size_t)(ss->shape[0] * ss->shape[1]) * sizeof(float),- ws->shape[0], ws->shape[1], 128, 128+ ws->shape[0], ws->shape[1], ws->packed_rows8 ? 8 : 128, 128};return 0;}@@ -4174,7 +4287,7 @@ static int fp8_view(ColiTensorView *view,COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_F32, data, scales,(size_t)(ws->shape[0] * ws->shape[1]),(size_t)(ss->shape[0] * ss->shape[1]) * sizeof(float),- ws->shape[0], ws->shape[1], 128, 128+ ws->shape[0], ws->shape[1], ws->packed_rows8 ? 8 : 128, 128};return 0;}@@ -4574,7 +4687,8 @@ static int fp8_view(ColiTensorView *view,COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_F32, data, scales,(size_t)(weight_spec->shape[0] * weight_spec->shape[1]),(size_t)(scale_spec->shape[0] * scale_spec->shape[1]) * sizeof(float),- weight_spec->shape[0], weight_spec->shape[1], 128, 128+ weight_spec->shape[0], weight_spec->shape[1],+ weight_spec->packed_rows8 ? 8 : 128, 128};return 0;}@@ -4904,7 +5018,9 @@ void coli_v4_attention_snapshot_destroy(ColiV4AttentionSnapshot *snapshot) {#define coli_deepseek_v4_expert_store_open \coli_deepseek_v4_expert_store_open_base/* ---- begin include deepseek_v4_expert_store.c ---- */+#ifndef _GNU_SOURCE#define _GNU_SOURCE+#endif#include "deepseek_v4_internal.h"#include <assert.h>@@ -4939,6 +5055,7 @@ typedef struct {unsigned references;uint64_t used;unsigned char *slab;+ int aligned_slab;} V4ExpertSlot;typedef struct {@@ -5104,11 +5221,13 @@ static int lookup(ColiExpertStore *store, ColiExpertKey key,}/* A short read must never expose a partially overwritten old slot. */slot->expert = -1;- if (coli_st_read_at(state->index, record->shard, record->scale_offset,- (size_t)record->scale_bytes, slot->slab) != 0 ||- coli_st_read_at(state->index, record->shard, record->weight_offset,- (size_t)record->weight_bytes,- slot->slab + record->scale_bytes) != 0) {+ if (coli_st_read_at_streaming(+ state->index, record->shard, record->scale_offset,+ (size_t)record->scale_bytes, slot->slab) != 0 ||+ coli_st_read_at_streaming(+ state->index, record->shard, record->weight_offset,+ (size_t)record->weight_bytes,+ slot->slab + record->scale_bytes) != 0) {pthread_mutex_unlock(&state->mutex);memset(view, 0, sizeof(*view));return -1;@@ -5308,6 +5427,8 @@ fail:#undef coli_deepseek_v4_expert_store_open#include "native_quant_fp4_rows16.h"+#include <limits.h>+#include <time.h>#include "deepseek_v4_internal.h"@@ -5320,12 +5441,141 @@ typedef struct V4HotPolicy {int *pins;unsigned char *packed;uint64_t packed_slots;+ uint64_t history_total;+ int history_seeded;+ char *history_path;struct V4HotPolicy *next;} V4HotPolicy;static pthread_mutex_t hot_policies_mutex = PTHREAD_MUTEX_INITIALIZER;static V4HotPolicy *hot_policies;+static double hot_now(void) {+ struct timespec value;+ clock_gettime(CLOCK_MONOTONIC, &value);+ return (double)value.tv_sec + value.tv_nsec * 1e-9;+}++static uint32_t hot_engine_id(void) {+ const unsigned char *text = (const unsigned char *)"deepseek_v4";+ uint32_t hash = 2166136261u;+ while (*text) { hash ^= *text++; hash *= 16777619u; }+ return hash;+}++static char *hot_history_path(const char *model_dir) {+ if (!model_dir) return NULL;+ const char suffix[] = "/.coli_usage";+ size_t length = strlen(model_dir) + sizeof(suffix);+ char *path = malloc(length);+ if (path) snprintf(path, length, "%s%s", model_dir, suffix);+ return path;+}++/* Seed the hot-expert ranking from the same sparse text history used by the+ * other engines. Placement is the only observable effect: expert ids and+ * weights are still selected by the V4 router on every forward. */+static uint64_t hot_usage_load(V4HotPolicy *policy,+ const V4ExpertStoreState *state,+ const char *model_dir) {+ if (!policy || !state || !model_dir) return 0;+ char *path = hot_history_path(model_dir);+ if (!path) return 0;+ FILE *stream = fopen(path, "rb");+ if (!stream) { free(path); return 0; }++ int layer = 0, expert = 0, dimensions_seen = 0;+ unsigned count = 0;+ while (fscanf(stream, "%d %d %u", &layer, &expert, &count) == 3) {+ if (layer == -1) {+ dimensions_seen = 1;+ if (expert != state->layers ||+ count != (unsigned)state->experts_per_layer) {+ fprintf(stderr,+ "v4_autopin ignored=%s dimensions=%d/%u expected=%d/%d\n",+ path, expert, count, state->layers,+ state->experts_per_layer);+ fclose(stream); free(path); return 0;+ }+ } else if (layer == -2 &&+ (expert > 1 || count != hot_engine_id())) {+ fprintf(stderr,+ "v4_autopin ignored=%s identity=version-%d/id-%u "+ "expected=version-1/id-%u\n",+ path, expert, count, hot_engine_id());+ fclose(stream); free(path); return 0;+ }+ }+ if (!dimensions_seen) {+ fclose(stream); free(path); return 0;+ }+ rewind(stream);+ uint64_t total = 0;+ while (fscanf(stream, "%d %d %u", &layer, &expert, &count) == 3) {+ if (layer < 0 || layer >= state->layers || expert < 0 ||+ expert >= state->experts_per_layer || !count) continue;+ uint64_t *slot = &policy->usage[+ (size_t)layer * state->experts_per_layer + expert];+ if (UINT64_MAX - *slot < count) *slot = UINT64_MAX;+ else *slot += count;+ if (UINT64_MAX - total < count) total = UINT64_MAX;+ else total += count;+ }+ fclose(stream);+ if (total)+ fprintf(stderr, "v4_autopin history=%s selections=%llu\n", path,+ (unsigned long long)total);+ free(path);+ return total;+}++static void hot_usage_save(const V4HotPolicy *policy,+ const V4ExpertStoreState *state) {+ if (!policy || !state || !policy->history_path || !policy->usage) return;+ const char *enabled = getenv("COLI_V4_SAVE_USAGE");+ if (enabled && atoi(enabled) == 0) return;+ size_t tmp_length = strlen(policy->history_path) + sizeof(".tmp");+ char *tmp = malloc(tmp_length);+ if (!tmp) return;+ snprintf(tmp, tmp_length, "%s.tmp", policy->history_path);+ FILE *stream = fopen(tmp, "wb");+ if (!stream) { free(tmp); return; }++ uint64_t total = 0, distinct = 0;+ for (int layer = 0; layer < state->layers; layer++)+ for (int expert = 0; expert < state->experts_per_layer; expert++) {+ uint64_t count = policy->usage[+ (size_t)layer * state->experts_per_layer + expert];+ if (!count) continue;+ total = UINT64_MAX - total < count ? UINT64_MAX : total + count;+ distinct++;+ }+ if (distinct) {+ fprintf(stream, "-1 %d %d\n", state->layers,+ state->experts_per_layer);+ fprintf(stream, "-2 1 %u\n", hot_engine_id());+ for (int layer = 0; layer < state->layers; layer++)+ for (int expert = 0; expert < state->experts_per_layer; expert++) {+ uint64_t count = policy->usage[+ (size_t)layer * state->experts_per_layer + expert];+ if (!count) continue;+ unsigned stored = count > UINT_MAX ? UINT_MAX : (unsigned)count;+ fprintf(stream, "%d %d %u\n", layer, expert, stored);+ }+ }+ int failed = ferror(stream);+ if (fclose(stream) != 0) failed = 1;+ if (!failed && rename(tmp, policy->history_path) == 0)+ fprintf(stderr,+ "v4_autopin saved=%s selections=%llu distinct=%llu\n",+ policy->history_path, (unsigned long long)total,+ (unsigned long long)distinct);+ else+ fprintf(stderr, "v4_autopin warning=cannot-save-history path=%s\n",+ policy->history_path);+ free(tmp);+}+static V4HotPolicy *hot_find(ColiExpertStore *store) {pthread_mutex_lock(&hot_policies_mutex);@@ -5343,7 +5593,7 @@ static int hot_is_pinned(const V4HotPolicy *policy, int layer, int expert) {if (!policy || policy->pin_count < 1) return 0;int active = policy->pin_count;#if COLI_V4_PIN_RAMP_REQUESTS > 0- if (active > 4) {+ if (!policy->history_seeded && active > 4) {uint64_t grown = policy->layer_requests[layer] /COLI_V4_PIN_RAMP_REQUESTS;active = grown >= (uint64_t)(active - 4)@@ -5361,6 +5611,98 @@ static size_t hot_slot_index(const V4ExpertStoreState *state,return (size_t)(slot - state->slots);}+/* Expert records dominate decode I/O. Read the large FP4 payload directly+ * into the final cache slot, instead of allocating a record-sized bounce+ * buffer and copying it for every miss. The extra two pages on each slot let+ * an unaligned safetensors range be expanded to an O_DIRECT window safely.+ *+ * FLOCK-packed checkpoints store [scales][weights] contiguously and need one+ * request. Standard HF checkpoints keep the ranges apart: weights use direct+ * I/O while the much smaller scales use buffered pread. Any direct-I/O error+ * falls back to the exact buffered path. */+static uint64_t v4_direct_reads;+static uint64_t v4_direct_flock_reads;+static uint64_t v4_direct_payload_bytes;+static uint64_t v4_direct_fallbacks;++static int v4_pread_full_try(int fd, void *destination, size_t length,+ uint64_t offset) {+ unsigned char *output = destination;+ size_t done = 0;+ while (done < length) {+ ssize_t count = pread(fd, output + done, length - done,+ (off_t)(offset + done));+ if (count < 0 && errno == EINTR) continue;+ if (count <= 0) return -1;+ done += (size_t)count;+ }+ return 0;+}++static int v4_read_direct_window(const V4ExpertStoreState *state, int shard,+ unsigned char *slab, uint64_t offset,+ size_t length, size_t destination_offset) {+ if (!state || !state->index || shard < 0 || shard >= state->index->nfd ||+ state->index->dfds[shard] < 0) return -1;+ const uint64_t alignment = 4096;+ uint64_t base = offset & ~(alignment - 1);+ size_t pad = (size_t)(offset - base);+ if (length > SIZE_MAX - pad) return -1;+ size_t wanted = pad + length;+ if (wanted > SIZE_MAX - (alignment - 1)) return -1;+ size_t direct_length = (wanted + alignment - 1) & ~(size_t)(alignment - 1);+ uint64_t file_bytes = (uint64_t)state->index->sizes[shard];+ if (base > file_bytes) return -1;+ uint64_t available = file_bytes - base;+ if ((uint64_t)direct_length > available)+ direct_length = (size_t)(available & ~(alignment - 1));+ if (direct_length && v4_pread_full_try(+ state->index->dfds[shard], slab, direct_length, base)) return -1;+ if (direct_length < wanted && v4_pread_full_try(+ state->index->fds[shard], slab + direct_length,+ wanted - direct_length, base + direct_length)) return -1;+ memmove(slab + destination_offset, slab + pad, length);+ return 0;+}++static int v4_read_expert_record(V4ExpertStoreState *state,+ const V4ExpertRecord *record,+ V4ExpertSlot *slot) {+ int direct_available = slot->aligned_slab &&+ coli_st_streaming_direct_available(state->index, record->shard);+ if (direct_available &&+ record->scale_offset + record->scale_bytes == record->weight_offset &&+ !v4_read_direct_window(state, record->shard, slot->slab,+ record->scale_offset,+ (size_t)record->record_bytes, 0)) {+ __atomic_fetch_add(&v4_direct_reads, UINT64_C(1), __ATOMIC_RELAXED);+ __atomic_fetch_add(&v4_direct_flock_reads, UINT64_C(1),+ __ATOMIC_RELAXED);+ __atomic_fetch_add(&v4_direct_payload_bytes, record->record_bytes,+ __ATOMIC_RELAXED);+ return 0;+ }++ int weight_direct = direct_available && !v4_read_direct_window(+ state, record->shard, slot->slab, record->weight_offset,+ (size_t)record->weight_bytes, (size_t)record->scale_bytes);+ if (weight_direct) {+ __atomic_fetch_add(&v4_direct_reads, UINT64_C(1), __ATOMIC_RELAXED);+ __atomic_fetch_add(&v4_direct_payload_bytes, record->weight_bytes,+ __ATOMIC_RELAXED);+ } else {+ if (direct_available)+ __atomic_fetch_add(&v4_direct_fallbacks, UINT64_C(1),+ __ATOMIC_RELAXED);+ if (coli_st_read_at(state->index, record->shard,+ record->weight_offset,+ (size_t)record->weight_bytes,+ slot->slab + record->scale_bytes)) return -1;+ }+ return coli_st_read_at(state->index, record->shard, record->scale_offset,+ (size_t)record->scale_bytes, slot->slab);+}+static void hot_fill_view(ColiTensorView *view,const V4ExpertRecord *record,const V4ExpertSlot *slot, int matrix,@@ -5443,7 +5785,7 @@ static void hot_repin_locked(V4HotPolicy *policy, V4ExpertStoreState *state,hot_is_pinned(policy, layer, slots[i].expert))slots[i].used = ++state->clock;uint64_t decay_interval = policy->repin_interval * 64;- if (decay_interval &&+ if (decay_interval && policy->layer_requests[layer] &&policy->layer_requests[layer] % decay_interval == 0)for (int expert = 0; expert < state->experts_per_layer; expert++)usage[expert] = (usage[expert] + 1) / 2;@@ -5504,12 +5846,14 @@ static int lookup_hot(ColiExpertStore *store, ColiExpertKey key,return -1;}if (!slot->slab) {- slot->slab = malloc((size_t)state->record_bytes);- if (!slot->slab) {+ size_t capacity = (size_t)state->record_bytes + 8192u;+ if (posix_memalign((void **)&slot->slab, 4096, capacity)) {+ slot->slab = NULL;pthread_mutex_unlock(&state->mutex);memset(view, 0, sizeof(*view));return -1;}+ slot->aligned_slab = 1;state->stats.resident_bytes += state->record_bytes;}policy->packed[hot_slot_index(state, slot)] = 0;@@ -5517,12 +5861,7 @@ static int lookup_hot(ColiExpertStore *store, ColiExpertKey key,state->active_leases++;slot->used = ++state->clock;pthread_mutex_unlock(&state->mutex);- int read_result = coli_st_read_at(- state->index, record->shard, record->scale_offset,- (size_t)record->scale_bytes, slot->slab);- if (!read_result) read_result = coli_st_read_at(- state->index, record->shard, record->weight_offset,- (size_t)record->weight_bytes, slot->slab + record->scale_bytes);+ int read_result = v4_read_expert_record(state, record, slot);pthread_mutex_lock(&state->mutex);if (read_result) {slot->references = 0; slot->expert = -1;@@ -5551,14 +5890,74 @@ static void destroy_hot(ColiExpertStore *store) {if (policy) *link = policy->next;pthread_mutex_unlock(&hot_policies_mutex);if (policy) {+ hot_usage_save(policy, store ? store->state : NULL);fprintf(stderr, "v4_rows16 packed_slots=%llu\n",(unsigned long long)policy->packed_slots);+ fprintf(stderr,+ "v4_direct reads=%llu flock_reads=%llu fallbacks=%llu "+ "payload_bytes=%llu\n",+ (unsigned long long)__atomic_load_n(+ &v4_direct_reads, __ATOMIC_RELAXED),+ (unsigned long long)__atomic_load_n(+ &v4_direct_flock_reads, __ATOMIC_RELAXED),+ (unsigned long long)__atomic_load_n(+ &v4_direct_fallbacks, __ATOMIC_RELAXED),+ (unsigned long long)__atomic_load_n(+ &v4_direct_payload_bytes, __ATOMIC_RELAXED));+ free(policy->history_path);free(policy->packed); free(policy->pins);free(policy->layer_requests); free(policy->usage); free(policy);}destroy(store);}+static int hot_prewarm_history(V4HotPolicy *policy,+ V4ExpertStoreState *state) {+ if (!policy || !state || !policy->history_seeded ||+ policy->pin_count < 1) return 0;+ size_t capacity = (size_t)state->layers * policy->pin_count;+ ColiExpertKey *keys = malloc(capacity * sizeof(*keys));+ if (!keys) return -1;+ size_t count = 0;+ for (int layer = 0; layer < state->layers; layer++) {+ hot_repin_locked(policy, state, layer);+ const int *pins = policy->pins + (size_t)layer * policy->pin_count;+ for (int rank = 0; rank < policy->pin_count; rank++)+ if (pins[rank] >= 0)+ keys[count++] = (ColiExpertKey){layer, pins[rank]};+ }++ double began = hot_now();+ int warmed = 0;+ uint64_t repin_interval = policy->repin_interval;+ policy->repin_interval = 0;+ #pragma omp parallel for schedule(dynamic, 1) reduction(+:warmed)+ for (size_t i = 0; i < count; i++) {+ ColiExpertView view;+ if (!lookup_hot(policy->store, keys[i], &view)) {+ release(policy->store, &view);+ warmed++;+ }+ }+ policy->repin_interval = repin_interval;+ free(keys);++ /* Warmup traffic must not dilute request hit-rate telemetry. */+ pthread_mutex_lock(&state->mutex);+ uint64_t resident = state->stats.resident_bytes;+ uint64_t capacity_bytes = state->stats.capacity_bytes;+ memset(&state->stats, 0, sizeof(state->stats));+ state->stats.resident_bytes = resident;+ state->stats.capacity_bytes = capacity_bytes;+ memset(policy->layer_requests, 0,+ (size_t)state->layers * sizeof(*policy->layer_requests));+ pthread_mutex_unlock(&state->mutex);+ fprintf(stderr,+ "v4_autopin warmed=%d requested=%zu bytes=%.3fGiB time=%.3fs\n",+ warmed, count, resident / 1073741824.0, hot_now() - began);+ return warmed == (int)count ? 0 : -1;+}+#ifndef COLI_V4_ROWS16_STORE_OPEN#define COLI_V4_ROWS16_STORE_OPEN coli_deepseek_v4_expert_store_open#endif@@ -5573,6 +5972,11 @@ int COLI_V4_ROWS16_STORE_OPEN(options, output, error, error_size);if (result) return result;V4ExpertStoreState *state = (*output)->state;+ int direct_io = state->layers > 0 && state->experts_per_layer > 0 &&+ coli_st_streaming_direct_available(+ state->index, state->records[0].shard);+ fprintf(stderr, "v4_ssd_io mode=%s fallback=buffered-pread\n",+ direct_io ? "direct-aligned" : "buffered-pread");int minimum_slots = state->experts_per_layer < 6? state->experts_per_layer : 6;int maximum_pins = state->slots_per_layer - minimum_slots;@@ -5608,9 +6012,20 @@ int COLI_V4_ROWS16_STORE_OPEN(}for (size_t i = 0; i < pins; i++) policy->pins[i] = -1;policy->store = *output; policy->pin_count = pin_count;+ policy->history_path = hot_history_path(options->model_dir);policy->repin_interval = options->repin_interval? options->repin_interval : (uint64_t)minimum_slots;if (!policy->repin_interval) policy->repin_interval = 1;+ const char *autopin = getenv("COLI_V4_AUTOPIN");+ if (!autopin || atoi(autopin) != 0) {+ policy->history_total = hot_usage_load(policy, state,+ options->model_dir);+ /* A tiny history is less predictive than the adaptive LRU. */+ policy->history_seeded = policy->history_total >= 5000;+ if (policy->history_seeded)+ for (int layer = 0; layer < state->layers; layer++)+ hot_repin_locked(policy, state, layer);+ }pthread_mutex_lock(&hot_policies_mutex);policy->next = hot_policies; hot_policies = policy;pthread_mutex_unlock(&hot_policies_mutex);@@ -5619,6 +6034,10 @@ int COLI_V4_ROWS16_STORE_OPEN("v4_hot_policy pin_slots_per_layer=%d repin_interval=%llu ""mode=resident-ram rows16=hot-pins\n", pin_count,(unsigned long long)policy->repin_interval);+ const char *prewarm = getenv("COLI_V4_PREWARM");+ if (policy->history_seeded && prewarm && atoi(prewarm) != 0 &&+ hot_prewarm_history(policy, state))+ fprintf(stderr, "v4_autopin warning=partial-warmup; continuing\n");return 0;}#endif /* COLI_V4_UNIT_EXPERT_STORE_HOT_ROWS16 */@@ -5853,6 +6272,116 @@ void coli_v4_engine_detach_session(ColiV4Engine *engine) {engine->active_sessions--;}+int coli_v4_full_dspark_wanted;++double coli_v4_dspark_cache_gb(void) {+ const char *setting = getenv("V4_MTP_GB");+ double value = setting ? atof(setting) : 0.45;+ if (value < 0.15) value = 0.15;+ /* The target cache is reduced by the same reservation, but keep a hard+ * ceiling so a typo cannot turn a lazy drafter into an OOM request. */+ if (value > 4.0) value = 4.0;+ return value;+}++static int v4_dspark_full_wanted(+ const ColiV4EngineOpenOptions *options) {+ if (!options || options->no_dspark) return 0;+ const char *mtp = getenv("V4_MTP");+ const char *draft = getenv("V4_DRAFT");+ return mtp && atoi(mtp) != 0 && draft && atoi(draft) > 0;+}++static uint64_t v4_dspark_full_reserve_bytes(void) {+ double cache = coli_v4_dspark_cache_gb() * 1e9;+ /* The released Flash checkpoint has ~0.56 GiB of resident MTP dense,+ * projection, Markov and norm tensors. Add head/scratch/tap margin. */+ double total = cache + 768.0 * 1024.0 * 1024.0;+ return total >= (double)UINT64_MAX ? UINT64_MAX : (uint64_t)total;+}++/* Lightweight fallback: keep only the official low-rank Markov head resident+ * when explicitly requested without the complete three-stage drafter. */+static int v4_dspark_markov_probe(ColiV4Engine *engine,+ const ColiSafetensorsTensor **w1_out,+ const ColiSafetensorsTensor **w2_out) {+ if (!engine || !engine->target_index || !w1_out || !w2_out) return -1;+ *w1_out = NULL;+ *w2_out = NULL;+ char name[160];+ for (int stage = 0; stage < 16; stage++) {+ snprintf(name, sizeof(name),+ "mtp.%d.markov_head.markov_w1.weight", stage);+ const ColiSafetensorsTensor *w1 = coli_st_find(engine->target_index,+ name);+ snprintf(name, sizeof(name),+ "mtp.%d.markov_head.markov_w2.weight", stage);+ const ColiSafetensorsTensor *w2 = coli_st_find(engine->target_index,+ name);+ if (!w1 || !w2) continue;+ int rank = engine->config.dspark_markov_rank;+ if (w1->dtype != COLI_ST_BF16 || w2->dtype != COLI_ST_BF16 ||+ w1->rank != 2 || w2->rank != 2 || rank < 1 ||+ w1->shape[0] != engine->config.vocab_size ||+ w2->shape[0] != engine->config.vocab_size ||+ w1->shape[1] != rank || w2->shape[1] != rank ||+ w1->nbytes <= 0 || w2->nbytes <= 0)+ return -1;+ engine->dspark.rank = rank;+ /* A Markov-only draft is intentionally conservative. Two tokens cap+ * rollback cost on prose where the low-rank bigram is uncertain; a+ * benchmarker may raise it up to the checkpoint's trained block. */+ engine->dspark.block_size = 2;+ const char *block_setting = getenv("COLI_V4_MARKOV_BLOCK");+ if (block_setting && atoi(block_setting) > 1)+ engine->dspark.block_size = atoi(block_setting);+ if (engine->dspark.block_size > engine->config.dspark_block_size)+ engine->dspark.block_size = engine->config.dspark_block_size;+ if (engine->dspark.block_size > 8) engine->dspark.block_size = 8;+ if (engine->dspark.block_size < 2) return -1;+ engine->dspark.stage = stage;+ engine->dspark.bytes = (uint64_t)w1->nbytes + (uint64_t)w2->nbytes;+ *w1_out = w1;+ *w2_out = w2;+ return 0;+ }+ return -1;+}++static int v4_dspark_markov_wanted(const ColiV4EngineOpenOptions *options) {+ if (!options || options->no_dspark) return 0;+ const char *setting = getenv("COLI_V4_MARKOV_SPEC");+ /* Opt-in while real-hardware acceptance is being measured. */+ return setting && atoi(setting) != 0;+}++static int v4_dspark_markov_load(ColiV4Engine *engine,+ const ColiSafetensorsTensor *w1,+ const ColiSafetensorsTensor *w2) {+ if (!engine || !w1 || !w2) return -1;+ engine->dspark.markov_w1 = malloc((size_t)w1->nbytes);+ engine->dspark.markov_w2 = malloc((size_t)w2->nbytes);+ if (!engine->dspark.markov_w1 || !engine->dspark.markov_w2 ||+ coli_st_read_tensor(engine->target_index, w1,+ engine->dspark.markov_w1) ||+ coli_st_read_tensor(engine->target_index, w2,+ engine->dspark.markov_w2)) {+ free(engine->dspark.markov_w2);+ free(engine->dspark.markov_w1);+ engine->dspark.markov_w2 = NULL;+ engine->dspark.markov_w1 = NULL;+ engine->dspark.bytes = 0;+ return -1;+ }+ engine->dspark.enabled = 1;+ fprintf(stderr,+ "v4_dspark mode=markov-draft stage=%d block=%d rank=%d "+ "resident=%.3fGiB verification=exact-target\n",+ engine->dspark.stage, engine->dspark.block_size,+ engine->dspark.rank, engine->dspark.bytes / 1073741824.0);+ return 0;+}+#ifdef COLI_V4_TEST_HOOKSColiV4Session *coli_v4_test_session_bare_create(ColiV4Engine *engine) {if (!engine) return NULL;@@ -5914,6 +6443,11 @@ void coli_v4_engine_destroy(ColiV4Engine *engine) {engine->dense_resident.index = NULL;engine->dense_resident.total_bytes = 0;+ free(engine->dspark.markov_w2);+ free(engine->dspark.markov_w1);+ engine->dspark.markov_w2 = NULL;+ engine->dspark.markov_w1 = NULL;+ engine->dspark.enabled = 0;free(engine->head_cache.data);engine->head_cache.data = NULL;if (engine->owns_experts && engine->experts && engine->experts->ops &&@@ -5971,6 +6505,26 @@ int coli_v4_engine_open(ColiV4Engine **output,error_size))goto fail;engine->owns_index = 1;+ const ColiSafetensorsTensor *dspark_w1 = NULL, *dspark_w2 = NULL;+ int want_full_dspark = v4_dspark_full_wanted(options);+ int want_dspark = !want_full_dspark && v4_dspark_markov_wanted(options);+ if (want_dspark && v4_dspark_markov_probe(engine, &dspark_w1,+ &dspark_w2)) {+ fprintf(stderr,+ "v4_dspark warning=compatible-markov-head-not-found; "+ "continuing-target-only\n");+ want_dspark = 0;+ }+ engine->runtime.dspark_reserve_bytes = want_full_dspark+ ? v4_dspark_full_reserve_bytes()+ : (want_dspark ? engine->dspark.bytes : 0);+ coli_v4_full_dspark_wanted = want_full_dspark;+ if (want_full_dspark)+ fprintf(stderr,+ "v4_dspark mode=full-3stage cache=%.2fGB reserve=%.2fGiB "+ "load=lazy verification=exact-target\n",+ coli_v4_dspark_cache_gb(),+ engine->runtime.dspark_reserve_bytes / 1073741824.0);#ifdef COLI_V4_TEST_HOOKSif (coli_v4_test_fail_expert_store_open) {if (error && error_size)@@ -5998,6 +6552,10 @@ int coli_v4_engine_open(ColiV4Engine **output,&engine->experts, error, error_size))goto fail;engine->owns_experts = 1;+ if (want_dspark && v4_dspark_markov_load(engine, dspark_w1, dspark_w2))+ fprintf(stderr,+ "v4_dspark warning=cannot-load-markov-head; "+ "continuing-target-only\n");engine->summary.dense_resident = engine->runtime.dense_resident;engine->summary.head_resident = engine->head_cache.data != NULL;engine->summary.expert_cache_bytes =@@ -6096,6 +6654,9 @@ int coli_v4_prompt_build(char **output, size_t *output_length,#include <stdio.h>#include <stdlib.h>#include <string.h>+#ifdef __AVX2__+#include <immintrin.h>+#endif#include "deepseek_v4_internal.h"#include "deepseek_v4_internal.h"@@ -6170,22 +6731,84 @@ static int final_hidden(float *output, const float *state,return 0;}+static float head_bf16_dot(const uint16_t *weight, const float *hidden,+ int dimension) {+ float sum = 0.0f;+ int column = 0;+#ifdef __AVX2__+ for (; column + 8 <= dimension; column += 8) {+ float products[8];+ __m128i packed = _mm_loadu_si128(+ (const __m128i *)(weight + column));+ __m256i bits = _mm256_slli_epi32(+ _mm256_cvtepu16_epi32(packed), 16);+ _mm256_storeu_ps(products, _mm256_mul_ps(+ _mm256_castsi256_ps(bits),+ _mm256_loadu_ps(hidden + column)));+ sum += products[0];+ sum += products[1];+ sum += products[2];+ sum += products[3];+ sum += products[4];+ sum += products[5];+ sum += products[6];+ sum += products[7];+ }+#endif+ for (; column < dimension; column++)+ sum += coli_bf16_decode(weight[column]) * hidden[column];+ return sum;+}+static int head_argmax(ColiV4Engine *engine, const float *hidden,const ColiSafetensorsIndex *index,const ColiDeepSeekV4Config *config,int *best_token, float *best_logit) {const ColiSafetensorsTensor *head = coli_st_find(index, "head.weight");int d = config->hidden_size, vocab = config->vocab_size;+ if (!head || head->dtype != COLI_ST_BF16 || d < 1 || vocab < 1)+ return -1;+ int shard = coli_st_tensor_shard(index, head);+ size_t resident_bytes = (size_t)vocab * (size_t)d * sizeof(uint16_t);+ const uint16_t *resident = coli_v4_head_cache_data(+ engine, shard, (uint64_t)head->off, resident_bytes);++ /* The normal V4 memory plan keeps the BF16 head resident. Compute all+ * rows in one OpenMP team directly from that allocation: the old tiled+ * path copied the complete ~1 GiB head and created ~2,000 teams per token.+ * Each row retains the same scalar accumulation order and the final scan+ * retains vocabulary order, so logits/tie-breaking do not change. */+ if (resident) {+ float *scores = malloc((size_t)vocab * sizeof(*scores));+ if (!scores) return -1;+ #pragma omp parallel for schedule(static)+ for (int row = 0; row < vocab; row++) {+ const uint16_t *weight = resident + (size_t)row * d;+ scores[row] = head_bf16_dot(weight, hidden, d);+ }+ int winner = -1;+ float maximum = -FLT_MAX;+ for (int row = 0; row < vocab; row++)+ if (scores[row] > maximum) {+ maximum = scores[row];+ winner = row;+ }+ free(scores);+ *best_token = winner;+ *best_logit = maximum;+ return winner < 0 ? -1 : 0;+ }++ /* Low-memory fallback: stream small row tiles exactly as before. */enum { ROWS = 64 };uint16_t *raw = malloc((size_t)ROWS * d * sizeof(*raw));float *scores = malloc((size_t)ROWS * sizeof(*scores));- if (!head || head->dtype != COLI_ST_BF16 || !raw || !scores) {+ if (!raw || !scores) {free(scores); free(raw);return -1;}int winner = -1;float maximum = -FLT_MAX;- int shard = coli_st_tensor_shard(index, head);for (int start = 0; start < vocab; start += ROWS) {int rows = vocab - start < ROWS ? vocab - start : ROWS;size_t bytes = (size_t)rows * d * sizeof(*raw);@@ -6198,11 +6821,8 @@ static int head_argmax(ColiV4Engine *engine, const float *hidden,}#pragma omp parallel forfor (int row = 0; row < rows; row++) {- float sum = 0.0f;const uint16_t *weight = raw + (size_t)row * d;- for (int i = 0; i < d; i++)- sum += coli_bf16_decode(weight[i]) * hidden[i];- scores[row] = sum;+ scores[row] = head_bf16_dot(weight, hidden, d);}for (int row = 0; row < rows; row++)if (scores[row] > maximum) {@@ -6216,6 +6836,83 @@ static int head_argmax(ColiV4Engine *engine, const float *hidden,return winner < 0 ? -1 : 0;}+static int head_argmax_batch(ColiV4Engine *engine, const float *hidden,+ const ColiSafetensorsIndex *index,+ const ColiDeepSeekV4Config *config, int batch,+ int *best_tokens, float *best_logits) {+ if (!engine || !hidden || !index || !config || batch < 1 ||+ !best_tokens || !best_logits) return -1;+ if (batch == 1)+ return head_argmax(engine, hidden, index, config, best_tokens,+ best_logits);+ const ColiSafetensorsTensor *head = coli_st_find(index, "head.weight");+ int d = config->hidden_size, vocab = config->vocab_size;+ if (!head || head->dtype != COLI_ST_BF16) return -1;+ int shard = coli_st_tensor_shard(index, head);+ const uint16_t *resident = coli_v4_head_cache_data(+ engine, shard, (uint64_t)head->off,+ (size_t)vocab * d * sizeof(uint16_t));+ if (!resident) {+ for (int item = 0; item < batch; item++)+ if (head_argmax(engine, hidden + (size_t)item * d, index, config,+ &best_tokens[item], &best_logits[item])) return -1;+ return 0;+ }+ float *scores = malloc((size_t)vocab * batch * sizeof(*scores));+ if (!scores) return -1;+ #pragma omp parallel for schedule(static)+ for (int row = 0; row < vocab; row++) {+ const uint16_t *weight = resident + (size_t)row * d;+ for (int item = 0; item < batch; item++)+ scores[(size_t)item * vocab + row] = head_bf16_dot(+ weight, hidden + (size_t)item * d, d);+ }+ for (int item = 0; item < batch; item++) {+ int winner = -1;+ float maximum = -FLT_MAX;+ const float *item_scores = scores + (size_t)item * vocab;+ for (int row = 0; row < vocab; row++)+ if (item_scores[row] > maximum) {+ maximum = item_scores[row];+ winner = row;+ }+ best_tokens[item] = winner;+ best_logits[item] = maximum;+ }+ free(scores);+ return 0;+}++static int dspark_markov_argmax(const ColiV4Engine *engine, int token,+ int *best_token) {+ if (!engine || !engine->dspark.enabled || !best_token || token < 0 ||+ token >= engine->config.vocab_size) return -1;+ int rank = engine->dspark.rank, vocab = engine->config.vocab_size;+ float *embedding = malloc((size_t)rank * sizeof(*embedding));+ float *scores = malloc((size_t)vocab * sizeof(*scores));+ if (!embedding || !scores) {+ free(scores); free(embedding); return -1;+ }+ const uint16_t *source = engine->dspark.markov_w1 + (size_t)token * rank;+ for (int column = 0; column < rank; column++)+ embedding[column] = coli_bf16_decode(source[column]);+ #pragma omp parallel for schedule(static)+ for (int row = 0; row < vocab; row++)+ scores[row] = head_bf16_dot(+ engine->dspark.markov_w2 + (size_t)row * rank,+ embedding, rank);+ int winner = -1;+ float maximum = -FLT_MAX;+ for (int row = 0; row < vocab; row++)+ if (scores[row] > maximum) {+ maximum = scores[row];+ winner = row;+ }+ free(scores); free(embedding);+ *best_token = winner;+ return winner < 0 ? -1 : 0;+}+static int has_sentence_end(const char *text, int length) {for (int i = 0; i < length; i++) {unsigned char value = (unsigned char)text[i];@@ -6398,6 +7095,55 @@ static int spec_sentence_end(const char *text, int length) {return 0;}+/* DSpark conditions on the target model's final three layer hiddens. Keep a+ * short ring only when the full drafter is enabled; target-only runs pay no+ * allocation and no per-layer reduction. */+#define V4_MAINH_TARGETS 3+#define V4_MAINH_RING 128+static float *g_mainh_ring;+static int64_t g_mainh_abs[V4_MAINH_RING];+static int g_mainh_dim;++static void v4_mainh_tap(const ColiDeepSeekV4Config *config, int layer_id,+ const float *row, int64_t position) {+ if (!coli_v4_full_dspark_wanted || !config || !row || position < 0) return;+ int first = config->num_hidden_layers - V4_MAINH_TARGETS;+ int target = layer_id - first;+ if (target < 0 || target >= V4_MAINH_TARGETS) return;+ int dimension = config->hidden_size;+ int copies = config->hc_mult;+ if (!g_mainh_ring) {+ g_mainh_ring = calloc(+ (size_t)V4_MAINH_RING * V4_MAINH_TARGETS * dimension,+ sizeof(float));+ if (!g_mainh_ring) return;+ for (int item = 0; item < V4_MAINH_RING; item++)+ g_mainh_abs[item] = -1;+ g_mainh_dim = dimension;+ }+ if (g_mainh_dim != dimension) return;+ int ring = (int)(position % V4_MAINH_RING);+ float *slot = g_mainh_ring ++ ((size_t)ring * V4_MAINH_TARGETS + target) * dimension;+ for (int column = 0; column < dimension; column++) {+ float sum = 0.0f;+ for (int copy = 0; copy < copies; copy++)+ sum += row[(size_t)copy * dimension + column];+ slot[column] = coli_bf16_round(sum / copies);+ }+ if (target == V4_MAINH_TARGETS - 1) g_mainh_abs[ring] = position;+}++static const float *v4_mainh_get(int64_t position) {+ if (!g_mainh_ring || position < 0) return NULL;+ int slot = (int)(position % V4_MAINH_RING);+ if (g_mainh_abs[slot] != position) return NULL;+ return g_mainh_ring ++ (size_t)slot * V4_MAINH_TARGETS * g_mainh_dim;+}++#include "deepseek_v4_dspark.inc"+static int target_batch(ColiV4Engine *engine, float **state_ptr, float **next_ptr,ColiDeepSeekV4WindowAttentionState **attention,const ColiSafetensorsIndex *index,@@ -6432,6 +7178,9 @@ static int target_batch(ColiV4Engine *engine, float **state_ptr, float **next_ptcoli_v4_layer_free(engine, &layer);if (result) return -1;float *swap = state; state = next; next = swap;+ for (int item = 0; item < batch; item++)+ v4_mainh_tap(config, layer_id, state + (size_t)item * hd,+ (int64_t)start + item);}*state_ptr = state;*next_ptr = next;@@ -6456,12 +7205,48 @@ static int target_token(ColiV4Engine *engine, float **state_ptr, float **next_ptcoli_v4_layer_free(engine, &layer);if (result) return -1;float *swap = state; state = next; next = swap;+ v4_mainh_tap(config, layer_id, state, position);}*state_ptr = state;*next_ptr = next;return 0;}+static ColiV4AttentionSnapshot **spec_attention_save(+ ColiDeepSeekV4WindowAttentionState **attention, int layers) {+ if (!attention || layers < 1) return NULL;+ ColiV4AttentionSnapshot **snapshots = calloc(+ (size_t)layers, sizeof(*snapshots));+ if (!snapshots) return NULL;+ for (int layer = 0; layer < layers; layer++)+ if (coli_v4_attention_snapshot_create(attention[layer],+ &snapshots[layer])) {+ for (int item = 0; item < layers; item++)+ coli_v4_attention_snapshot_destroy(snapshots[item]);+ free(snapshots);+ return NULL;+ }+ return snapshots;+}++static int spec_attention_restore(+ ColiDeepSeekV4WindowAttentionState **attention,+ ColiV4AttentionSnapshot **snapshots, int layers) {+ if (!attention || !snapshots) return -1;+ for (int layer = 0; layer < layers; layer++)+ if (coli_v4_attention_snapshot_restore(attention[layer],+ snapshots[layer])) return -1;+ return 0;+}++static void spec_attention_free(ColiV4AttentionSnapshot **snapshots,+ int layers) {+ if (!snapshots) return;+ for (int layer = 0; layer < layers; layer++)+ coli_v4_attention_snapshot_destroy(snapshots[layer]);+ free(snapshots);+}+#undef spec_print#undef COLI_V4_GENERATE_HELPERS_ONLY#undef COLI_V4_GENERATE_MAIN@@ -6523,7 +7308,7 @@ static void v4_cli_usage(FILE *stream, const char *program) {" --thinking enable the official V4 thinking prefix\n"" --raw-prompt bypass the default V4 chat template\n"" --stop-sentence stop after the first sentence terminator\n"- " --no-dspark accepted for compatibility; target-only is always greedy\n"+ " --no-dspark disable verified speculative drafting\n"" --oracle FILE validate against an oracle JSON fixture\n"" --teacher-forcing N oracle: compare top-1 on N prompt positions\n"" --greedy N oracle: compare N greedy continuation tokens\n"@@ -6862,6 +7647,50 @@ static int session_emit_token(ColiV4Session *session,return token == 1;}+static int v4_context_token(const int *prompt, int prompt_count,+ const int *generated, int generated_count,+ int position) {+ if (position < 0 || position >= prompt_count + generated_count) return -1;+ return position < prompt_count ? prompt[position]+ : generated[position - prompt_count];+}++/* Prompt-lookup drafting costs no model I/O. Match the context tail against+ * its most recent earlier 3-gram/2-gram occurrence and propose the following+ * tokens. Every proposal is still checked by the exact target batch below. */+static int v4_ngram_draft(const int *prompt, int prompt_count,+ const int *generated, int generated_count,+ int *output, int maximum) {+ int count = prompt_count + generated_count;+ if (!prompt || !generated || !output || maximum < 1) return 0;+ for (int gram = 3; gram >= 2; gram--) {+ if (count < gram + 1) continue;+ int tail = count - gram;+ for (int start = count - gram - 1; start >= 0; start--) {+ int matches = 1;+ for (int item = 0; item < gram; item++)+ if (v4_context_token(prompt, prompt_count, generated,+ generated_count, start + item) !=+ v4_context_token(prompt, prompt_count, generated,+ generated_count, tail + item)) {+ matches = 0;+ break;+ }+ if (!matches) continue;+ int from = start + gram;+ int take = count - from;+ if (take > maximum) take = maximum;+ if (take < 1) break;+ for (int item = 0; item < take; item++)+ output[item] = v4_context_token(+ prompt, prompt_count, generated, generated_count,+ from + item);+ return take;+ }+ }+ return 0;+}+int coli_v4_session_generate(ColiV4Session *session,const char *prompt, size_t prompt_length,const ColiV4SessionGenerateOptions *options,@@ -6881,6 +7710,10 @@ int coli_v4_session_generate(ColiV4Session *session,session->prompt_count = 0;session->generated_count = 0;session->prefix_reused = 0;+ session->spec_attempts = 0;+ session->spec_drafted = 0;+ session->spec_accepted = 0;+ session->spec_disabled = 0;int max_new = options->max_new_tokens;if (max_new > session->max_new_tokens_cap)@@ -6926,9 +7759,11 @@ int coli_v4_session_generate(ColiV4Session *session,* would have produced.* ------------------------------------------------------------------- */int reuse = kv_prefix_reuse(&session->fed, session->prompt_ids, prompt_count);- if (!reuse)+ if (!reuse) {for (int layer = 0; layer < config->num_hidden_layers; layer++)coli_v4_window_attention_reset(session->attention[layer]);+ if (coli_v4_full_dspark_wanted) v4_ds_reset_history();+ }session->prefix_reused = reuse;if (reuse && getenv("V4_PREFIX_LOG"))fprintf(stderr, "[PREFIX] reusing %d of %d prompt tokens\n",@@ -6977,7 +7812,220 @@ int coli_v4_session_generate(ColiV4Session *session,generated_count, options->stop_at_sentence);double first_at = spec_now();+ int draft_limit = getenv("V4_DRAFT") ? atoi(getenv("V4_DRAFT")) : 0;+ if (draft_limit < 0) draft_limit = 0;+ if (draft_limit > 24) draft_limit = 24;+ int markov_disabled = 0;+ int mtp_disabled = 0;+ int full_mtp_ready = 0;+while (!done && generated_count < max_new) {+ int remaining = max_new - generated_count;+ if (!options->no_dspark && !session->spec_disabled && remaining >= 3) {+ int inputs[25] = {0}, drafts[24] = {0};+ int predictions[25] = {0};+ float logits[25] = {0};+ int room = remaining - 1; /* verification emits one exact fallback */+ int proposals = draft_limit < room ? draft_limit : room;+ const char *ngram_env = getenv("V4_NGRAM");+ int ngram_enabled = !ngram_env || atoi(ngram_env) != 0;+ if (proposals > 1 && ngram_enabled)+ proposals = v4_ngram_draft(+ session->prompt_ids, prompt_count, generated,+ generated_count, drafts, proposals);+ else+ proposals = 0;+ int using_full_mtp = 0;+ int using_markov = 0;+ int using_ngram = proposals > 1;+ inputs[0] = current;+ int draft_ready = proposals > 1;+ if (!draft_ready && coli_v4_full_dspark_wanted && full_mtp_ready &&+ !mtp_disabled) {+ int mtp_min = getenv("V4_MTP_MIN")+ ? atoi(getenv("V4_MTP_MIN")) : 3;+ if (mtp_min < 1) mtp_min = 1;+ int maximum = draft_limit < room ? draft_limit : room;+ int mtp_max = getenv("V4_MTP_DRAFT")+ ? atoi(getenv("V4_MTP_DRAFT")) : 3;+ if (mtp_max < 1) mtp_max = 1;+ if (maximum > mtp_max) maximum = mtp_max;+ if (maximum >= mtp_min) {+ proposals = v4_dspark_draft(+ engine, index, config, current, last_processed + 1,+ drafts, maximum);+ draft_ready = proposals > 0;+ using_full_mtp = draft_ready;+ if (!draft_ready) mtp_disabled = 1;+ }+ } else if (!draft_ready && engine->dspark.enabled &&+ !markov_disabled) {+ proposals = engine->dspark.block_size;+ if (proposals > room) proposals = room;+ draft_ready = proposals > 1;+ using_markov = draft_ready;+ for (int item = 0; draft_ready && item < proposals; item++) {+ if (dspark_markov_argmax(engine, inputs[item],+ &drafts[item])) {+ draft_ready = 0;+ break;+ }+ if (item + 1 < proposals) inputs[item + 1] = drafts[item];+ }+ }+ if (!draft_ready) {+ if (using_markov) markov_disabled = 1;+ } else {+ int batch = proposals + 1;+ for (int item = 1; item < batch; item++)+ inputs[item] = drafts[item - 1];+ ColiV4AttentionSnapshot **snapshots = spec_attention_save(+ attention, config->num_hidden_layers);+ if (!snapshots) {+ session->spec_disabled = 1;+ } else {+ int old_last = last_processed;+ for (int item = 0; item < batch; item++)+ if (load_embedding(state + (size_t)item * hd, index,+ config, inputs[item])) {+ spec_attention_free(snapshots,+ config->num_hidden_layers);+ kv_prefix_taint(&session->fed);+ if (error && error_size)+ snprintf(error, error_size,+ "cannot load speculative embedding");+ return -1;+ }+ if (target_batch(engine, &state, &next, attention, index,+ config, experts, inputs, old_last + 1,+ batch, error, error_size)) {+ (void)spec_attention_restore(+ attention, snapshots, config->num_hidden_layers);+ spec_attention_free(snapshots,+ config->num_hidden_layers);+ kv_prefix_taint(&session->fed);+ return -1;+ }+ float *batch_hidden = malloc(+ (size_t)batch * config->hidden_size * sizeof(float));+ int heads_ok = batch_hidden != NULL;+ for (int item = 0; heads_ok && item < batch; item++)+ if (final_hidden(+ batch_hidden ++ (size_t)item * config->hidden_size,+ state + (size_t)item * hd, index, config,+ error, error_size)) heads_ok = 0;+ if (heads_ok && head_argmax_batch(+ engine, batch_hidden, index, config, batch,+ predictions, logits)) heads_ok = 0;+ free(batch_hidden);+ if (!heads_ok) {+ (void)spec_attention_restore(+ attention, snapshots, config->num_hidden_layers);+ spec_attention_free(snapshots,+ config->num_hidden_layers);+ kv_prefix_taint(&session->fed);+ if (error && error_size && !error[0])+ snprintf(error, error_size,+ "speculative target head failed");+ return -1;+ }++ int accepted = 0;+ while (accepted < proposals &&+ predictions[accepted] == drafts[accepted])+ accepted++;+ session->spec_attempts++;+ session->spec_drafted += (uint64_t)proposals;+ session->spec_accepted += (uint64_t)accepted;+ if (using_full_mtp)+ v4_dspark_feedback(proposals, accepted);+ /* With recurrent compressed attention, a rejected suffix+ * cannot be truncated in place: the accepted prefix must+ * be replayed. Real chat measured 10/24 accepted and+ * 495 s for 14 visible tokens, so stop full MTP after the+ * first non-perfect block unless explicitly benchmarking+ * it. Exact prompt lookup remains available. */+ if (using_full_mtp && accepted < proposals) {+ const char *keep = getenv("V4_MTP_PARTIAL_KEEP");+ if (!keep || atoi(keep) == 0) mtp_disabled = 1;+ }+ if (using_ngram && accepted < proposals) {+ const char *keep = getenv("V4_NGRAM_PARTIAL_KEEP");+ if (!keep || atoi(keep) == 0)+ session->spec_disabled = 1;+ }+ int available_outputs = accepted + 1;+ int retained = 0;+ for (int item = 0;+ item < available_outputs && !done &&+ generated_count < max_new; item++) {+ current = predictions[item];+ current_logit = logits[item];+ generated[generated_count++] = current;+ retained++;+ done = session_emit_token(+ session, on_token, user_data, current,+ current_logit, old_last + 1 + item,+ generated_count, options->stop_at_sentence);+ }++ /* A mismatch in the first row (or an early callback stop)+ * leaves unverified draft inputs in the batched KV state.+ * Restore the exact snapshot and replay only inputs that+ * really correspond to emitted outputs. */+ if (retained < batch) {+ if (spec_attention_restore(+ attention, snapshots,+ config->num_hidden_layers)) {+ spec_attention_free(+ snapshots, config->num_hidden_layers);+ kv_prefix_taint(&session->fed);+ if (error && error_size)+ snprintf(error, error_size,+ "cannot restore speculative KV state");+ return -1;+ }+ if (coli_v4_full_dspark_wanted)+ v4_ds_invalidate_from(old_last + 1);+ for (int item = 0; item < retained; item++)+ if (load_embedding(state + (size_t)item * hd,+ index, config, inputs[item])) {+ spec_attention_free(+ snapshots, config->num_hidden_layers);+ kv_prefix_taint(&session->fed);+ if (error && error_size)+ snprintf(error, error_size,+ "cannot replay speculative input");+ return -1;+ }+ if (retained > 0 && target_batch(+ engine, &state, &next, attention, index,+ config, experts, inputs, old_last + 1,+ retained, error, error_size)) {+ spec_attention_free(+ snapshots, config->num_hidden_layers);+ kv_prefix_taint(&session->fed);+ return -1;+ }+ }+ spec_attention_free(snapshots,+ config->num_hidden_layers);+ if (retained > 0) {+ kv_prefix_record(&session->fed, inputs, old_last + 1,+ retained);+ last_processed = old_last + retained;+ full_mtp_ready = 1;+ }+ session->state = state;+ session->next = next;+ const char *keep = getenv("COLI_V4_MARKOV_KEEP");+ if (using_markov && accepted == 0 &&+ (!keep || atoi(keep) == 0)) markov_disabled = 1;+ continue;+ }+ }+ }int position = last_processed + 1;if (target_token(engine, &state, &next, attention, index, config, experts,current, position, error, error_size)) {@@ -7001,7 +8049,9 @@ int coli_v4_session_generate(ColiV4Session *session,current_logit, last_processed,generated_count,options->stop_at_sentence);+ full_mtp_ready = 1;}+ if (coli_v4_full_dspark_wanted) v4_dspark_report();double ended = spec_now();session->state = state;session->next = next;@@ -7027,7 +8077,20 @@ int coli_v4_session_generate(ColiV4Session *session,generated[generated_count - 1] == 1;stats_out->time_to_first_token_sec = first_at - setup_done;stats_out->decode_sec = ended - first_at;+ stats_out->speculative_drafted = session->spec_drafted;+ stats_out->speculative_accepted = session->spec_accepted;}+ if (session->spec_attempts)+ fprintf(stderr,+ "v4_dspark attempts=%llu drafted=%llu accepted=%llu "+ "acceptance=%.1f%% adaptive_disabled=%d\n",+ (unsigned long long)session->spec_attempts,+ (unsigned long long)session->spec_drafted,+ (unsigned long long)session->spec_accepted,+ session->spec_drafted+ ? 100.0 * session->spec_accepted / session->spec_drafted+ : 0.0,+ session->spec_disabled);return 0;}@@ -7343,7 +8406,7 @@ static void v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,&(ColiV4SessionGenerateOptions){.max_new_tokens = request->max_tokens,.stop_at_sentence = 0,- .no_dspark = 1,+ .no_dspark = 0,},v4_serve_token, &stream, &stats, error, sizeof(error));double elapsed = spec_now() - started;@@ -7390,7 +8453,7 @@ static int v4_serve_main(void) {.target_model_dir = model_dir,.context_tokens = context,.pin_slots_per_layer = -1,- .no_dspark = 1,+ .no_dspark = 0,};const char *ram = getenv("RAM_GB");if (ram && atof(ram) > 0.0)@@ -7595,13 +8658,13 @@ int main(int argc, char **argv) {fprintf(stderr, "cannot build DeepSeek V4 prompt\n");goto cleanup;}- if (cli.no_dspark)- fprintf(stderr, "note: --no-dspark is a compatibility no-op; "- "this build is target-only\n");- fprintf(stderr, "v4_cli mode=%s memory=%s target_only=1\n",+ int target_only = !engine->dspark.enabled || cli.no_dspark;+ if (cli.no_dspark && engine->dspark.enabled)+ fprintf(stderr, "note: speculative drafting disabled by CLI\n");+ fprintf(stderr, "v4_cli mode=%s memory=%s target_only=%d\n",cli.prompt_mode == COLI_V4_PROMPT_RAW ? "raw" :cli.prompt_mode == COLI_V4_PROMPT_THINKING ? "thinking" : "chat",- cli.memory_gib > 0.0 ? "limited" : "auto");+ cli.memory_gib > 0.0 ? "limited" : "auto", target_only);int session_context = engine->runtime.context_tokens;ColiV4SessionCreateOptions session_opts = {@@ -7639,13 +8702,13 @@ int main(int argc, char **argv) {experts->ops->stats(experts, &stats_end);fprintf(stderr, "v4_tokens prompt=%d generated=%d total=%d ""expert_requests=%llu hits=%llu misses=%llu hit_rate=%.3f "- "bytes=%llu target_only=1\n",+ "bytes=%llu target_only=%d\n",gen_stats.prompt_tokens, gen_stats.generated_tokens,gen_stats.prompt_tokens + gen_stats.generated_tokens,(unsigned long long)stats_end.requests,(unsigned long long)stats_end.hits,(unsigned long long)stats_end.misses, stats_hit_rate(stats_end),- (unsigned long long)stats_end.bytes_read);+ (unsigned long long)stats_end.bytes_read, target_only);fprintf(stderr, "generated_text=");if (out_len) fwrite(out_text, 1, out_len, stderr);fprintf(stderr, "\ntiming time_to_first_token=%.3fs after_first=%.3fs\n",@@ -8003,7 +9066,9 @@ int coli_v4_shared_expert_forward_ref(float *output,#ifdef COLI_V4_UNIT_EXPERT_STORE/* ######## deepseek_v4_expert_store.c ######## */+#ifndef _GNU_SOURCE#define _GNU_SOURCE+#endif#include "deepseek_v4_internal.h"#include <assert.h>@@ -8203,11 +9268,13 @@ static int lookup(ColiExpertStore *store, ColiExpertKey key,}/* A short read must never expose a partially overwritten old slot. */slot->expert = -1;- if (coli_st_read_at(state->index, record->shard, record->scale_offset,- (size_t)record->scale_bytes, slot->slab) != 0 ||- coli_st_read_at(state->index, record->shard, record->weight_offset,- (size_t)record->weight_bytes,- slot->slab + record->scale_bytes) != 0) {+ if (coli_st_read_at_streaming(+ state->index, record->shard, record->scale_offset,+ (size_t)record->scale_bytes, slot->slab) != 0 ||+ coli_st_read_at_streaming(+ state->index, record->shard, record->weight_offset,+ (size_t)record->weight_bytes,+ slot->slab + record->scale_bytes) != 0) {pthread_mutex_unlock(&state->mutex);memset(view, 0, sizeof(*view));return -1;@@ -8683,6 +9750,16 @@ static int required_int(jval *root, const char *name, int *output,return 0;}+static int optional_int(jval *root, const char *name, int *output,+ char *error, size_t error_size) {+ jval *value = json_get(root, name);+ if (!value) return 0;+ if (json_int_value(value, output) != 0)+ return set_error(error, error_size,+ "invalid integer config field: %s", name);+ return 0;+}+static int required_float(jval *root, const char *name, float *output,char *error, size_t error_size) {jval *value = json_get(root, name);@@ -8752,6 +9829,16 @@ int coli_v4_config_parse(ColiDeepSeekV4Config *config, const char *json,free(arena);return -1;}+ if (optional_int(root, "dspark_block_size", &config->dspark_block_size,+ error, error_size) ||+ optional_int(root, "dspark_noise_token_id",+ &config->dspark_noise_token_id, error, error_size) ||+ optional_int(root, "dspark_markov_rank",+ &config->dspark_markov_rank, error, error_size)) {+ json_free(root);+ free(arena);+ return -1;+ }jval *rope = json_get(root, "rope_scaling");if (!rope || rope->t != J_OBJ ||required_int(rope, "original_max_position_embeddings",@@ -8863,6 +9950,9 @@ int coli_v4_config_load(ColiDeepSeekV4Config *config, const char *model_dir,#include <math.h>#include <stdlib.h>#include <string.h>+#ifdef __AVX2__+#include <immintrin.h>+#endiffloat coli_e8m0_decode(uint8_t value) {if (value == 0xff) return NAN;@@ -9088,7 +10178,8 @@ int coli_fp8_matvec_ref(float *output, const ColiTensorView *weight,weight->scale_format != COLI_SCALE_F32 ||!weight->data || !weight->scales || weight->rows < 1 ||weight->columns < 1 || weight->columns % 128 ||- weight->block_rows != 128 || weight->block_columns != 128)+ (weight->block_rows != 128 && weight->block_rows != 8) ||+ weight->block_columns != 128)return -1;size_t rows = (size_t)weight->rows;size_t columns = (size_t)weight->columns;@@ -9110,6 +10201,39 @@ int coli_fp8_matvec_ref(float *output, const ColiTensorView *weight,free(activation_scales);return -1;}+#ifdef __AVX2__+ if (weight->block_rows == 8) {+ if (rows % 8) {+ free(activation_scales); free(activation); return -1;+ }+ float fp8[256];+ for (int code = 0; code < 256; code++)+ fp8[code] = coli_e4m3fn_decode((uint8_t)code);+ const uint8_t *data = weight->data;+ const float *scales = weight->scales;+ #pragma omp parallel for schedule(static)+ for (int64_t tile = 0; tile < weight->rows / 8; tile++) {+ __m256 sum = _mm256_setzero_ps();+ size_t scale_row = ((size_t)tile * 8) / 128;+ for (size_t base = 0; base < columns; base += 128) {+ __m256 scale = _mm256_set1_ps(+ scales[scale_row * scale_columns + base / 128]);+ for (size_t offset = 0; offset < 128; offset++) {+ size_t column = base + offset;+ __m128i bytes = _mm_loadl_epi64((const __m128i *)(data ++ ((size_t)tile * columns + column) * 8));+ __m256i codes = _mm256_cvtepu8_epi32(bytes);+ __m256 values = _mm256_i32gather_ps(fp8, codes, 4);+ __m256 x = _mm256_set1_ps(activation[column]);+ sum = _mm256_add_ps(sum, _mm256_mul_ps(+ _mm256_mul_ps(x, values), scale));+ }+ }+ _mm256_storeu_ps(output + (size_t)tile * 8, sum);+ }+ free(activation_scales); free(activation); return 0;+ }+#endifmatmul_fp8(output, activation, weight->data, weight->scales,1, (int)columns, (int)rows);free(activation_scales);@@ -9124,6 +10248,9 @@ int coli_fp8_matvec_ref(float *output, const ColiTensorView *weight,#include <stdint.h>#include <stdlib.h>+#ifdef __AVX2__+#include <immintrin.h>+#endif#include "native_quant.h"#if defined(__GNUC__)@@ -9185,7 +10312,8 @@ int coli_fp8_dual_matvec_ref(float *output_a, float *output_b,a->scale_format != COLI_SCALE_F32 ||b->scale_format != COLI_SCALE_F32 || !a->data || !b->data ||!a->scales || !b->scales || a->rows < 1 || a->columns < 1 ||- a->columns % 128 || a->block_rows != 128 ||+ a->columns % 128 ||+ (a->block_rows != 128 && a->block_rows != 8) ||a->block_columns != 128) return -1;size_t rows = (size_t)a->rows, columns = (size_t)a->columns;size_t scale_rows = (rows + 127) / 128, scale_columns = columns / 128;@@ -9201,6 +10329,47 @@ int coli_fp8_dual_matvec_ref(float *output_a, float *output_b,input, columns, 128) != 0) {free(activation_scales); free(activation); return -1;}+#ifdef __AVX2__+ if (a->block_rows == 8) {+ if (rows % 8) {+ free(activation_scales); free(activation); return -1;+ }+ float fp8[256];+ for (int code = 0; code < 256; code++)+ fp8[code] = coli_e4m3fn_decode((uint8_t)code);+ const uint8_t *data_a = a->data, *data_b = b->data;+ const float *scales_a = a->scales, *scales_b = b->scales;+ #pragma omp parallel for schedule(static)+ for (int64_t tile = 0; tile < a->rows / 8; tile++) {+ __m256 sum_a = _mm256_setzero_ps();+ __m256 sum_b = _mm256_setzero_ps();+ size_t scale_row = ((size_t)tile * 8) / 128;+ for (size_t base = 0; base < columns; base += 128) {+ size_t scale_index = scale_row * scale_columns + base / 128;+ __m256 scale_a = _mm256_set1_ps(scales_a[scale_index]);+ __m256 scale_b = _mm256_set1_ps(scales_b[scale_index]);+ for (size_t offset = 0; offset < 128; offset++) {+ size_t column = base + offset;+ size_t packed = ((size_t)tile * columns + column) * 8;+ __m256i codes_a = _mm256_cvtepu8_epi32(_mm_loadl_epi64(+ (const __m128i *)(data_a + packed)));+ __m256i codes_b = _mm256_cvtepu8_epi32(_mm_loadl_epi64(+ (const __m128i *)(data_b + packed)));+ __m256 values_a = _mm256_i32gather_ps(fp8, codes_a, 4);+ __m256 values_b = _mm256_i32gather_ps(fp8, codes_b, 4);+ __m256 x = _mm256_set1_ps(activation[column]);+ sum_a = _mm256_add_ps(sum_a, _mm256_mul_ps(+ _mm256_mul_ps(x, values_a), scale_a));+ sum_b = _mm256_add_ps(sum_b, _mm256_mul_ps(+ _mm256_mul_ps(x, values_b), scale_b));+ }+ }+ _mm256_storeu_ps(output_a + (size_t)tile * 8, sum_a);+ _mm256_storeu_ps(output_b + (size_t)tile * 8, sum_b);+ }+ free(activation_scales); free(activation); return 0;+ }+#endifmatmul_fp8(output_a, activation, a->data, a->scales,1, (int)columns, (int)rows);matmul_fp8(output_b, activation, b->data, b->scales,@@ -9215,6 +10384,9 @@ int coli_fp8_dual_matvec_ref(float *output_a, float *output_b,#include <stdint.h>#include <stdlib.h>+#ifdef __AVX2__+#include <immintrin.h>+#endif#include "native_quant.h"#if defined(__GNUC__)@@ -9233,7 +10405,8 @@ int coli_fp8_matmul_batch_ref(float *outputs, const ColiTensorView *weight,weight->scale_format != COLI_SCALE_F32 ||!weight->data || !weight->scales || weight->rows < 1 ||weight->columns < 1 || weight->columns % 128 ||- weight->block_rows != 128 || weight->block_columns != 128) return -1;+ (weight->block_rows != 128 && weight->block_rows != 8) ||+ weight->block_columns != 128) return -1;size_t rows = (size_t)weight->rows, columns = (size_t)weight->columns;size_t scale_rows = (rows + 127) / 128;size_t scale_columns = columns / 128;@@ -9251,6 +10424,47 @@ int coli_fp8_matmul_batch_ref(float *outputs, const ColiTensorView *weight,inputs + (size_t)item * columns, columns, 128) != 0) {free(activation_scales); free(activations); return -1;}+#ifdef __AVX2__+ if (weight->block_rows == 8) {+ if (rows % 8) {+ free(activation_scales); free(activations); return -1;+ }+ float fp8[256];+ for (int code = 0; code < 256; code++)+ fp8[code] = coli_e4m3fn_decode((uint8_t)code);+ const uint8_t *data = weight->data;+ const float *scales = weight->scales;+ #pragma omp parallel for schedule(static)+ for (int64_t tile = 0; tile < weight->rows / 8; tile++) {+ __m256 sums[64];+ for (int item = 0; item < batch; item++)+ sums[item] = _mm256_setzero_ps();+ size_t scale_row = ((size_t)tile * 8) / 128;+ for (size_t base = 0; base < columns; base += 128) {+ __m256 scale = _mm256_set1_ps(+ scales[scale_row * scale_columns + base / 128]);+ for (size_t offset = 0; offset < 128; offset++) {+ size_t column = base + offset;+ __m128i bytes = _mm_loadl_epi64((const __m128i *)(data ++ ((size_t)tile * columns + column) * 8));+ __m256i codes = _mm256_cvtepu8_epi32(bytes);+ __m256 values = _mm256_mul_ps(+ _mm256_i32gather_ps(fp8, codes, 4), scale);+ for (int item = 0; item < batch; item++) {+ __m256 x = _mm256_set1_ps(+ activations[(size_t)item * columns + column]);+ sums[item] = _mm256_add_ps(+ sums[item], _mm256_mul_ps(x, values));+ }+ }+ }+ for (int item = 0; item < batch; item++)+ _mm256_storeu_ps(outputs + (size_t)item * rows ++ (size_t)tile * 8, sums[item]);+ }+ free(activation_scales); free(activations); return 0;+ }+#endifmatmul_fp8(outputs, activations, weight->data, weight->scales,batch, (int)columns, (int)rows);free(activation_scales); free(activations); return 0;@@ -9295,7 +10509,7 @@ int coli_fp4_matmul_batch_ref(float *outputs, const ColiTensorView *weight,* shared layer has no resident-cache rows16 layout yet. Migrate and delete it* when that performance API lands upstream. */-#ifdef __AVX512F__+#if defined(__AVX512F__) || defined(__AVX2__)#include <immintrin.h>#elif defined(__aarch64__)#include <arm_neon.h>@@ -9368,6 +10582,46 @@ static __m512 decode_scales_rows16(const unsigned char *scales,_mm_loadu_si128((const __m128i *)scales));return _mm512_i32gather_ps(codes, e8, 4);}+#elif defined(__AVX2__)+typedef struct Avx2Rows16Tables {+ float fp4[16];+ float e8[256];+} Avx2Rows16Tables;++static void avx2_rows16_tables(Avx2Rows16Tables *tables) {+ for (int i = 0; i < 16; i++)+ tables->fp4[i] = coli_e2m1_decode((uint8_t)i);+ for (int i = 0; i < 256; i++)+ tables->e8[i] = coli_e8m0_decode((uint8_t)i);+}++static inline void avx2_decode_rows16(__m256 values[2],+ const unsigned char *data, int high,+ const Avx2Rows16Tables *tables) {+ __m128i bytes = _mm_loadu_si128((const __m128i *)data);+ __m256i lo = _mm256_cvtepu8_epi32(bytes);+ __m256i hi = _mm256_cvtepu8_epi32(_mm_srli_si128(bytes, 8));+ __m256i mask = _mm256_set1_epi32(15);+ if (high) {+ lo = _mm256_srli_epi32(lo, 4);+ hi = _mm256_srli_epi32(hi, 4);+ } else {+ lo = _mm256_and_si256(lo, mask);+ hi = _mm256_and_si256(hi, mask);+ }+ values[0] = _mm256_i32gather_ps(tables->fp4, lo, 4);+ values[1] = _mm256_i32gather_ps(tables->fp4, hi, 4);+}++static inline void avx2_decode_scales(__m256 values[2],+ const unsigned char *codes,+ const Avx2Rows16Tables *tables) {+ __m128i bytes = _mm_loadu_si128((const __m128i *)codes);+ __m256i lo = _mm256_cvtepu8_epi32(bytes);+ __m256i hi = _mm256_cvtepu8_epi32(_mm_srli_si128(bytes, 8));+ values[0] = _mm256_i32gather_ps(tables->e8, lo, 4);+ values[1] = _mm256_i32gather_ps(tables->e8, hi, 4);+}#elif defined(__aarch64__)/* The 16 packed rows map onto four float32x4 accumulators. Per-row work is* (activation * value) * scale added once per column, columns ascending —@@ -9470,6 +10724,46 @@ int coli_fp4_matvec_rows16_v10(float *output,_mm512_storeu_ps(output + (size_t)tile * 16, sum);}free(activation_scales); free(activation); return 0;+#elif defined(__AVX2__)+ if (!output || !input || !packed_valid(weight)) return -1;+ size_t columns = (size_t)weight->columns;+ size_t data_stride = columns / 2, scale_stride = columns / 32;+ float *activation = malloc(columns * sizeof(*activation));+ uint8_t *activation_scales = malloc(columns / 128);+ if (!activation || !activation_scales) {+ free(activation_scales); free(activation); return -1;+ }+ if (coli_fp8_activation_qdq_ref(activation, activation_scales,+ input, columns, 128)) {+ free(activation_scales); free(activation); return -1;+ }+ Avx2Rows16Tables tables; avx2_rows16_tables(&tables);+ const unsigned char *data = weight->data, *scales = weight->scales;+ #pragma omp parallel for schedule(static)+ for (int64_t tile = 0; tile < weight->rows / 16; tile++) {+ __m256 sum[2] = {_mm256_setzero_ps(), _mm256_setzero_ps()};+ for (size_t base = 0; base < columns; base += 32) {+ __m256 scale[2];+ avx2_decode_scales(+ scale,+ scales + ((size_t)tile * scale_stride + base / 32) * 16,+ &tables);+ for (size_t offset = 0; offset < 32; offset++) {+ size_t column = base + offset;+ const unsigned char *codes = data ++ ((size_t)tile * data_stride + column / 2) * 16;+ __m256 values[2];+ avx2_decode_rows16(values, codes, column & 1, &tables);+ __m256 x = _mm256_set1_ps(activation[column]);+ for (int half = 0; half < 2; half++)+ sum[half] = _mm256_add_ps(sum[half], _mm256_mul_ps(+ _mm256_mul_ps(x, values[half]), scale[half]));+ }+ }+ _mm256_storeu_ps(output + (size_t)tile * 16, sum[0]);+ _mm256_storeu_ps(output + (size_t)tile * 16 + 8, sum[1]);+ }+ free(activation_scales); free(activation); return 0;#else /* __aarch64__ */if (!output || !input || !packed_valid(weight)) return -1;size_t columns = (size_t)weight->columns;@@ -9568,6 +10862,58 @@ int coli_fp4_dual_matvec_rows16_v10(float *output_a, float *output_b,_mm512_storeu_ps(output_b + (size_t)tile * 16, sum_b);}free(activation_scales); free(activation); return 0;+#elif defined(__AVX2__)+ if (!output_a || !output_b || !input || !packed_valid(a) ||+ !packed_valid(b) || a->rows != b->rows ||+ a->columns != b->columns) return -1;+ size_t columns = (size_t)a->columns;+ size_t data_stride = columns / 2, scale_stride = columns / 32;+ float *activation = malloc(columns * sizeof(*activation));+ uint8_t *activation_scales = malloc(columns / 128);+ if (!activation || !activation_scales) {+ free(activation_scales); free(activation); return -1;+ }+ if (coli_fp8_activation_qdq_ref(activation, activation_scales,+ input, columns, 128)) {+ free(activation_scales); free(activation); return -1;+ }+ Avx2Rows16Tables tables; avx2_rows16_tables(&tables);+ const unsigned char *data_a = a->data, *data_b = b->data;+ const unsigned char *scales_a = a->scales, *scales_b = b->scales;+ #pragma omp parallel for schedule(static)+ for (int64_t tile = 0; tile < a->rows / 16; tile++) {+ __m256 sum_a[2] = {_mm256_setzero_ps(), _mm256_setzero_ps()};+ __m256 sum_b[2] = {_mm256_setzero_ps(), _mm256_setzero_ps()};+ for (size_t base = 0; base < columns; base += 32) {+ size_t scale_offset =+ ((size_t)tile * scale_stride + base / 32) * 16;+ __m256 scale_a[2], scale_b[2];+ avx2_decode_scales(scale_a, scales_a + scale_offset, &tables);+ avx2_decode_scales(scale_b, scales_b + scale_offset, &tables);+ for (size_t offset = 0; offset < 32; offset++) {+ size_t column = base + offset;+ size_t packed_offset =+ ((size_t)tile * data_stride + column / 2) * 16;+ __m256 values_a[2], values_b[2];+ avx2_decode_rows16(values_a, data_a + packed_offset,+ column & 1, &tables);+ avx2_decode_rows16(values_b, data_b + packed_offset,+ column & 1, &tables);+ __m256 x = _mm256_set1_ps(activation[column]);+ for (int half = 0; half < 2; half++) {+ sum_a[half] = _mm256_add_ps(sum_a[half], _mm256_mul_ps(+ _mm256_mul_ps(x, values_a[half]), scale_a[half]));+ sum_b[half] = _mm256_add_ps(sum_b[half], _mm256_mul_ps(+ _mm256_mul_ps(x, values_b[half]), scale_b[half]));+ }+ }+ }+ _mm256_storeu_ps(output_a + (size_t)tile * 16, sum_a[0]);+ _mm256_storeu_ps(output_a + (size_t)tile * 16 + 8, sum_a[1]);+ _mm256_storeu_ps(output_b + (size_t)tile * 16, sum_b[0]);+ _mm256_storeu_ps(output_b + (size_t)tile * 16 + 8, sum_b[1]);+ }+ free(activation_scales); free(activation); return 0;#else /* __aarch64__ */if (!output_a || !output_b || !input || !packed_valid(a) ||!packed_valid(b) || a->rows != b->rows ||diff --git a/c/deepseek_v4.h b/c/deepseek_v4.hindex 107a0653..f38d8be3 100644--- a/c/deepseek_v4.h+++ b/c/deepseek_v4.h@@ -31,6 +31,9 @@ typedef struct {int moe_intermediate_size;int num_hash_layers;int num_nextn_predict_layers;+ int dspark_block_size;+ int dspark_noise_token_id;+ int dspark_markov_rank;int hc_mult;int hc_sinkhorn_iters;int vocab_size;@@ -81,7 +84,7 @@ typedef struct {int context_tokens; /* 0 => 4096 */int pin_slots_per_layer; /* -1 => auto */uint64_t repin_interval; /* 0 => auto */- int no_dspark; /* compatibility no-op in target-only engine */+ int no_dspark; /* disable speculative draft/verification */} ColiV4EngineOpenOptions;typedef struct {@@ -119,7 +122,7 @@ typedef struct {typedef struct {int max_new_tokens; /* required; clamped by session cap */int stop_at_sentence;- int no_dspark; /* compatibility no-op in target-only engine */+ int no_dspark; /* disable speculative draft/verification */} ColiV4SessionGenerateOptions;typedef struct {@@ -128,6 +131,8 @@ typedef struct {int eos_stopped;double time_to_first_token_sec;double decode_sec;+ uint64_t speculative_drafted;+ uint64_t speculative_accepted;} ColiV4SessionGenerateStats;/* Return non-zero to stop generation. */diff --git a/c/deepseek_v4_dspark.inc b/c/deepseek_v4_dspark.incnew file mode 100644index 00000000..628db053--- /dev/null+++ b/c/deepseek_v4_dspark.inc@@ -0,0 +1,1208 @@+/* ================= Colibri DSpark-SSD: complete native drafter =============+ *+ * This is an original C implementation of the architecture described by the+ * DeepSeek-V4 checkpoint's own inference/model.py. It intentionally does not+ * depend on another inference runtime. The invariants that matter here are:+ *+ * - all three mtp.* transformer stages are evaluated;+ * - the five draft positions attend bidirectionally inside their block;+ * - mtp.2's HC head, Markov logit bias and confidence head are used;+ * - routed MTP experts live in a separate bounded LRU, so drafting cannot+ * evict the target model's learned expert cache;+ * - candidates are never committed without exact target verification.+ *+ * DSpark is opt-in while the real-checkpoint A/B gate is being calibrated:+ * V4_DRAFT=5 V4_MTP=1 V4_MTP_DRAFT=3. V4_MTP_GB is a TOTAL cache budget+ * across the three stages, and V4_MTP_MISS caps cold expert records loaded by+ * one draft round. Prompt lookup may still use all five draft slots; the+ * trained MTP path defaults to three to avoid rejected-suffix replay.+ * ========================================================================== */++#define V4_DSPARK_STAGES 3+#define V4_DSPARK_BLOCK 5+#define V4_DSPARK_WIN 128+#define V4_DSPARK_RANK 256++typedef struct {+ int id;+ uint8_t *slab;+ uint64_t used;+} V4DSparkSlot;++typedef struct {+ int ready;+ int stage;++ /* FP8 weights stay E4M3. Expand the tiny UE8M0 sidecars once at load:+ * the current native kernel deliberately has one branch-free F32-scale+ * hot path (the target model uses the same representation). */+ uint8_t *wq_a_w, *wq_b_w, *wkv_w, *wo_a_w, *wo_b_w;+ uint8_t *sh1_w, *sh2_w, *sh3_w;+ float *wq_a_s, *wq_b_s, *wkv_s, *wo_a_s, *wo_b_s;+ float *sh1_s, *sh2_s, *sh3_s;++ ColiFloatTensor q_norm, kv_norm, attn_norm, ffn_norm, attn_sink;+ ColiFloatTensor hc_attn_fn, hc_attn_base, hc_attn_scale;+ ColiFloatTensor hc_ffn_fn, hc_ffn_base, hc_ffn_scale;+ ColiFloatTensor gate_w, gate_b;++ /* Context injection KV is independent for every MTP stage. */+ float *kv; /* [window][head_dim] */+ int64_t kv_abs[V4_DSPARK_WIN];++ /* FP4 expert cache. One slab is scales+payload for w1,w2,w3. */+ V4DSparkSlot *slots;+ int nslots, cap_slots;+ int64_t slab_bytes;+ uint64_t clock;+ const ColiSafetensorsTensor *ew[256][3], *es[256][3];+ int64_t hits, misses, aborted;+} V4DSparkStage;++static V4DSparkStage g_v4ds[V4_DSPARK_STAGES];++static struct {+ int ready;+ uint8_t *main_proj_w;+ float *main_proj_s;+ int64_t main_proj_rows, main_proj_cols;+ ColiFloatTensor main_norm;++ ColiFloatTensor head_fn, head_base, head_scale, norm;+ ColiFloatTensor confidence_proj;+ uint16_t *markov_w1, *markov_w2; /* [vocab][256], BF16 */+ size_t markov_w1_bytes, markov_w2_bytes;++ float *main_x; /* projected context ring */+ int64_t main_x_abs[V4_DSPARK_WIN];++ uint64_t rounds, drafted, accepted;+ double accept_ewma[V4_DSPARK_BLOCK];+} g_v4ds_core;++static void v4_ds_bf16(float *values, size_t count) {+ coli_bf16_round_array(values, count);+}++static void v4_ds_rmsnorm(float *out, const float *in, const float *weight,+ int n, float eps) {+ float square = 0.0f;+ for (int i = 0; i < n; i++) square += in[i] * in[i];+ float scale = 1.0f / sqrtf(square / n + eps);+ for (int i = 0; i < n; i++) out[i] = in[i] * scale * weight[i];+ v4_ds_bf16(out, (size_t)n);+}++static int v4_ds_read_raw(const ColiSafetensorsIndex *index, const char *name,+ int dtype, void **output, size_t *bytes) {+ const ColiSafetensorsTensor *tensor = coli_st_find(index, name);+ if (!tensor || tensor->dtype != dtype || !tensor->nbytes) return -1;+ void *data = malloc((size_t)tensor->nbytes);+ if (!data || coli_st_read_tensor(index, tensor, data)) {+ free(data);+ return -1;+ }+ *output = data;+ if (bytes) *bytes = (size_t)tensor->nbytes;+ return 0;+}++/* DSpark's resident FP8 matrices use the same column-major rows8 tile consumed+ * by the AVX2 target kernel. Packing is local to this translation unit because+ * the split build keeps the target layer loader in a different object. */+static int v4_ds_pack_rows8(uint8_t *data, int64_t rows, int64_t columns) {+#ifndef __AVX2__+ (void)data; (void)rows; (void)columns;+ return 0;+#else+ if (!data || rows < 8 || rows % 8 || columns < 128 || columns % 128)+ return -1;+ size_t tile_bytes = (size_t)8 * (size_t)columns;+ uint8_t *scratch = malloc(tile_bytes);+ if (!scratch) return -1;+ for (int64_t tile = 0; tile < rows / 8; tile++) {+ uint8_t *target = data + (size_t)tile * tile_bytes;+ memcpy(scratch, target, tile_bytes);+ for (int64_t column = 0; column < columns; column++)+ for (int lane = 0; lane < 8; lane++)+ target[(size_t)column * 8 + lane] =+ scratch[(size_t)lane * columns + column];+ }+ free(scratch);+ return 1;+#endif+}++static int v4_ds_load_fp8(const ColiSafetensorsIndex *index,+ const char *weight_name, const char *scale_name,+ uint8_t **weight, float **scale) {+ const ColiSafetensorsTensor *weight_tensor = coli_st_find(index, weight_name);+ const ColiSafetensorsTensor *scale_tensor = coli_st_find(index, scale_name);+ size_t wb = 0;+ if (!weight_tensor || weight_tensor->dtype != COLI_ST_F8_E4M3 ||+ weight_tensor->rank != 2 ||+ weight_tensor->nbytes != weight_tensor->shape[0] *+ weight_tensor->shape[1] ||+ !scale_tensor || scale_tensor->rank != 2 ||+ scale_tensor->dtype != COLI_ST_F8_E8M0 ||+ scale_tensor->shape[0] != (weight_tensor->shape[0] + 127) / 128 ||+ scale_tensor->shape[1] != (weight_tensor->shape[1] + 127) / 128 ||+ scale_tensor->numel < 1)+ return -1;+ if (v4_ds_read_raw(index, weight_name, COLI_ST_F8_E4M3,+ (void **)weight, &wb))+ return -1;+ *scale = malloc((size_t)scale_tensor->numel * sizeof(**scale));+ if (!*scale ||+ st_read_scale_f32((ColiSafetensorsIndex *)index, scale_name, *scale,+ scale_tensor->numel, 0) != scale_tensor->numel) {+ free(*weight);+ free(*scale);+ *weight = NULL;+ *scale = NULL;+ return -1;+ }+ if (v4_ds_pack_rows8(*weight, weight_tensor->shape[0],+ weight_tensor->shape[1]) < 0) {+ free(*weight);+ free(*scale);+ *weight = NULL;+ *scale = NULL;+ return -1;+ }+ return wb ? 0 : -1;+}++static int v4_ds_name(char *out, size_t size, int stage, const char *suffix) {+ int n = snprintf(out, size, "mtp.%d.%s", stage, suffix);+ return n < 0 || (size_t)n >= size ? -1 : 0;+}++static int v4_ds_load_fp8_suffix(const ColiSafetensorsIndex *index, int stage,+ const char *stem,+ uint8_t **weight, float **scale) {+ char wn[160], sn[160];+ if (snprintf(wn, sizeof(wn), "mtp.%d.%s.weight", stage, stem) >= (int)sizeof(wn) ||+ snprintf(sn, sizeof(sn), "mtp.%d.%s.scale", stage, stem) >= (int)sizeof(sn))+ return -1;+ return v4_ds_load_fp8(index, wn, sn, weight, scale);+}++static int v4_ds_load_f32_suffix(ColiFloatTensor *output,+ const ColiSafetensorsIndex *index, int stage,+ const char *suffix) {+ char name[160], error[256];+ if (v4_ds_name(name, sizeof(name), stage, suffix)) return -1;+ return coli_tensor_load_f32(output, index, name, error, sizeof(error));+}++static void v4_ds_fp8_view(ColiTensorView *view,+ const uint8_t *weight, const float *scale,+ int64_t rows, int64_t columns) {+ memset(view, 0, sizeof(*view));+ view->format = COLI_TENSOR_FP8_E4M3_BLOCK;+ view->scale_format = COLI_SCALE_F32;+ view->data = weight;+ view->scales = scale;+ view->data_bytes = (size_t)(rows * columns);+ view->scale_bytes = (size_t)(((rows + 127) / 128) *+ ((columns + 127) / 128)) * sizeof(float);+ view->rows = rows;+ view->columns = columns;+#ifdef __AVX2__+ view->block_rows = 8;+#else+ view->block_rows = 128;+#endif+ view->block_columns = 128;+}++/* Official FP8 linears quantize activations to E4M3/UE8M0 first. Reusing the+ * engine's native reference kernel is both more faithful and considerably+ * faster than the scalar, unquantized prototype. */+static int v4_ds_mm(float *out, const uint8_t *weight, const float *scale,+ int64_t rows, int64_t columns, const float *input) {+ ColiTensorView view;+ v4_ds_fp8_view(&view, weight, scale, rows, columns);+ if (coli_fp8_matvec_ref(out, &view, input)) return -1;+ v4_ds_bf16(out, (size_t)rows);+ return 0;+}++static int v4_ds_core_load(const ColiSafetensorsIndex *index,+ const ColiDeepSeekV4Config *config) {+ if (g_v4ds_core.ready) return g_v4ds_core.ready > 0;+ g_v4ds_core.ready = -1;+ if (config->hc_mult < 1 || config->hc_mult > 16 ||+ config->n_routed_experts < 1 || config->n_routed_experts > 256 ||+ config->num_experts_per_tok < 1 ||+ config->num_experts_per_tok > 16)+ return 0;++ const ColiSafetensorsTensor *mp =+ coli_st_find(index, "mtp.0.main_proj.weight");+ const ColiSafetensorsTensor *ms =+ coli_st_find(index, "mtp.0.main_proj.scale");+ if (!mp || !ms || mp->dtype != COLI_ST_F8_E4M3 ||+ ms->dtype != COLI_ST_F8_E8M0 || mp->rank != 2)+ return 0;+ g_v4ds_core.main_proj_rows = config->hidden_size;+ g_v4ds_core.main_proj_cols =+ (int64_t)config->hidden_size * V4_MAINH_TARGETS;+ if (mp->shape[0] != g_v4ds_core.main_proj_rows ||+ mp->shape[1] != g_v4ds_core.main_proj_cols ||+ v4_ds_load_fp8(index, "mtp.0.main_proj.weight",+ "mtp.0.main_proj.scale",+ &g_v4ds_core.main_proj_w,+ &g_v4ds_core.main_proj_s) ||+ v4_ds_load_f32_suffix(&g_v4ds_core.main_norm, index, 0,+ "main_norm.weight") ||+ v4_ds_load_f32_suffix(&g_v4ds_core.head_fn, index, 2,+ "hc_head_fn") ||+ v4_ds_load_f32_suffix(&g_v4ds_core.head_base, index, 2,+ "hc_head_base") ||+ v4_ds_load_f32_suffix(&g_v4ds_core.head_scale, index, 2,+ "hc_head_scale") ||+ v4_ds_load_f32_suffix(&g_v4ds_core.norm, index, 2,+ "norm.weight") ||+ v4_ds_load_f32_suffix(&g_v4ds_core.confidence_proj, index, 2,+ "confidence_head.proj.weight") ||+ v4_ds_read_raw(index, "mtp.2.markov_head.markov_w1.weight",+ COLI_ST_BF16, (void **)&g_v4ds_core.markov_w1,+ &g_v4ds_core.markov_w1_bytes) ||+ v4_ds_read_raw(index, "mtp.2.markov_head.markov_w2.weight",+ COLI_ST_BF16, (void **)&g_v4ds_core.markov_w2,+ &g_v4ds_core.markov_w2_bytes))+ return 0;++ size_t markov_bytes = (size_t)config->vocab_size * V4_DSPARK_RANK *+ sizeof(uint16_t);+ if (g_v4ds_core.markov_w1_bytes != markov_bytes ||+ g_v4ds_core.markov_w2_bytes != markov_bytes)+ return 0;++ g_v4ds_core.main_x = calloc((size_t)V4_DSPARK_WIN * config->hidden_size,+ sizeof(float));+ if (!g_v4ds_core.main_x) return 0;+ for (int i = 0; i < V4_DSPARK_WIN; i++)+ g_v4ds_core.main_x_abs[i] = -1;+ for (int i = 0; i < V4_DSPARK_BLOCK; i++)+ g_v4ds_core.accept_ewma[i] = 0.5;++ g_v4ds_core.ready = 1;+ fprintf(stderr,+ "[MTP] DSpark core ready: stages=%d block=%d main_proj=%lldx%lld "+ "markov_rank=%d\n",+ V4_DSPARK_STAGES, V4_DSPARK_BLOCK,+ (long long)g_v4ds_core.main_proj_rows,+ (long long)g_v4ds_core.main_proj_cols, V4_DSPARK_RANK);+ return 1;+}++static int v4_ds_stage_load(const ColiSafetensorsIndex *index,+ const ColiDeepSeekV4Config *config, int stage_id) {+ V4DSparkStage *stage = &g_v4ds[stage_id];+ if (stage->ready) return stage->ready > 0;+ stage->ready = -1;+ stage->stage = stage_id;++#define V4_DS_LOAD_FP8(field, stem) \+ do { if (v4_ds_load_fp8_suffix(index, stage_id, stem, \+ &stage->field##_w, &stage->field##_s)) \+ return 0; } while (0)+ V4_DS_LOAD_FP8(wq_a, "attn.wq_a");+ V4_DS_LOAD_FP8(wq_b, "attn.wq_b");+ V4_DS_LOAD_FP8(wkv, "attn.wkv");+ V4_DS_LOAD_FP8(wo_a, "attn.wo_a");+ V4_DS_LOAD_FP8(wo_b, "attn.wo_b");+ V4_DS_LOAD_FP8(sh1, "ffn.shared_experts.w1");+ V4_DS_LOAD_FP8(sh2, "ffn.shared_experts.w2");+ V4_DS_LOAD_FP8(sh3, "ffn.shared_experts.w3");+#undef V4_DS_LOAD_FP8++#define V4_DS_LOAD_F32(field, suffix) \+ do { if (v4_ds_load_f32_suffix(&stage->field, index, stage_id, suffix)) \+ return 0; } while (0)+ V4_DS_LOAD_F32(q_norm, "attn.q_norm.weight");+ V4_DS_LOAD_F32(kv_norm, "attn.kv_norm.weight");+ V4_DS_LOAD_F32(attn_norm, "attn_norm.weight");+ V4_DS_LOAD_F32(ffn_norm, "ffn_norm.weight");+ V4_DS_LOAD_F32(attn_sink, "attn.attn_sink");+ V4_DS_LOAD_F32(hc_attn_fn, "hc_attn_fn");+ V4_DS_LOAD_F32(hc_attn_base, "hc_attn_base");+ V4_DS_LOAD_F32(hc_attn_scale,"hc_attn_scale");+ V4_DS_LOAD_F32(hc_ffn_fn, "hc_ffn_fn");+ V4_DS_LOAD_F32(hc_ffn_base, "hc_ffn_base");+ V4_DS_LOAD_F32(hc_ffn_scale, "hc_ffn_scale");+ V4_DS_LOAD_F32(gate_w, "ffn.gate.weight");+ V4_DS_LOAD_F32(gate_b, "ffn.gate.bias");+#undef V4_DS_LOAD_F32++ static const char *matrices[3] = {"w1", "w2", "w3"};+ stage->slab_bytes = 0;+ for (int expert = 0; expert < config->n_routed_experts; expert++) {+ for (int matrix = 0; matrix < 3; matrix++) {+ char name[192];+ snprintf(name, sizeof(name), "mtp.%d.ffn.experts.%d.%s.weight",+ stage_id, expert, matrices[matrix]);+ stage->ew[expert][matrix] = coli_st_find(index, name);+ snprintf(name, sizeof(name), "mtp.%d.ffn.experts.%d.%s.scale",+ stage_id, expert, matrices[matrix]);+ stage->es[expert][matrix] = coli_st_find(index, name);+ if (!stage->ew[expert][matrix] || !stage->es[expert][matrix])+ return 0;+ if (!expert)+ stage->slab_bytes += stage->ew[expert][matrix]->nbytes ++ stage->es[expert][matrix]->nbytes;+ }+ }++ /* V4_MTP_GB is intentionally total, not per stage. */+ double total_gb = coli_v4_dspark_cache_gb();+ double stage_bytes = total_gb * 1e9 / V4_DSPARK_STAGES;+ stage->cap_slots = (int)(stage_bytes / (double)stage->slab_bytes);+ if (stage->cap_slots < 4) stage->cap_slots = 4;+ if (stage->cap_slots > config->n_routed_experts)+ stage->cap_slots = config->n_routed_experts;+ stage->slots = calloc((size_t)stage->cap_slots, sizeof(*stage->slots));+ stage->kv = calloc((size_t)V4_DSPARK_WIN * config->head_dim, sizeof(float));+ if (!stage->slots || !stage->kv) return 0;+ for (int i = 0; i < stage->cap_slots; i++) stage->slots[i].id = -1;+ for (int i = 0; i < V4_DSPARK_WIN; i++) stage->kv_abs[i] = -1;++ stage->ready = 1;+ fprintf(stderr,+ "[MTP] stage %d ready: expert_cache=%d/%d %.2fGB\n",+ stage_id, stage->cap_slots, config->n_routed_experts,+ stage->cap_slots * (double)stage->slab_bytes / 1e9);+ return 1;+}++static int v4_ds_load_all(const ColiSafetensorsIndex *index,+ const ColiDeepSeekV4Config *config) {+ if (!v4_ds_core_load(index, config)) return 0;+ for (int stage = 0; stage < V4_DSPARK_STAGES; stage++)+ if (!v4_ds_stage_load(index, config, stage)) return 0;+ return 1;+}++static int v4_ds_expert(const ColiSafetensorsIndex *index,+ V4DSparkStage *stage, int expert,+ ColiExpertView *view, int *miss_budget) {+ int slot = -1;+ int lru = 0;+ for (int i = 0; i < stage->nslots; i++) {+ if (stage->slots[i].id == expert) {+ slot = i;+ stage->hits++;+ break;+ }+ if (stage->slots[i].used < stage->slots[lru].used) lru = i;+ }+ if (slot < 0) {+ if (*miss_budget <= 0) {+ stage->aborted++;+ return 0;+ }+ (*miss_budget)--;+ stage->misses++;+ slot = stage->nslots < stage->cap_slots ? stage->nslots++ : lru;+ if (!stage->slots[slot].slab) {+ stage->slots[slot].slab = malloc((size_t)stage->slab_bytes);+ if (!stage->slots[slot].slab) return 0;+ }+ stage->slots[slot].id = -1;+ uint8_t *cursor = stage->slots[slot].slab;+ for (int matrix = 0; matrix < 3; matrix++) {+ const ColiSafetensorsTensor *scale = stage->es[expert][matrix];+ const ColiSafetensorsTensor *weight = stage->ew[expert][matrix];+ if (coli_st_read_tensor(index, scale, cursor))+ return 0;+ cursor += scale->nbytes;+ int shard = coli_st_tensor_shard(index, weight);+ if (shard < 0 || weight->off < 0 ||+ coli_st_read_at_streaming(+ index, shard, (uint64_t)weight->off,+ (size_t)weight->nbytes, cursor))+ return 0;+ cursor += weight->nbytes;+ }+ stage->slots[slot].id = expert;+ }+ stage->slots[slot].used = ++stage->clock;++ uint8_t *cursor = stage->slots[slot].slab;+ ColiTensorView *views[3] = {&view->gate, &view->down, &view->up};+ for (int matrix = 0; matrix < 3; matrix++) {+ const ColiSafetensorsTensor *scale = stage->es[expert][matrix];+ const ColiSafetensorsTensor *weight = stage->ew[expert][matrix];+ ColiTensorView *tensor = views[matrix];+ memset(tensor, 0, sizeof(*tensor));+ tensor->format = COLI_TENSOR_FP4_NATIVE_BLOCK;+ tensor->scale_format = COLI_SCALE_UE8M0;+ tensor->scales = cursor;+ cursor += scale->nbytes;+ tensor->data = cursor;+ cursor += weight->nbytes;+ tensor->scale_bytes = (size_t)scale->nbytes;+ tensor->data_bytes = (size_t)weight->nbytes;+ tensor->rows = weight->shape[0];+ tensor->columns = weight->shape[1] * 2;+ tensor->block_rows = 1;+ tensor->block_columns = 32;+ }+ view->lease = NULL;+ return 1;+}++/* Hyper-Connections: exact ordering of the checkpoint's split-Sinkhorn. */+static void v4_ds_sinkhorn(const float *mix, const float *scale,+ const float *base, int hc, int iterations,+ float eps, float *pre, float *post, float *comb) {+ for (int j = 0; j < hc; j++) {+ float z = mix[j] * scale[0] + base[j];+ pre[j] = 1.0f / (1.0f + expf(-z)) + eps;+ z = mix[j + hc] * scale[1] + base[j + hc];+ post[j] = 2.0f / (1.0f + expf(-z));+ }+ for (int j = 0; j < hc; j++) {+ float maximum = -FLT_MAX;+ for (int k = 0; k < hc; k++) {+ float value = mix[2 * hc + j * hc + k] * scale[2] ++ base[2 * hc + j * hc + k];+ comb[j * hc + k] = value;+ if (value > maximum) maximum = value;+ }+ float sum = 0.0f;+ for (int k = 0; k < hc; k++) {+ comb[j * hc + k] = expf(comb[j * hc + k] - maximum);+ sum += comb[j * hc + k];+ }+ for (int k = 0; k < hc; k++)+ comb[j * hc + k] = comb[j * hc + k] / sum + eps;+ }++ /* Initial column normalisation, then (row,column) x (iters-1). */+ for (int k = 0; k < hc; k++) {+ float sum = 0.0f;+ for (int j = 0; j < hc; j++) sum += comb[j * hc + k];+ for (int j = 0; j < hc; j++) comb[j * hc + k] /= sum + eps;+ }+ for (int iteration = 0; iteration < iterations - 1; iteration++) {+ for (int j = 0; j < hc; j++) {+ float sum = 0.0f;+ for (int k = 0; k < hc; k++) sum += comb[j * hc + k];+ for (int k = 0; k < hc; k++) comb[j * hc + k] /= sum + eps;+ }+ for (int k = 0; k < hc; k++) {+ float sum = 0.0f;+ for (int j = 0; j < hc; j++) sum += comb[j * hc + k];+ for (int j = 0; j < hc; j++) comb[j * hc + k] /= sum + eps;+ }+ }+}++static void v4_ds_hc_pre(float *out, const float *state,+ const ColiFloatTensor *function,+ const ColiFloatTensor *base,+ const ColiFloatTensor *scale,+ const ColiDeepSeekV4Config *config,+ float *post, float *comb) {+ int hc = config->hc_mult;+ int hidden = config->hidden_size;+ int flat = hc * hidden;+ int mixes = (2 + hc) * hc;+ float square = 0.0f;+ for (int i = 0; i < flat; i++) square += state[i] * state[i];+ float inverse_rms =+ 1.0f / sqrtf(square / flat + config->rms_norm_eps);+ float mixed[64], pre[16];+ for (int row = 0; row < mixes; row++) {+ float sum = 0.0f;+ const float *weight = function->data + (size_t)row * flat;+ for (int i = 0; i < flat; i++) sum += weight[i] * state[i];+ mixed[row] = sum * inverse_rms;+ }+ v4_ds_sinkhorn(mixed, scale->data, base->data, hc,+ config->hc_sinkhorn_iters, config->hc_eps,+ pre, post, comb);+ for (int i = 0; i < hidden; i++) {+ float sum = 0.0f;+ for (int copy = 0; copy < hc; copy++)+ sum += pre[copy] * state[(size_t)copy * hidden + i];+ out[i] = coli_bf16_round(sum);+ }+}++static void v4_ds_hc_post(float *out, const float *branch,+ const float *residual, const float *post,+ const float *comb,+ const ColiDeepSeekV4Config *config) {+ int hc = config->hc_mult;+ int hidden = config->hidden_size;+ for (int dst = 0; dst < hc; dst++) {+ for (int i = 0; i < hidden; i++) {+ float sum = post[dst] * branch[i];+ for (int src = 0; src < hc; src++)+ sum += comb[src * hc + dst] *+ residual[(size_t)src * hidden + i];+ out[(size_t)dst * hidden + i] = coli_bf16_round(sum);+ }+ }+}++static int v4_ds_main_x(float *out, int64_t position,+ const ColiDeepSeekV4Config *config) {+ if (position < 0 || !g_v4ds_core.main_x) return 0;+ int slot = (int)(position % V4_DSPARK_WIN);+ float *cached = g_v4ds_core.main_x ++ (size_t)slot * config->hidden_size;+ if (g_v4ds_core.main_x_abs[slot] == position) {+ memcpy(out, cached, (size_t)config->hidden_size * sizeof(float));+ return 1;+ }+ const float *main_hidden = v4_mainh_get(position);+ if (!main_hidden) return 0;+ if (v4_ds_mm(cached, g_v4ds_core.main_proj_w,+ g_v4ds_core.main_proj_s,+ g_v4ds_core.main_proj_rows,+ g_v4ds_core.main_proj_cols, main_hidden))+ return 0;+ v4_ds_rmsnorm(cached, cached, g_v4ds_core.main_norm.data,+ config->hidden_size, config->rms_norm_eps);+ g_v4ds_core.main_x_abs[slot] = position;+ memcpy(out, cached, (size_t)config->hidden_size * sizeof(float));+ return 1;+}++/* Speculative target batches can write hidden taps for a rejected suffix.+ * Purge them before refeeding the accepted prefix; otherwise future drafts+ * would condition on a history that never happened. */+static void v4_ds_invalidate_from(int64_t position) {+ for (int slot = 0; slot < V4_MAINH_RING; slot++) {+ if (g_mainh_abs[slot] >= position) g_mainh_abs[slot] = -1;+ }+ for (int slot = 0; slot < V4_DSPARK_WIN; slot++) {+ if (g_v4ds_core.main_x_abs[slot] >= position)+ g_v4ds_core.main_x_abs[slot] = -1;+ for (int stage = 0; stage < V4_DSPARK_STAGES; stage++)+ if (g_v4ds[stage].kv_abs[slot] >= position)+ g_v4ds[stage].kv_abs[slot] = -1;+ }+}++static void v4_ds_reset_history(void) {+ v4_ds_invalidate_from(0);+}++/* DSpark QAT keeps the RoPE tail in BF16 and simulates FP8 only for the+ * non-positional 448 dimensions, in blocks of 64. */+static int v4_ds_kv_qdq(float *kv, int head_dim, int rope_dim) {+ int nope = head_dim - rope_dim;+ uint8_t scales[16];+ if (nope < 1 || nope > 1024 ||+ coli_fp8_activation_qdq_ref(kv, scales, kv, (size_t)nope, 64))+ return -1;+ v4_ds_bf16(kv, (size_t)head_dim);+ return 0;+}++static int v4_ds_backfill_context(const ColiDeepSeekV4Config *config,+ int64_t position,+ const float *cosines, const float *sines) {+ int hidden = config->hidden_size;+ int head_dim = config->head_dim;+ int rope_dim = config->qk_rope_head_dim;+ int pairs = rope_dim / 2;+ float *main_x = malloc((size_t)hidden * sizeof(float));+ float *kv = malloc((size_t)head_dim * sizeof(float));+ if (!main_x || !kv) {+ free(kv); free(main_x); return -1;+ }+ int64_t first = position > V4_DSPARK_WIN+ ? position - V4_DSPARK_WIN : 0;+ for (int64_t absolute = first; absolute < position; absolute++) {+ int slot = (int)(absolute % V4_DSPARK_WIN);+ int needed = 0;+ for (int stage = 0; stage < V4_DSPARK_STAGES; stage++)+ if (g_v4ds[stage].kv_abs[slot] != absolute) needed = 1;+ if (!needed || !v4_ds_main_x(main_x, absolute, config)) continue;+ for (int stage_id = 0; stage_id < V4_DSPARK_STAGES; stage_id++) {+ V4DSparkStage *stage = &g_v4ds[stage_id];+ if (stage->kv_abs[slot] == absolute) continue;+ if (v4_ds_mm(kv, stage->wkv_w, stage->wkv_s,+ head_dim, hidden, main_x)) {+ free(kv); free(main_x); return -1;+ }+ v4_ds_rmsnorm(kv, kv, stage->kv_norm.data, head_dim,+ config->rms_norm_eps);+ coli_v4_rope_apply(kv + head_dim - rope_dim, 1, rope_dim,+ cosines + (size_t)absolute * pairs,+ sines + (size_t)absolute * pairs, 0);+ v4_ds_bf16(kv + head_dim - rope_dim, (size_t)rope_dim);+ if (v4_ds_kv_qdq(kv, head_dim, rope_dim)) {+ free(kv); free(main_x); return -1;+ }+ memcpy(stage->kv + (size_t)slot * head_dim, kv,+ (size_t)head_dim * sizeof(float));+ stage->kv_abs[slot] = absolute;+ }+ }+ free(kv);+ free(main_x);+ return 0;+}++static int v4_ds_attention(float *out, V4DSparkStage *stage,+ const float *input, int64_t query_position,+ const float *block_kv, int block_count,+ const ColiDeepSeekV4Config *config,+ const float *cosines, const float *sines) {+ int hidden = config->hidden_size;+ int head_dim = config->head_dim;+ int rope_dim = config->qk_rope_head_dim;+ int heads = config->num_attention_heads;+ int groups = config->o_groups;+ int q_rank = config->q_lora_rank;+ int o_rank = config->o_lora_rank;+ int pairs = rope_dim / 2;++ float *qa = malloc((size_t)q_rank * sizeof(float));+ float *query = malloc((size_t)heads * head_dim * sizeof(float));+ float *attended = calloc((size_t)heads * head_dim, sizeof(float));+ float *oa = malloc((size_t)groups * o_rank * sizeof(float));+ if (!qa || !query || !attended || !oa) {+ free(oa); free(attended); free(query); free(qa); return -1;+ }+ if (v4_ds_mm(qa, stage->wq_a_w, stage->wq_a_s,+ q_rank, hidden, input))+ goto fail;+ v4_ds_rmsnorm(qa, qa, stage->q_norm.data, q_rank,+ config->rms_norm_eps);+ if (v4_ds_mm(query, stage->wq_b_w, stage->wq_b_s,+ (int64_t)heads * head_dim, q_rank, qa))+ goto fail;+ for (int head = 0; head < heads; head++) {+ float *row = query + (size_t)head * head_dim;+ float square = 0.0f;+ for (int d = 0; d < head_dim; d++) square += row[d] * row[d];+ float inverse = 1.0f /+ sqrtf(square / head_dim + config->rms_norm_eps);+ for (int d = 0; d < head_dim; d++)+ row[d] = coli_bf16_round(row[d] * inverse);+ coli_v4_rope_apply(row + head_dim - rope_dim, 1, rope_dim,+ cosines + (size_t)query_position * pairs,+ sines + (size_t)query_position * pairs, 0);+ v4_ds_bf16(row + head_dim - rope_dim, (size_t)rope_dim);+ }++ float softmax_scale = 1.0f / sqrtf((float)head_dim);+ for (int head = 0; head < heads; head++) {+ const float *q = query + (size_t)head * head_dim;+ const float *keys[V4_DSPARK_WIN + V4_DSPARK_BLOCK];+ float scores[V4_DSPARK_WIN + V4_DSPARK_BLOCK];+ int count = 0;+ float maximum = stage->attn_sink.data[head];+ for (int slot = 0; slot < V4_DSPARK_WIN; slot++) {+ if (stage->kv_abs[slot] < 0 ||+ stage->kv_abs[slot] >= query_position)+ continue;+ keys[count] = stage->kv + (size_t)slot * head_dim;+ float dot = 0.0f;+ for (int d = 0; d < head_dim; d++) dot += q[d] * keys[count][d];+ scores[count] = dot * softmax_scale;+ if (scores[count] > maximum) maximum = scores[count];+ count++;+ }+ /* The parallel backbone is deliberately bidirectional in-block. */+ for (int item = 0; item < block_count; item++) {+ keys[count] = block_kv + (size_t)item * head_dim;+ float dot = 0.0f;+ for (int d = 0; d < head_dim; d++) dot += q[d] * keys[count][d];+ scores[count] = dot * softmax_scale;+ if (scores[count] > maximum) maximum = scores[count];+ count++;+ }+ float denominator = expf(stage->attn_sink.data[head] - maximum);+ for (int item = 0; item < count; item++) {+ scores[item] = expf(scores[item] - maximum);+ denominator += scores[item];+ }+ float *result = attended + (size_t)head * head_dim;+ for (int item = 0; item < count; item++) {+ float probability = scores[item] / denominator;+ for (int d = 0; d < head_dim; d++)+ result[d] += probability * keys[item][d];+ }+ v4_ds_bf16(result, (size_t)head_dim);+ coli_v4_rope_apply(result + head_dim - rope_dim, 1, rope_dim,+ cosines + (size_t)query_position * pairs,+ sines + (size_t)query_position * pairs, 1);+ v4_ds_bf16(result + head_dim - rope_dim, (size_t)rope_dim);+ }++ int heads_per_group = heads / groups;+ int group_width = heads_per_group * head_dim;+ int scale_columns = (group_width + 127) / 128;+ int scale_rows = (o_rank + 127) / 128;+ for (int group = 0; group < groups; group++) {+ ColiTensorView view;+ v4_ds_fp8_view(&view,+ stage->wo_a_w + (size_t)group * o_rank * group_width,+ stage->wo_a_s + (size_t)group * scale_rows * scale_columns,+ o_rank, group_width);+ if (coli_fp8_matvec_ref(oa + (size_t)group * o_rank, &view,+ attended + (size_t)group * group_width))+ goto fail;+ v4_ds_bf16(oa + (size_t)group * o_rank, (size_t)o_rank);+ }+ if (v4_ds_mm(out, stage->wo_b_w, stage->wo_b_s,+ hidden, (int64_t)groups * o_rank, oa))+ goto fail;+ free(oa); free(attended); free(query); free(qa);+ return 0;++fail:+ free(oa); free(attended); free(query); free(qa);+ return -1;+}++static int v4_ds_moe(float *out, V4DSparkStage *stage,+ const float *input,+ const ColiSafetensorsIndex *index,+ const ColiDeepSeekV4Config *config,+ int *miss_budget) {+ int hidden = config->hidden_size;+ int intermediate = config->moe_intermediate_size;+ int topk = config->num_experts_per_tok;+ float weights[16];+ int experts[16];+ if (topk > 16 || coli_v4_route(weights, experts, input,+ stage->gate_w.data, stage->gate_b.data, NULL,+ config->n_routed_experts, hidden, topk,+ config->routed_scaling_factor))+ return -1;++ ColiTensorView shared_gate, shared_down, shared_up;+ v4_ds_fp8_view(&shared_gate, stage->sh1_w, stage->sh1_s,+ intermediate, hidden);+ v4_ds_fp8_view(&shared_down, stage->sh2_w, stage->sh2_s,+ hidden, intermediate);+ v4_ds_fp8_view(&shared_up, stage->sh3_w, stage->sh3_s,+ intermediate, hidden);+ if (coli_v4_shared_expert_forward_ref(out, &shared_gate, &shared_down,+ &shared_up, input,+ config->swiglu_limit))+ return -1;++ float *branch = malloc((size_t)hidden * sizeof(float));+ if (!branch) return -1;+ for (int rank = 0; rank < topk; rank++) {+ ColiExpertView expert;+ if (!v4_ds_expert(index, stage, experts[rank], &expert, miss_budget)) {+ free(branch);+ return 1; /* bounded cold-start abort */+ }+ if (coli_v4_expert_forward_ref(branch, &expert, input,+ weights[rank],+ config->swiglu_limit)) {+ free(branch);+ return -1;+ }+ for (int i = 0; i < hidden; i++) out[i] += branch[i];+ }+ v4_ds_bf16(out, (size_t)hidden);+ free(branch);+ return 0;+}++/* One complete MTP transformer stage over all five positions. Attention is+ * split into prepare/all-KV then query phases so every position can see the+ * entire block, exactly as the parallel reference does. */+static int v4_ds_stage_forward(float *state, V4DSparkStage *stage,+ int64_t first_position,+ const ColiSafetensorsIndex *index,+ const ColiDeepSeekV4Config *config,+ const float *cosines, const float *sines,+ int *miss_budget) {+ int hidden = config->hidden_size;+ int hc = config->hc_mult;+ int head_dim = config->head_dim;+ int rope_dim = config->qk_rope_head_dim;+ int pairs = rope_dim / 2;+ size_t flat = (size_t)hc * hidden;++ float *residual = malloc((size_t)V4_DSPARK_BLOCK * flat * sizeof(float));+ float *normed = malloc((size_t)V4_DSPARK_BLOCK * hidden * sizeof(float));+ float *block_kv = malloc((size_t)V4_DSPARK_BLOCK * head_dim * sizeof(float));+ float *posts = malloc((size_t)V4_DSPARK_BLOCK * hc * sizeof(float));+ float *combs = malloc((size_t)V4_DSPARK_BLOCK * hc * hc * sizeof(float));+ float *branch = malloc((size_t)config->num_attention_heads * head_dim *+ sizeof(float));+ if (!residual || !normed || !block_kv || !posts || !combs || !branch) {+ free(branch); free(combs); free(posts); free(block_kv);+ free(normed); free(residual); return -1;+ }++ memcpy(residual, state,+ (size_t)V4_DSPARK_BLOCK * flat * sizeof(float));+ for (int item = 0; item < V4_DSPARK_BLOCK; item++) {+ float *x = state + (size_t)item * flat;+ float *xn = normed + (size_t)item * hidden;+ float *post = posts + (size_t)item * hc;+ float *comb = combs + (size_t)item * hc * hc;+ v4_ds_hc_pre(xn, x, &stage->hc_attn_fn,+ &stage->hc_attn_base, &stage->hc_attn_scale,+ config, post, comb);+ v4_ds_rmsnorm(xn, xn, stage->attn_norm.data, hidden,+ config->rms_norm_eps);+ float *kv = block_kv + (size_t)item * head_dim;+ if (v4_ds_mm(kv, stage->wkv_w, stage->wkv_s,+ head_dim, hidden, xn))+ goto fail;+ v4_ds_rmsnorm(kv, kv, stage->kv_norm.data, head_dim,+ config->rms_norm_eps);+ int64_t absolute = first_position + item;+ coli_v4_rope_apply(kv + head_dim - rope_dim, 1, rope_dim,+ cosines + (size_t)absolute * pairs,+ sines + (size_t)absolute * pairs, 0);+ v4_ds_bf16(kv + head_dim - rope_dim, (size_t)rope_dim);+ if (v4_ds_kv_qdq(kv, head_dim, rope_dim)) goto fail;+ }+ for (int item = 0; item < V4_DSPARK_BLOCK; item++) {+ float *x = state + (size_t)item * flat;+ const float *xn = normed + (size_t)item * hidden;+ if (v4_ds_attention(branch, stage, xn, first_position + item,+ block_kv, V4_DSPARK_BLOCK, config,+ cosines, sines))+ goto fail;+ v4_ds_hc_post(x, branch, residual + (size_t)item * flat,+ posts + (size_t)item * hc,+ combs + (size_t)item * hc * hc, config);+ }++ memcpy(residual, state,+ (size_t)V4_DSPARK_BLOCK * flat * sizeof(float));+ for (int item = 0; item < V4_DSPARK_BLOCK; item++) {+ float *x = state + (size_t)item * flat;+ float *xn = normed + (size_t)item * hidden;+ float *post = posts + (size_t)item * hc;+ float *comb = combs + (size_t)item * hc * hc;+ v4_ds_hc_pre(xn, x, &stage->hc_ffn_fn,+ &stage->hc_ffn_base, &stage->hc_ffn_scale,+ config, post, comb);+ v4_ds_rmsnorm(xn, xn, stage->ffn_norm.data, hidden,+ config->rms_norm_eps);+ int result = v4_ds_moe(branch, stage, xn, index, config, miss_budget);+ if (result < 0) goto fail;+ if (result > 0) {+ free(branch); free(combs); free(posts); free(block_kv);+ free(normed); free(residual); return 1;+ }+ v4_ds_hc_post(x, branch, residual + (size_t)item * flat,+ post, comb, config);+ }++ free(branch); free(combs); free(posts); free(block_kv);+ free(normed); free(residual);+ return 0;++fail:+ free(branch); free(combs); free(posts); free(block_kv);+ free(normed); free(residual);+ return -1;+}++static int v4_ds_final_hidden(float *out, const float *state,+ const ColiDeepSeekV4Config *config) {+ int hidden = config->hidden_size;+ int hc = config->hc_mult;+ int flat = hidden * hc;+ float square = 0.0f;+ for (int i = 0; i < flat; i++) square += state[i] * state[i];+ float inverse = 1.0f /+ sqrtf(square / flat + config->rms_norm_eps);+ float pre[16];+ for (int copy = 0; copy < hc; copy++) {+ float mix = 0.0f;+ const float *weight = g_v4ds_core.head_fn.data ++ (size_t)copy * flat;+ for (int i = 0; i < flat; i++) mix += weight[i] * state[i];+ float z = mix * inverse * g_v4ds_core.head_scale.data[0] ++ g_v4ds_core.head_base.data[copy];+ pre[copy] = 1.0f / (1.0f + expf(-z)) + config->hc_eps;+ }+ for (int i = 0; i < hidden; i++) {+ float sum = 0.0f;+ for (int copy = 0; copy < hc; copy++)+ sum += pre[copy] * state[(size_t)copy * hidden + i];+ out[i] = coli_bf16_round(sum);+ }+ v4_ds_rmsnorm(out, out, g_v4ds_core.norm.data, hidden,+ config->rms_norm_eps);+ return 0;+}++static void v4_ds_markov_embed(float out[V4_DSPARK_RANK], int token) {+ const uint16_t *row = g_v4ds_core.markov_w1 ++ (size_t)token * V4_DSPARK_RANK;+ for (int i = 0; i < V4_DSPARK_RANK; i++) out[i] = coli_bf16_decode(row[i]);+}++/* Compute all backbone logits in one vocabulary sweep so head.weight is read+ * once for the five positions. The small Markov correction remains serial,+ * which is the defining semi-autoregressive part of DSpark. */+static int v4_ds_heads(ColiV4Engine *engine,+ const ColiSafetensorsIndex *index,+ const ColiDeepSeekV4Config *config,+ const float *state, int anchor_token,+ int *draft, float *confidence) {+ const ColiSafetensorsTensor *head = coli_st_find(index, "head.weight");+ if (!head || head->dtype != COLI_ST_BF16) return -1;+ int head_shard = coli_st_tensor_shard(index, head);+ if (head_shard < 0 || head->off < 0) return -1;+ uint64_t head_offset = (uint64_t)head->off;+ int hidden = config->hidden_size;+ int vocab = config->vocab_size;+ int hc = config->hc_mult;+ size_t flat = (size_t)hc * hidden;+ float *hiddens = malloc((size_t)V4_DSPARK_BLOCK * hidden * sizeof(float));+ float *logits = malloc((size_t)V4_DSPARK_BLOCK * vocab * sizeof(float));+ if (!hiddens || !logits) {+ free(logits); free(hiddens); return -1;+ }+ for (int item = 0; item < V4_DSPARK_BLOCK; item++)+ if (v4_ds_final_hidden(hiddens + (size_t)item * hidden,+ state + (size_t)item * flat, config)) {+ free(logits); free(hiddens); return -1;+ }++ const uint16_t *resident = coli_v4_head_cache_data(+ engine, head_shard, head_offset, (size_t)head->nbytes);+ enum { ROWS = 64 };+ uint16_t *raw = resident ? NULL :+ malloc((size_t)ROWS * hidden * sizeof(uint16_t));+ if (!resident && !raw) {+ free(logits); free(hiddens); return -1;+ }+ for (int start = 0; start < vocab; start += ROWS) {+ int rows = vocab - start < ROWS ? vocab - start : ROWS;+ const uint16_t *weights = resident+ ? resident + (size_t)start * hidden : raw;+ if (!resident) {+ if (coli_st_read_at_engine(engine, index, head_shard,+ head_offset + (uint64_t)start * hidden * sizeof(uint16_t),+ (size_t)rows * hidden * sizeof(uint16_t), raw)) {+ free(raw); free(logits); free(hiddens); return -1;+ }+ weights = raw;+ }+ #pragma omp parallel for+ for (int row = 0; row < rows; row++) {+ const uint16_t *weight = weights + (size_t)row * hidden;+ float sums[V4_DSPARK_BLOCK] = {0};+ for (int d = 0; d < hidden; d++) {+ float w = coli_bf16_decode(weight[d]);+ for (int item = 0; item < V4_DSPARK_BLOCK; item++)+ sums[item] += w * hiddens[(size_t)item * hidden + d];+ }+ for (int item = 0; item < V4_DSPARK_BLOCK; item++)+ logits[(size_t)item * vocab + start + row] = sums[item];+ }+ }+ free(raw);++ int previous = anchor_token;+ for (int item = 0; item < V4_DSPARK_BLOCK; item++) {+ float embedding[V4_DSPARK_RANK];+ v4_ds_markov_embed(embedding, previous);+ int winner = -1;+ float maximum = -FLT_MAX;+ #pragma omp parallel+ {+ int local_winner = -1;+ float local_maximum = -FLT_MAX;+ #pragma omp for nowait+ for (int token = 0; token < vocab; token++) {+ const uint16_t *weight = g_v4ds_core.markov_w2 ++ (size_t)token * V4_DSPARK_RANK;+ float score = logits[(size_t)item * vocab + token];+ for (int rank = 0; rank < V4_DSPARK_RANK; rank++)+ score += coli_bf16_decode(weight[rank]) * embedding[rank];+ if (score > local_maximum) {+ local_maximum = score;+ local_winner = token;+ }+ }+ #pragma omp critical+ if (local_maximum > maximum) {+ maximum = local_maximum;+ winner = local_winner;+ }+ }+ if (winner < 0) {+ free(logits); free(hiddens); return -1;+ }+ draft[item] = winner;+ previous = winner;++ float score = 0.0f;+ const float *hidden_row = hiddens + (size_t)item * hidden;+ for (int d = 0; d < hidden; d++)+ score += g_v4ds_core.confidence_proj.data[d] * hidden_row[d];+ for (int rank = 0; rank < V4_DSPARK_RANK; rank++)+ score += g_v4ds_core.confidence_proj.data[hidden + rank] *+ embedding[rank];+ confidence[item] = score >= 0.0f+ ? 1.0f / (1.0f + expf(-score))+ : expf(score) / (1.0f + expf(score));+ }+ free(logits);+ free(hiddens);+ return 0;+}++static int v4_dspark_draft(ColiV4Engine *engine,+ const ColiSafetensorsIndex *index,+ const ColiDeepSeekV4Config *config,+ int anchor_token, int64_t first_position,+ int *draft, int max_out) {+ if (!engine || !index || !config || !draft || max_out < 1) return 0;+ double began = spec_now();+ if (!v4_ds_load_all(index, config)) return 0;+ double loaded = spec_now();++ int hidden = config->hidden_size;+ int hc = config->hc_mult;+ int rope_dim = config->qk_rope_head_dim;+ int pairs = rope_dim / 2;+ int64_t table_rows = first_position + V4_DSPARK_BLOCK;+ if (table_rows < V4_DSPARK_BLOCK || table_rows > INT_MAX) return 0;+ float *cosines = malloc((size_t)table_rows * pairs * sizeof(float));+ float *sines = malloc((size_t)table_rows * pairs * sizeof(float));+ float *state = malloc((size_t)V4_DSPARK_BLOCK * hc * hidden *+ sizeof(float));+ if (!cosines || !sines || !state) {+ free(state); free(sines); free(cosines); return 0;+ }++ /* MTP stages have compress_ratio=0, hence base RoPE without YaRN. */+ if (coli_v4_rope_precompute(cosines, sines, rope_dim, (int)table_rows,+ 0, config->rope_theta, config->rope_factor,+ config->rope_beta_fast, config->rope_beta_slow) ||+ v4_ds_backfill_context(config, first_position, cosines, sines)) {+ free(state); free(sines); free(cosines); return 0;+ }++ int noise_token = getenv("V4_MTP_NOISE")+ ? atoi(getenv("V4_MTP_NOISE")) : 128799;+ size_t flat = (size_t)hc * hidden;+ for (int item = 0; item < V4_DSPARK_BLOCK; item++) {+ int token = item == 0 ? anchor_token : noise_token;+ if (load_embedding(state + (size_t)item * flat,+ index, config, token)) {+ free(state); free(sines); free(cosines); return 0;+ }+ }++ /* A cold first block can route to at most 5*6*3 distinct records. The+ * cache normally makes subsequent rounds zero-I/O; users can lower this+ * cap to warm gradually on slower or memory-constrained machines. */+ int miss_budget = getenv("V4_MTP_MISS")+ ? atoi(getenv("V4_MTP_MISS")) : 96;+ if (miss_budget < 0) miss_budget = 0;+ double prepared = spec_now();+ double stage_seconds[V4_DSPARK_STAGES] = {0};+ for (int stage_id = 0; stage_id < V4_DSPARK_STAGES; stage_id++) {+ double stage_began = spec_now();+ int result = v4_ds_stage_forward(state, &g_v4ds[stage_id], first_position,+ index, config, cosines, sines,+ &miss_budget);+ stage_seconds[stage_id] = spec_now() - stage_began;+ if (result != 0) {+ free(state); free(sines); free(cosines); return 0;+ }+ }++ int candidates[V4_DSPARK_BLOCK];+ float confidence[V4_DSPARK_BLOCK];+ if (v4_ds_heads(engine, index, config, state, anchor_token,+ candidates, confidence)) {+ free(state); free(sines); free(cosines); return 0;+ }+ free(state);+ free(sines);+ free(cosines);+ double headed = spec_now();++ int count = max_out < V4_DSPARK_BLOCK ? max_out : V4_DSPARK_BLOCK;+ double threshold = getenv("V4_MTP_CONF")+ ? atof(getenv("V4_MTP_CONF")) : 0.55;+ if (threshold > 0.0) {+ for (int item = 0; item < count; item++) {+ /* Blend the trained confidence with observed survival on this+ * exact machine/workload. SSD verification is kept only while+ * its estimated prefix survival clears the configured price. */+ double survival = 0.70 * confidence[item] ++ 0.30 * g_v4ds_core.accept_ewma[item];+ if (survival < threshold) {+ count = item;+ break;+ }+ }+ }+ if (count < 1) return 0;+ memcpy(draft, candidates, (size_t)count * sizeof(int));+ g_v4ds_core.rounds++;+ g_v4ds_core.drafted += (uint64_t)count;++ if (getenv("V4_MTP_LOG") && atoi(getenv("V4_MTP_LOG"))) {+ fprintf(stderr,+ "[MTP] draft n=%d conf=%.3f,%.3f,%.3f,%.3f,%.3f "+ "cold_left=%d time=load:%.3f prep:%.3f "+ "stages:%.3f/%.3f/%.3f head:%.3f total:%.3f\n",+ count, confidence[0], confidence[1], confidence[2],+ confidence[3], confidence[4], miss_budget,+ loaded - began, prepared - loaded,+ stage_seconds[0], stage_seconds[1], stage_seconds[2],+ headed - prepared - stage_seconds[0] - stage_seconds[1] -+ stage_seconds[2],+ headed - began);+ }+ return count;+}++static void v4_dspark_feedback(int drafted, int accepted) {+ if (drafted < 0) return;+ if (accepted < 0) accepted = 0;+ if (accepted > drafted) accepted = drafted;+ g_v4ds_core.accepted += (uint64_t)accepted;+ for (int item = 0; item < drafted && item < V4_DSPARK_BLOCK; item++) {+ double sample = item < accepted ? 1.0 : 0.0;+ g_v4ds_core.accept_ewma[item] =+ 0.90 * g_v4ds_core.accept_ewma[item] + 0.10 * sample;+ }+}++static void v4_dspark_report(void) {+ if (!g_v4ds_core.rounds) return;+ fprintf(stderr,+ "[MTP] rounds=%llu drafted=%llu accepted=%llu rate=%.1f%% "+ "ewma=%.2f,%.2f,%.2f,%.2f,%.2f\n",+ (unsigned long long)g_v4ds_core.rounds,+ (unsigned long long)g_v4ds_core.drafted,+ (unsigned long long)g_v4ds_core.accepted,+ g_v4ds_core.drafted+ ? 100.0 * g_v4ds_core.accepted / g_v4ds_core.drafted : 0.0,+ g_v4ds_core.accept_ewma[0], g_v4ds_core.accept_ewma[1],+ g_v4ds_core.accept_ewma[2], g_v4ds_core.accept_ewma[3],+ g_v4ds_core.accept_ewma[4]);+ for (int stage_id = 0; stage_id < V4_DSPARK_STAGES; stage_id++) {+ V4DSparkStage *stage = &g_v4ds[stage_id];+ fprintf(stderr,+ "[MTP] stage=%d cache=%d/%d hits=%lld misses=%lld aborts=%lld\n",+ stage_id, stage->nslots, stage->cap_slots,+ (long long)stage->hits, (long long)stage->misses,+ (long long)stage->aborted);+ }+}diff --git a/c/deepseek_v4_internal.h b/c/deepseek_v4_internal.hindex b62e24da..63b8a8fc 100644--- a/c/deepseek_v4_internal.h+++ b/c/deepseek_v4_internal.h@@ -56,6 +56,13 @@ int coli_st_read_tensor(const ColiSafetensorsIndex *index,const ColiSafetensorsTensor *tensor, void *destination);int coli_st_read_at(const ColiSafetensorsIndex *index, int shard,uint64_t offset, size_t length, void *destination);+/* Large, transient SSD read: prefer the index's O_DIRECT twin and use an+ * aligned bounce buffer, falling back to the ordinary buffered path. */+int coli_st_read_at_streaming(const ColiSafetensorsIndex *index, int shard,+ uint64_t offset, size_t length,+ void *destination);+int coli_st_streaming_direct_available(const ColiSafetensorsIndex *index,+ int shard);int coli_st_prefetch_at(const ColiSafetensorsIndex *index, int shard,uint64_t offset, size_t length);const char *coli_st_dtype_name(ColiSafetensorsDType dtype);@@ -71,6 +78,11 @@ void coli_float_tensor_free(ColiFloatTensor *tensor);typedef struct ColiV4Engine ColiV4Engine;+/* Runtime-selected full DSpark profile, shared with the separately compiled+ * generation unit. */+extern int coli_v4_full_dspark_wanted;+double coli_v4_dspark_cache_gb(void);+/* ==== begin deepseek_v4_math.h ==== */#ifdef __cplusplus@@ -135,6 +147,10 @@ typedef struct {ColiSafetensorsDType dtype;int rank;int64_t shape[COLI_ST_MAX_RANK];+ /* Dense FP8 weights may be transposed inside each 8-row tile after load.+ * This is an in-memory execution layout only; checkpoint bytes and scales+ * remain unchanged. */+ int packed_rows8;} ColiDeepSeekV4TensorSpec;typedef struct {@@ -606,6 +622,7 @@ typedef struct {uint64_t target_expert_cache_bytes;int pin_slots_per_layer;uint64_t repin_interval;+ uint64_t dspark_reserve_bytes;} ColiDeepSeekV4RuntimeOptions;enum { COLI_V4_RESIDENT_MAX_LAYERS = 128 };@@ -628,6 +645,15 @@ struct ColiV4Engine {const ColiSafetensorsIndex *index;uint64_t total_bytes;} dense_resident;+ struct {+ uint16_t *markov_w1;+ uint16_t *markov_w2;+ uint64_t bytes;+ int rank;+ int block_size;+ int stage;+ int enabled;+ } dspark;char *owned_target_model_dir;int owns_experts;int owns_index;@@ -664,6 +690,10 @@ struct ColiV4Session {* from that position instead of re-prefilling it. */kv_prefix fed;int prefix_reused; /* reuse length of the request in flight, for stats */+ uint64_t spec_attempts;+ uint64_t spec_drafted;+ uint64_t spec_accepted;+ int spec_disabled;};/* RAM-tiered expert open used by coli_v4_engine_open (replaces ld --wrap). */diff --git a/c/native_quant_fp4_rows16.h b/c/native_quant_fp4_rows16.hindex dbe855b1..ed73fbf4 100644--- a/c/native_quant_fp4_rows16.h+++ b/c/native_quant_fp4_rows16.h@@ -6,7 +6,7 @@/* Targets with a rows16 fast kernel: the matvec entry points below return 0* and the engine may pack hot experts into the 16-row interleaved layout.* Elsewhere they return -1 and callers keep the flat reference path. */-#if defined(__AVX512F__) || defined(__aarch64__)+#if defined(__AVX512F__) || defined(__AVX2__) || defined(__aarch64__)#define COLI_FP4_ROWS16_KERNEL 1#endifdiff --git a/c/openai_server.py b/c/openai_server.pyindex 28903240..7c312c7a 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -1415,6 +1415,35 @@ def cap_for_arch(arch, cap):return 0 if arch == "glm" else 8+def tune_child_env(env, arch):+ """Apply the engine-local defaults that a direct server launch otherwise misses.++ ``coli chat`` already supplies these values, but users also launch this file+ directly. Keep setdefault semantics so every explicit operator setting wins.+ """+ if arch != "deepseek_v4":+ return env+ if not env.get("COLI_NO_OMP_TUNE"):+ from resource_plan import physical_cpu_count+ env.setdefault("OMP_NUM_THREADS", str(physical_cpu_count()))+ env.setdefault("OMP_WAIT_POLICY", "active")+ env.setdefault("GOMP_SPINCOUNT", "200000")+ env.setdefault("OMP_DYNAMIC", "FALSE")+ if sys.platform != "win32":+ env.setdefault("OMP_PROC_BIND", "close")+ env.setdefault("OMP_PLACES", "cores")+ # All speculative paths stay opt-in: partial acceptance requires expensive+ # recurrent-attention replay on this engine.+ env.setdefault("V4_DRAFT", "0")+ env.setdefault("V4_MTP", "0")+ env.setdefault("V4_MTP_DRAFT", "3")+ env.setdefault("V4_MTP_GB", "0.45")+ env.setdefault("V4_MTP_MISS", "96")+ env.setdefault("V4_MTP_MIN", "3")+ env.setdefault("V4_MTP_CONF", "0.55")+ return env++class Engine:# cap=None = "not explicitly set": a glm-arch model's engine resolves the# 0 sentinel (8 historically, 1 on Metal+darwin+fast SSD -- colibri.c@@ -1423,10 +1452,12 @@ class Engine:# main() below, so programmatic callers that never pass cap get the same# auto behavior as the CLI; an explicit int (0 included) is verbatim.def __init__(self, executable, model, cap=None, max_tokens=1024, env=None, kv_slots=1):+ arch = model_arch(model)child_env = dict(env or os.environ, SNAP=str(model), SERVE="1", SERVE_BATCH="1",NGEN=str(max_tokens), KV_SLOTS=str(kv_slots))+ tune_child_env(child_env, arch)self.process = subprocess.Popen(- [str(executable), str(cap_for_arch(model_arch(model), cap))], env=child_env,+ [str(executable), str(cap_for_arch(arch, cap))], env=child_env,stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=0,)self.write_lock = threading.Lock()diff --git a/c/st.h b/c/st.hindex 35cbd6c2..7261adf4 100644--- a/c/st.h+++ b/c/st.h@@ -6,7 +6,9 @@* - converte sempre in float32 in uscita (BF16/F16/F32 supportati). */#ifndef ST_H#define ST_H+#ifndef _GNU_SOURCE#define _GNU_SOURCE+#endif#include <stdio.h>#include <stdlib.h>#include <string.h>diff --git a/c/tests/test_cli_output.py b/c/tests/test_cli_output.pyindex 0c5a3f68..9659e940 100644--- a/c/tests/test_cli_output.py+++ b/c/tests/test_cli_output.py@@ -227,5 +227,54 @@ class BannerModelLineTest(unittest.TestCase):self.coli.banner("run", True)+class OmpThreadsForEveryEngineTest(unittest.TestCase):+ """#805 set OMP_NUM_THREADS from physical cores -- for glm only.++ env_for_engine() forwarded to env_for() when arch was "glm" and built its+ own environment otherwise, so inkling, kimi_k3, olmoe and deepseek_v4 kept+ libgomp's nproc default: logical cores, a 2x over-subscription of a+ memory-bound int4 GEMV on any SMT host.+ """++ @classmethod+ def setUpClass(cls):+ loader = SourceFileLoader("coli_omp_under_test", str(CLI))+ spec = importlib.util.spec_from_loader(loader.name, loader)+ cls.coli = importlib.util.module_from_spec(spec)+ loader.exec_module(cls.coli)++ def args(self):+ return types.SimpleNamespace(model="/x", ram=None, ctx=None, ngen=None,+ temp=None, cap=None)++ def test_every_engine_gets_physical_cores(self):+ with mock.patch("resource_plan.physical_cpu_count", return_value=6):+ for arch in ("inkling", "kimi", "olmoe", "deepseek_v4"):+ with self.subTest(arch=arch):+ env = self.coli.env_for_engine(self.args(), arch)+ self.assertEqual(env.get("OMP_NUM_THREADS"), "6")++ def test_v4_gets_memory_bound_affinity_defaults(self):+ with mock.patch.object(self.coli.sys, "platform", "linux"), \+ mock.patch("resource_plan.physical_cpu_count", return_value=6):+ env = self.coli.env_for_engine(self.args(), "deepseek_v4")+ self.assertEqual(env.get("OMP_PROC_BIND"), "close")+ self.assertEqual(env.get("OMP_PLACES"), "cores")+ self.assertEqual(env.get("OMP_WAIT_POLICY"), "active")+ self.assertEqual(env.get("OMP_DYNAMIC"), "FALSE")++ def test_explicit_setting_still_wins(self):+ with mock.patch.dict(os.environ, {"OMP_NUM_THREADS": "3"}), \+ mock.patch("resource_plan.physical_cpu_count", return_value=6):+ env = self.coli.env_for_engine(self.args(), "deepseek_v4")+ self.assertEqual(env["OMP_NUM_THREADS"], "3")++ def test_kill_switch_is_honoured(self):+ with mock.patch.dict(os.environ, {"COLI_NO_OMP_TUNE": "1"}, clear=False):+ os.environ.pop("OMP_NUM_THREADS", None)+ env = self.coli.env_for_engine(self.args(), "deepseek_v4")+ self.assertNotIn("OMP_NUM_THREADS", env)++if __name__ == "__main__":unittest.main()diff --git a/c/tests/test_deepseek_v4.c b/c/tests/test_deepseek_v4.cindex b29690e5..dd87043c 100644--- a/c/tests/test_deepseek_v4.c+++ b/c/tests/test_deepseek_v4.c@@ -84,6 +84,8 @@ static int test_config(int argc, char **argv) {"\"index_topk\":8,\"n_routed_experts\":8,\"num_experts_per_tok\":2,""\"n_shared_experts\":1,\"moe_intermediate_size\":32,""\"num_hash_layers\":1,\"num_nextn_predict_layers\":1,"+ "\"dspark_block_size\":5,\"dspark_noise_token_id\":255,"+ "\"dspark_markov_rank\":16,""\"hc_mult\":4,\"hc_sinkhorn_iters\":5,\"vocab_size\":256,""\"max_position_embeddings\":4096,\"rms_norm_eps\":1e-6,""\"hc_eps\":1e-6,\"routed_scaling_factor\":1.5,\"swiglu_limit\":10,"@@ -100,6 +102,8 @@ static int test_config(int argc, char **argv) {}if (config.hidden_size != 128 || config.num_hidden_layers != 3 ||config.n_routed_experts != 8 || config.num_experts_per_tok != 2 ||+ config.dspark_block_size != 5 || config.dspark_noise_token_id != 255 ||+ config.dspark_markov_rank != 16 ||config.compress_ratio_count != 4 || config.compress_ratios[1] != 4 ||config.compress_ratios[2] != 128 || config.rope_factor != 4.0f)return 1;diff --git a/c/tests/test_deepseek_v4_dspark_source.py b/c/tests/test_deepseek_v4_dspark_source.pynew file mode 100644index 00000000..57fb4663--- /dev/null+++ b/c/tests/test_deepseek_v4_dspark_source.py@@ -0,0 +1,79 @@+import pathlib+import unittest+++ROOT = pathlib.Path(__file__).resolve().parents[1]+++class DeepSeekV4DSparkSourceTest(unittest.TestCase):+ @classmethod+ def setUpClass(cls):+ cls.engine = (ROOT / "deepseek_v4.c").read_text(encoding="utf-8")+ cls.drafter = (ROOT / "deepseek_v4_dspark.inc").read_text(+ encoding="utf-8"+ )+ cls.launcher = (ROOT / "coli").read_text(encoding="utf-8")++ def test_complete_three_stage_drafter_is_built(self):+ self.assertEqual(self.drafter.count("static int v4_dspark_draft("), 1)+ self.assertIn("#define V4_DSPARK_STAGES 3", self.drafter)+ self.assertIn('"mtp.2.markov_head.markov_w1.weight"', self.drafter)+ self.assertIn('"confidence_head.proj.weight"', self.drafter)++ def test_drafter_is_lazy_and_budgeted_before_target_cache(self):+ reserve = self.engine.index("engine->runtime.dspark_reserve_bytes =")+ store = self.engine.index("coli_v4_expert_store_open_planned(", reserve)+ self.assertLess(reserve, store)+ self.assertIn("load=lazy verification=exact-target", self.engine)+ self.assertIn("double total_gb = coli_v4_dspark_cache_gb()", self.drafter)++ def test_exact_target_hidden_precedes_full_mtp(self):+ self.assertIn("int full_mtp_ready = 0;", self.engine)+ target = self.engine.index("if (target_token(engine, &state, &next")+ ready = self.engine.index("full_mtp_ready = 1;", target)+ draft = self.engine.index("proposals = v4_dspark_draft(")+ self.assertLess(draft, target)+ self.assertLess(target, ready)++ def test_rejected_suffix_invalidates_hidden_taps(self):+ restore = self.engine.index("if (spec_attention_restore(")+ invalidate = self.engine.index("v4_ds_invalidate_from(old_last + 1)")+ replay = self.engine.index("if (retained > 0 && target_batch(", invalidate)+ self.assertLess(restore, invalidate)+ self.assertLess(invalidate, replay)++ def test_prompt_lookup_and_full_mtp_are_exactly_verified(self):+ self.assertIn("static int v4_ngram_draft(", self.engine)+ self.assertIn('getenv("V4_NGRAM")', self.engine)+ self.assertIn("int batch = proposals + 1;", self.engine)+ self.assertIn("predictions[accepted] == drafts[accepted]", self.engine)++ def test_full_mtp_expands_ue8m0_scales_once(self):+ self.assertIn("float *wq_a_s, *wq_b_s", self.drafter)+ self.assertIn("st_read_scale_f32(", self.drafter)+ self.assertIn("view->scale_format = COLI_SCALE_F32", self.drafter)+ self.assertIn("* sizeof(float);", self.drafter)+ self.assertIn("static int v4_ds_pack_rows8(", self.drafter)+ self.assertIn("view->block_rows = 8", self.drafter)++ def test_chat_keeps_all_speculation_opt_in(self):+ for setting in (+ 'env.setdefault("V4_DRAFT", "0")',+ 'env.setdefault("V4_MTP", "0")',+ 'env.setdefault("V4_MTP_DRAFT", "3")',+ 'env.setdefault("V4_MTP_GB", "0.45")',+ ):+ self.assertIn(setting, self.launcher)+ self.assertIn(".no_dspark = 0,", self.engine)+ self.assertIn("accepted < proposals", self.engine)+ self.assertIn('getenv("V4_MTP_PARTIAL_KEEP")', self.engine)+ self.assertIn('getenv("V4_NGRAM_PARTIAL_KEEP")', self.engine)++ def test_target_ssd_reader_uses_aligned_final_slots(self):+ self.assertIn("v4_read_expert_record(state, record, slot)", self.engine)+ self.assertIn("posix_memalign((void **)&slot->slab, 4096", self.engine)+ self.assertIn("payload_bytes=%llu", self.engine)+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tests/test_deepseek_v4_tiny.py b/c/tests/test_deepseek_v4_tiny.pyindex 9fae7c2c..e07de41b 100644--- a/c/tests/test_deepseek_v4_tiny.py+++ b/c/tests/test_deepseek_v4_tiny.py@@ -161,8 +161,20 @@ def check_session(f"session {name}: length mismatch: "f"prompt={prompt_count} generated={generated_count}")- if compatibility_flag and "compatibility no-op" not in result.stderr:- raise AssertionError("--no-dspark compatibility notice was not emitted")+ if compatibility_flag:+ # --no-dspark used to be a no-op that only printed a notice, because the+ # engine was target-only and had nothing to disable. It now disables+ # verified speculative drafting for real, so asserting the old notice+ # would require the engine to keep claiming it does nothing.+ #+ # What matters either way is that the run stays token-exact, which the+ # checks above already prove, and that the flag actually suppresses+ # speculation: with drafting off no attempt is ever made, so the+ # counters the engine prints at exit must be zero.+ attempts = re.search(r"v4_dspark attempts=(\d+)", result.stderr)+ if attempts and int(attempts.group(1)) != 0:+ raise AssertionError(+ f"--no-dspark still attempted {attempts.group(1)} speculations")print(f"PASS target session {name}: exact IDs and exact length")return actual["full_ids"]diff --git a/c/tests/test_native_quant.c b/c/tests/test_native_quant.cindex 480e87da..8e7ed4c7 100644--- a/c/tests/test_native_quant.c+++ b/c/tests/test_native_quant.c@@ -97,6 +97,32 @@ int main(void) {return 1;for (int i = 0; i < 128; i++)if (!close_enough(fp8_output[i], 128.0f)) return 1;++ /* Flash layout: eight rows are interleaved by column. It must be+ * numerically equivalent to the guarded row-major reference path while+ * exercising different values in every lane. */+ uint8_t row_major[8 * 128], rows8[8 * 128];+ float row_major_out[8], rows8_out[8];+ for (int row = 0; row < 8; row++)+ for (int column = 0; column < 128; column++)+ row_major[row * 128 + column] =+ (uint8_t)(0x30 + ((row + column) & 15));+ for (int column = 0; column < 128; column++)+ for (int row = 0; row < 8; row++)+ rows8[column * 8 + row] = row_major[row * 128 + column];+ ColiTensorView row_major_view = {+ COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_F32,+ row_major, fp8_scales, sizeof(row_major), sizeof(fp8_scales),+ 8, 128, 128, 128+ };+ ColiTensorView rows8_view = row_major_view;+ rows8_view.data = rows8;+ rows8_view.block_rows = 8;+ if (coli_fp8_matvec_ref(row_major_out, &row_major_view, input) != 0 ||+ coli_fp8_matvec_ref(rows8_out, &rows8_view, input) != 0)+ return 1;+ for (int row = 0; row < 8; row++)+ if (!close_enough(rows8_out[row], row_major_out[row])) return 1;puts("native quant tests: ok");return 0;}diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex 5465499d..796bba02 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -20,7 +20,7 @@ from openai_server import (APIError, APIHandler, APIServer, ClientCancelled,_engine_error, cap_for_arch, conversation_cache_slot, model_arch,generation_options, parse_tool_calls, read_engine_turn,render_chat, render_chat_kimi, serve,- split_thinking_reply, stop_policy)+ split_thinking_reply, stop_policy, tune_child_env)class FakeEngine:@@ -670,8 +670,22 @@ class CapSentinelShimTest(unittest.TestCase):self.assertEqual(model_arch(self._model("glm_moe_dsa")), "glm")self.assertEqual(model_arch(self._model("inkling")), "inkling")self.assertEqual(model_arch(self._model("kimi_k3")), "kimi")+ self.assertEqual(model_arch(self._model("deepseek_v4")), "deepseek_v4")self.assertEqual(model_arch("/nonexistent"), "glm")+ def test_direct_v4_server_gets_bounded_dspark_defaults(self):+ env = {"V4_MTP_CONF": "0.7"}+ with patch("resource_plan.physical_cpu_count", return_value=6), \+ patch("openai_server.sys.platform", "linux"):+ tune_child_env(env, "deepseek_v4")+ self.assertEqual(env["OMP_NUM_THREADS"], "6")+ self.assertEqual(env["OMP_PROC_BIND"], "close")+ self.assertEqual(env["V4_DRAFT"], "0")+ self.assertEqual(env["V4_MTP"], "0")+ self.assertEqual(env["V4_MTP_DRAFT"], "3")+ self.assertEqual(env["V4_MTP_GB"], "0.45")+ self.assertEqual(env["V4_MTP_CONF"], "0.7") # explicit override wins+class HTTPTest(unittest.TestCase):@classmethoddiff --git a/c/tok.h b/c/tok.hindex 66853f68..54c5c699 100644--- a/c/tok.h+++ b/c/tok.h@@ -12,7 +12,9 @@*/#ifndef TOK_H#define TOK_H+#ifndef _GNU_SOURCE#define _GNU_SOURCE+#endif#include <stdio.h>#include <stdlib.h>#include <string.h>