Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

deepseek_v4: AVX2 kernels, direct expert I/O, warm pins, verified drafting

JustVuggb7117bfd7cf2
diff --git a/c/Makefile.deepseek-v4 b/c/Makefile.deepseek-v4
index 7aee0a92..0aa3a5ad 100644
--- a/c/Makefile.deepseek-v4
+++ b/c/Makefile.deepseek-v4
@@ -28,7 +28,7 @@ endif
ifneq ($(IS_WIN),)
CC = gcc
ARCH ?= x86-64-v3
-CFLAGS = -D_FILE_OFFSET_BITS=64 -O3 -march=$(ARCH) -fopenmp \
+CFLAGS = -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -O3 -march=$(ARCH) -fopenmp \
-include pthread.h -Wall -Wextra -Wno-unused-parameter \
-Wno-misleading-indentation -Wno-unused-function \
-DCOLI_V4_MAX_PIN_SLOTS_PER_LAYER=16 \
@@ -38,7 +38,7 @@ V4_BINARY := deepseek_v4.exe
else
CC = gcc
ARCH ?= native
-CFLAGS = -O3 -march=$(ARCH) -fopenmp -pthread -include pthread.h \
+CFLAGS = -D_GNU_SOURCE -O3 -march=$(ARCH) -fopenmp -pthread -include pthread.h \
-Wall -Wextra -Wno-unused-parameter \
-Wno-misleading-indentation -Wno-unused-function \
-DCOLI_V4_MAX_PIN_SLOTS_PER_LAYER=16 \
@@ -47,6 +47,12 @@ LDFLAGS = -lm -fopenmp -pthread
V4_BINARY := deepseek_v4
endif
+LTO ?= 1
+ifeq ($(LTO),1)
+CFLAGS += -flto
+LDFLAGS += -flto
+endif
+
include Makefile.deepseek-v4.units
TARGET_OBJS = $(addsuffix .o,$(V4_TARGET_UNITS))
@@ -60,7 +66,7 @@ $(V4_BINARY): $(V4_OBJS)
$(CC) $(V4_OBJS) -o $@ $(LDFLAGS)
$(TARGET_OBJS) $(TEST_UNIT_OBJS): %.o: deepseek_v4.c deepseek_v4.h \
- deepseek_v4_internal.h st.h json.h compat.h tensor.h quant.h \
+ deepseek_v4_internal.h deepseek_v4_dspark.inc st.h json.h compat.h tensor.h quant.h \
native_quant.h native_quant_batch.h native_quant_dual.h \
native_quant_fp4_rows16.h
$(CC) $(CFLAGS) -D$* -c deepseek_v4.c -o $@
diff --git a/c/coli b/c/coli
index 0d6d8f88..6f6942a9 100755
--- a/c/coli
+++ b/c/coli
@@ -267,11 +267,50 @@ def env_for_engine(a, arch):
if arch == "glm":
return env_for(a)
env = os.environ.copy()
+ # OMP_NUM_THREADS for the sister engines too.
+ #
+ # #805 set this from physical cores "on every platform" -- but only inside
+ # env_for(), which this function calls only for glm. inkling, kimi_k3,
+ # olmoe and deepseek_v4 all took the branch below and got libgomp's
+ # default, nproc, i.e. LOGICAL cores. On any SMT host that is a 2x
+ # over-subscription of a memory-bound int4 GEMV, which is the collapse
+ # measured in #718 (2.3x on Zen3) and again here: an i7-1355U reports 6
+ # physical and 12 logical, and DeepSeek V4 was running the 12.
+ #
+ # Same defect shape as the one #805 fixed, one level out: a mechanism that
+ # reaches one engine and not its siblings.
+ #
+ # setdefault, so an explicit OMP_NUM_THREADS still wins -- which is also
+ # how to A/B it: OMP_NUM_THREADS=12 reproduces the old behaviour.
+ if not env.get("COLI_NO_OMP_TUNE"):
+ from resource_plan import physical_cpu_count
+ env.setdefault("OMP_NUM_THREADS", str(physical_cpu_count()))
+ if arch == "deepseek_v4":
+ env.setdefault("OMP_WAIT_POLICY", "active")
+ env.setdefault("GOMP_SPINCOUNT", "200000")
+ env.setdefault("OMP_DYNAMIC", "FALSE")
+ if sys.platform != "win32":
+ env.setdefault("OMP_PROC_BIND", "close")
+ env.setdefault("OMP_PLACES", "cores")
if a.ngen: env["NGEN"] = str(a.ngen)
if a.temp is not None: env["COLI_TEMP"] = str(a.temp)
if arch == "deepseek_v4":
if a.ram: env["RAM_GB"] = str(a.ram)
if a.ctx: env["CTX"] = str(a.ctx)
+ # Speculation stays opt-in. Real multi-turn chat measured only 1/15
+ # accepted prompt-lookup candidates; recurrent-state replay dominated
+ # the visible decode. V4_DRAFT=5 remains available for repeated code.
+ env.setdefault("V4_DRAFT", "0")
+ # Full three-stage DSpark remains opt-in. In a real multi-turn chat it
+ # accepted only 10/24 candidates and rejected-suffix replay turned a
+ # 14-token answer into 495 seconds. V4_MTP=1 still enables A/B runs;
+ # exact zero-I/O prompt lookup above remains enabled by default.
+ env.setdefault("V4_MTP", "0")
+ env.setdefault("V4_MTP_DRAFT", "3")
+ env.setdefault("V4_MTP_GB", "0.45")
+ env.setdefault("V4_MTP_MISS", "96")
+ env.setdefault("V4_MTP_MIN", "3")
+ env.setdefault("V4_MTP_CONF", "0.55")
return env
def cuda_binary():
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.c
index 31cb57ec..7f740c2d 100644
--- a/c/deepseek_v4.c
+++ b/c/deepseek_v4.c
@@ -1,3 +1,6 @@
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
/* Amalgamated deepseek_v4.c — GLM-style source; compile with -DCOLI_V4_UNIT_* per object */
/* Umbrella API: deepseek_v4.h (included by units) */
@@ -103,6 +106,71 @@ int coli_st_read_at(const ColiSafetensorsIndex *index, int shard,
return 0;
}
+int coli_st_streaming_direct_available(const ColiSafetensorsIndex *index,
+ int shard) {
+ if (!index || shard < 0 || shard >= index->nfd) return 0;
+ const char *setting = getenv("COLI_V4_DIRECT");
+ if (setting && atoi(setting) == 0) return 0;
+ return index->dfds[shard] >= 0;
+}
+
+int coli_st_read_at_streaming(const ColiSafetensorsIndex *index, int shard,
+ uint64_t offset, size_t length,
+ void *destination) {
+ if (!index || !destination || shard < 0 || shard >= index->nfd ||
+ index->sizes[shard] < 0 || offset > (uint64_t)index->sizes[shard] ||
+ length > (uint64_t)index->sizes[shard] - offset)
+ return -1;
+ if (!length) return 0;
+ if (!coli_st_streaming_direct_available(index, shard))
+ return coli_st_read_at(index, shard, offset, length, destination);
+
+ const uint64_t alignment = 4096;
+ uint64_t base = offset & ~(alignment - 1);
+ uint64_t pad = offset - base;
+ if ((uint64_t)length > UINT64_MAX - pad) return -1;
+ uint64_t needed = pad + (uint64_t)length;
+ if (needed > UINT64_MAX - (alignment - 1) ||
+ needed + alignment - 1 > SIZE_MAX) return -1;
+ size_t allocation_bytes = (size_t)((needed + alignment - 1) &
+ ~(alignment - 1));
+ unsigned char *bounce = NULL;
+ if (posix_memalign((void **)&bounce, (size_t)alignment,
+ allocation_bytes) != 0)
+ return coli_st_read_at(index, shard, offset, length, destination);
+
+ uint64_t file_bytes = (uint64_t)index->sizes[shard];
+ uint64_t available = file_bytes - base;
+ uint64_t direct_bytes = allocation_bytes;
+ if (direct_bytes > available)
+ direct_bytes = available & ~(alignment - 1);
+ size_t done = 0;
+ int failed = 0;
+ while ((uint64_t)done < direct_bytes) {
+ ssize_t count = pread(index->dfds[shard], bounce + done,
+ (size_t)(direct_bytes - done),
+ (off_t)(base + done));
+ if (count < 0 && errno == EINTR) continue;
+ if (count <= 0) { failed = 1; break; }
+ done += (size_t)count;
+ }
+ if (failed) {
+ free(bounce);
+ return coli_st_read_at(index, shard, offset, length, destination);
+ }
+ while ((uint64_t)done < needed) {
+ ssize_t count = pread(index->fds[shard], bounce + done,
+ (size_t)(needed - done),
+ (off_t)(base + done));
+ if (count < 0 && errno == EINTR) continue;
+ if (count <= 0) { free(bounce); return -1; }
+ done += (size_t)count;
+ }
+ memcpy(destination, bounce + pad, length);
+ free(bounce);
+ return 0;
+}
+
int coli_st_read_tensor(const ColiSafetensorsIndex *index,
const ColiSafetensorsTensor *tensor, void *destination) {
int shard = coli_st_tensor_shard(index, tensor);
@@ -408,6 +476,33 @@ void coli_v4_layer_free(ColiV4Engine *engine,
memset(weights, 0, sizeof(*weights));
}
+/* AVX2 consumes eight output rows at once. Interleave those rows by column
+ * once when a dense layer becomes resident, instead of gathering eight
+ * distant cache lines in every matvec. */
+static int v4_fp8_pack_rows8_inplace(unsigned char *data,
+ int64_t rows, int64_t columns) {
+#ifndef __AVX2__
+ (void)data; (void)rows; (void)columns;
+ return 0;
+#else
+ if (!data || rows < 8 || rows % 8 || columns < 128 || columns % 128)
+ return 0;
+ size_t tile_bytes = (size_t)8 * (size_t)columns;
+ unsigned char *scratch = malloc(tile_bytes);
+ if (!scratch) return -1;
+ for (int64_t tile = 0; tile < rows / 8; tile++) {
+ unsigned char *target = data + (size_t)tile * tile_bytes;
+ memcpy(scratch, target, tile_bytes);
+ for (int64_t column = 0; column < columns; column++)
+ for (int lane = 0; lane < 8; lane++)
+ target[(size_t)column * 8 + lane] =
+ scratch[(size_t)lane * columns + column];
+ }
+ free(scratch);
+ return 1;
+#endif
+}
+
int coli_v4_layer_load(ColiV4Engine *engine,
ColiDeepSeekV4LayerWeights *weights,
const ColiDeepSeekV4Config *config,
@@ -438,6 +533,17 @@ int coli_v4_layer_load(ColiV4Engine *engine,
coli_v4_layer_free(NULL, weights);
return set_error(error, error_size, "cannot read tensor: %s", spec->name);
}
+ if (spec->dtype == COLI_ST_F8_E4M3 && spec->rank == 2) {
+ int packed = v4_fp8_pack_rows8_inplace(
+ weights->data[i], spec->shape[0], spec->shape[1]);
+ if (packed < 0) {
+ coli_v4_layer_free(NULL, weights);
+ return set_error(error, error_size,
+ "out of memory packing FP8 rows8: %s",
+ spec->name);
+ }
+ weights->plan.tensors[i].packed_rows8 = packed > 0;
+ }
}
return 0;
}
@@ -848,6 +954,11 @@ static int build_runtime_plan(ColiV4Engine *engine,
sizeof(float) * 2;
uint64_t scratch = 512 * MIB;
uint64_t runtime_other = context_bytes(&config, context) + hidden + scratch;
+ if (UINT64_MAX - runtime_other < runtime->dspark_reserve_bytes) {
+ snprintf(error, error_size, "V4 DSpark reserve overflow");
+ return -1;
+ }
+ runtime_other += runtime->dspark_reserve_bytes;
uint64_t available = coli_v4_os_available_memory();
if (!available) {
snprintf(error, error_size, "cannot determine OS available memory");
@@ -1404,7 +1515,8 @@ static int fp8_view(ColiTensorView *view,
COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_F32, data, scales,
(size_t)(weight_spec->shape[0] * weight_spec->shape[1]),
(size_t)(scale_spec->shape[0] * scale_spec->shape[1]) * sizeof(float),
- weight_spec->shape[0], weight_spec->shape[1], 128, 128
+ weight_spec->shape[0], weight_spec->shape[1],
+ weight_spec->packed_rows8 ? 8 : 128, 128
};
return 0;
}
@@ -1801,7 +1913,8 @@ static int fp8_view(ColiTensorView *view,
COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_F32, data, scales,
(size_t)(weight_spec->shape[0] * weight_spec->shape[1]),
(size_t)(scale_spec->shape[0] * scale_spec->shape[1]) * sizeof(float),
- weight_spec->shape[0], weight_spec->shape[1], 128, 128
+ weight_spec->shape[0], weight_spec->shape[1],
+ weight_spec->packed_rows8 ? 8 : 128, 128
};
return 0;
}
@@ -2611,7 +2724,7 @@ static int fp8_view(ColiTensorView *view,
COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_F32, data, scales,
(size_t)(ws->shape[0] * ws->shape[1]),
(size_t)(ss->shape[0] * ss->shape[1]) * sizeof(float),
- ws->shape[0], ws->shape[1], 128, 128
+ ws->shape[0], ws->shape[1], ws->packed_rows8 ? 8 : 128, 128
};
return 0;
}
@@ -2930,7 +3043,7 @@ static int fp8_view(ColiTensorView *view,
COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_F32, data, scales,
(size_t)(ws->shape[0] * ws->shape[1]),
(size_t)(ss->shape[0] * ss->shape[1]) * sizeof(float),
- ws->shape[0], ws->shape[1], 128, 128
+ ws->shape[0], ws->shape[1], ws->packed_rows8 ? 8 : 128, 128
};
return 0;
}
@@ -4174,7 +4287,7 @@ static int fp8_view(ColiTensorView *view,
COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_F32, data, scales,
(size_t)(ws->shape[0] * ws->shape[1]),
(size_t)(ss->shape[0] * ss->shape[1]) * sizeof(float),
- ws->shape[0], ws->shape[1], 128, 128
+ ws->shape[0], ws->shape[1], ws->packed_rows8 ? 8 : 128, 128
};
return 0;
}
@@ -4574,7 +4687,8 @@ static int fp8_view(ColiTensorView *view,
COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_F32, data, scales,
(size_t)(weight_spec->shape[0] * weight_spec->shape[1]),
(size_t)(scale_spec->shape[0] * scale_spec->shape[1]) * sizeof(float),
- weight_spec->shape[0], weight_spec->shape[1], 128, 128
+ weight_spec->shape[0], weight_spec->shape[1],
+ weight_spec->packed_rows8 ? 8 : 128, 128
};
return 0;
}
@@ -4904,7 +5018,9 @@ void coli_v4_attention_snapshot_destroy(ColiV4AttentionSnapshot *snapshot) {
#define coli_deepseek_v4_expert_store_open \
coli_deepseek_v4_expert_store_open_base
/* ---- begin include deepseek_v4_expert_store.c ---- */
+#ifndef _GNU_SOURCE
#define _GNU_SOURCE
+#endif
#include "deepseek_v4_internal.h"
#include <assert.h>
@@ -4939,6 +5055,7 @@ typedef struct {
unsigned references;
uint64_t used;
unsigned char *slab;
+ int aligned_slab;
} V4ExpertSlot;
typedef struct {
@@ -5104,11 +5221,13 @@ static int lookup(ColiExpertStore *store, ColiExpertKey key,
}
/* A short read must never expose a partially overwritten old slot. */
slot->expert = -1;
- if (coli_st_read_at(state->index, record->shard, record->scale_offset,
- (size_t)record->scale_bytes, slot->slab) != 0 ||
- coli_st_read_at(state->index, record->shard, record->weight_offset,
- (size_t)record->weight_bytes,
- slot->slab + record->scale_bytes) != 0) {
+ if (coli_st_read_at_streaming(
+ state->index, record->shard, record->scale_offset,
+ (size_t)record->scale_bytes, slot->slab) != 0 ||
+ coli_st_read_at_streaming(
+ state->index, record->shard, record->weight_offset,
+ (size_t)record->weight_bytes,
+ slot->slab + record->scale_bytes) != 0) {
pthread_mutex_unlock(&state->mutex);
memset(view, 0, sizeof(*view));
return -1;
@@ -5308,6 +5427,8 @@ fail:
#undef coli_deepseek_v4_expert_store_open
#include "native_quant_fp4_rows16.h"
+#include <limits.h>
+#include <time.h>
#include "deepseek_v4_internal.h"
@@ -5320,12 +5441,141 @@ typedef struct V4HotPolicy {
int *pins;
unsigned char *packed;
uint64_t packed_slots;
+ uint64_t history_total;
+ int history_seeded;
+ char *history_path;
struct V4HotPolicy *next;
} V4HotPolicy;
static pthread_mutex_t hot_policies_mutex = PTHREAD_MUTEX_INITIALIZER;
static V4HotPolicy *hot_policies;
+static double hot_now(void) {
+ struct timespec value;
+ clock_gettime(CLOCK_MONOTONIC, &value);
+ return (double)value.tv_sec + value.tv_nsec * 1e-9;
+}
+
+static uint32_t hot_engine_id(void) {
+ const unsigned char *text = (const unsigned char *)"deepseek_v4";
+ uint32_t hash = 2166136261u;
+ while (*text) { hash ^= *text++; hash *= 16777619u; }
+ return hash;
+}
+
+static char *hot_history_path(const char *model_dir) {
+ if (!model_dir) return NULL;
+ const char suffix[] = "/.coli_usage";
+ size_t length = strlen(model_dir) + sizeof(suffix);
+ char *path = malloc(length);
+ if (path) snprintf(path, length, "%s%s", model_dir, suffix);
+ return path;
+}
+
+/* Seed the hot-expert ranking from the same sparse text history used by the
+ * other engines. Placement is the only observable effect: expert ids and
+ * weights are still selected by the V4 router on every forward. */
+static uint64_t hot_usage_load(V4HotPolicy *policy,
+ const V4ExpertStoreState *state,
+ const char *model_dir) {
+ if (!policy || !state || !model_dir) return 0;
+ char *path = hot_history_path(model_dir);
+ if (!path) return 0;
+ FILE *stream = fopen(path, "rb");
+ if (!stream) { free(path); return 0; }
+
+ int layer = 0, expert = 0, dimensions_seen = 0;
+ unsigned count = 0;
+ while (fscanf(stream, "%d %d %u", &layer, &expert, &count) == 3) {
+ if (layer == -1) {
+ dimensions_seen = 1;
+ if (expert != state->layers ||
+ count != (unsigned)state->experts_per_layer) {
+ fprintf(stderr,
+ "v4_autopin ignored=%s dimensions=%d/%u expected=%d/%d\n",
+ path, expert, count, state->layers,
+ state->experts_per_layer);
+ fclose(stream); free(path); return 0;
+ }
+ } else if (layer == -2 &&
+ (expert > 1 || count != hot_engine_id())) {
+ fprintf(stderr,
+ "v4_autopin ignored=%s identity=version-%d/id-%u "
+ "expected=version-1/id-%u\n",
+ path, expert, count, hot_engine_id());
+ fclose(stream); free(path); return 0;
+ }
+ }
+ if (!dimensions_seen) {
+ fclose(stream); free(path); return 0;
+ }
+ rewind(stream);
+ uint64_t total = 0;
+ while (fscanf(stream, "%d %d %u", &layer, &expert, &count) == 3) {
+ if (layer < 0 || layer >= state->layers || expert < 0 ||
+ expert >= state->experts_per_layer || !count) continue;
+ uint64_t *slot = &policy->usage[
+ (size_t)layer * state->experts_per_layer + expert];
+ if (UINT64_MAX - *slot < count) *slot = UINT64_MAX;
+ else *slot += count;
+ if (UINT64_MAX - total < count) total = UINT64_MAX;
+ else total += count;
+ }
+ fclose(stream);
+ if (total)
+ fprintf(stderr, "v4_autopin history=%s selections=%llu\n", path,
+ (unsigned long long)total);
+ free(path);
+ return total;
+}
+
+static void hot_usage_save(const V4HotPolicy *policy,
+ const V4ExpertStoreState *state) {
+ if (!policy || !state || !policy->history_path || !policy->usage) return;
+ const char *enabled = getenv("COLI_V4_SAVE_USAGE");
+ if (enabled && atoi(enabled) == 0) return;
+ size_t tmp_length = strlen(policy->history_path) + sizeof(".tmp");
+ char *tmp = malloc(tmp_length);
+ if (!tmp) return;
+ snprintf(tmp, tmp_length, "%s.tmp", policy->history_path);
+ FILE *stream = fopen(tmp, "wb");
+ if (!stream) { free(tmp); return; }
+
+ uint64_t total = 0, distinct = 0;
+ for (int layer = 0; layer < state->layers; layer++)
+ for (int expert = 0; expert < state->experts_per_layer; expert++) {
+ uint64_t count = policy->usage[
+ (size_t)layer * state->experts_per_layer + expert];
+ if (!count) continue;
+ total = UINT64_MAX - total < count ? UINT64_MAX : total + count;
+ distinct++;
+ }
+ if (distinct) {
+ fprintf(stream, "-1 %d %d\n", state->layers,
+ state->experts_per_layer);
+ fprintf(stream, "-2 1 %u\n", hot_engine_id());
+ for (int layer = 0; layer < state->layers; layer++)
+ for (int expert = 0; expert < state->experts_per_layer; expert++) {
+ uint64_t count = policy->usage[
+ (size_t)layer * state->experts_per_layer + expert];
+ if (!count) continue;
+ unsigned stored = count > UINT_MAX ? UINT_MAX : (unsigned)count;
+ fprintf(stream, "%d %d %u\n", layer, expert, stored);
+ }
+ }
+ int failed = ferror(stream);
+ if (fclose(stream) != 0) failed = 1;
+ if (!failed && rename(tmp, policy->history_path) == 0)
+ fprintf(stderr,
+ "v4_autopin saved=%s selections=%llu distinct=%llu\n",
+ policy->history_path, (unsigned long long)total,
+ (unsigned long long)distinct);
+ else
+ fprintf(stderr, "v4_autopin warning=cannot-save-history path=%s\n",
+ policy->history_path);
+ free(tmp);
+}
+
static V4HotPolicy *hot_find(ColiExpertStore *store) {
pthread_mutex_lock(&hot_policies_mutex);
@@ -5343,7 +5593,7 @@ static int hot_is_pinned(const V4HotPolicy *policy, int layer, int expert) {
if (!policy || policy->pin_count < 1) return 0;
int active = policy->pin_count;
#if COLI_V4_PIN_RAMP_REQUESTS > 0
- if (active > 4) {
+ if (!policy->history_seeded && active > 4) {
uint64_t grown = policy->layer_requests[layer] /
COLI_V4_PIN_RAMP_REQUESTS;
active = grown >= (uint64_t)(active - 4)
@@ -5361,6 +5611,98 @@ static size_t hot_slot_index(const V4ExpertStoreState *state,
return (size_t)(slot - state->slots);
}
+/* Expert records dominate decode I/O. Read the large FP4 payload directly
+ * into the final cache slot, instead of allocating a record-sized bounce
+ * buffer and copying it for every miss. The extra two pages on each slot let
+ * an unaligned safetensors range be expanded to an O_DIRECT window safely.
+ *
+ * FLOCK-packed checkpoints store [scales][weights] contiguously and need one
+ * request. Standard HF checkpoints keep the ranges apart: weights use direct
+ * I/O while the much smaller scales use buffered pread. Any direct-I/O error
+ * falls back to the exact buffered path. */
+static uint64_t v4_direct_reads;
+static uint64_t v4_direct_flock_reads;
+static uint64_t v4_direct_payload_bytes;
+static uint64_t v4_direct_fallbacks;
+
+static int v4_pread_full_try(int fd, void *destination, size_t length,
+ uint64_t offset) {
+ unsigned char *output = destination;
+ size_t done = 0;
+ while (done < length) {
+ ssize_t count = pread(fd, output + done, length - done,
+ (off_t)(offset + done));
+ if (count < 0 && errno == EINTR) continue;
+ if (count <= 0) return -1;
+ done += (size_t)count;
+ }
+ return 0;
+}
+
+static int v4_read_direct_window(const V4ExpertStoreState *state, int shard,
+ unsigned char *slab, uint64_t offset,
+ size_t length, size_t destination_offset) {
+ if (!state || !state->index || shard < 0 || shard >= state->index->nfd ||
+ state->index->dfds[shard] < 0) return -1;
+ const uint64_t alignment = 4096;
+ uint64_t base = offset & ~(alignment - 1);
+ size_t pad = (size_t)(offset - base);
+ if (length > SIZE_MAX - pad) return -1;
+ size_t wanted = pad + length;
+ if (wanted > SIZE_MAX - (alignment - 1)) return -1;
+ size_t direct_length = (wanted + alignment - 1) & ~(size_t)(alignment - 1);
+ uint64_t file_bytes = (uint64_t)state->index->sizes[shard];
+ if (base > file_bytes) return -1;
+ uint64_t available = file_bytes - base;
+ if ((uint64_t)direct_length > available)
+ direct_length = (size_t)(available & ~(alignment - 1));
+ if (direct_length && v4_pread_full_try(
+ state->index->dfds[shard], slab, direct_length, base)) return -1;
+ if (direct_length < wanted && v4_pread_full_try(
+ state->index->fds[shard], slab + direct_length,
+ wanted - direct_length, base + direct_length)) return -1;
+ memmove(slab + destination_offset, slab + pad, length);
+ return 0;
+}
+
+static int v4_read_expert_record(V4ExpertStoreState *state,
+ const V4ExpertRecord *record,
+ V4ExpertSlot *slot) {
+ int direct_available = slot->aligned_slab &&
+ coli_st_streaming_direct_available(state->index, record->shard);
+ if (direct_available &&
+ record->scale_offset + record->scale_bytes == record->weight_offset &&
+ !v4_read_direct_window(state, record->shard, slot->slab,
+ record->scale_offset,
+ (size_t)record->record_bytes, 0)) {
+ __atomic_fetch_add(&v4_direct_reads, UINT64_C(1), __ATOMIC_RELAXED);
+ __atomic_fetch_add(&v4_direct_flock_reads, UINT64_C(1),
+ __ATOMIC_RELAXED);
+ __atomic_fetch_add(&v4_direct_payload_bytes, record->record_bytes,
+ __ATOMIC_RELAXED);
+ return 0;
+ }
+
+ int weight_direct = direct_available && !v4_read_direct_window(
+ state, record->shard, slot->slab, record->weight_offset,
+ (size_t)record->weight_bytes, (size_t)record->scale_bytes);
+ if (weight_direct) {
+ __atomic_fetch_add(&v4_direct_reads, UINT64_C(1), __ATOMIC_RELAXED);
+ __atomic_fetch_add(&v4_direct_payload_bytes, record->weight_bytes,
+ __ATOMIC_RELAXED);
+ } else {
+ if (direct_available)
+ __atomic_fetch_add(&v4_direct_fallbacks, UINT64_C(1),
+ __ATOMIC_RELAXED);
+ if (coli_st_read_at(state->index, record->shard,
+ record->weight_offset,
+ (size_t)record->weight_bytes,
+ slot->slab + record->scale_bytes)) return -1;
+ }
+ return coli_st_read_at(state->index, record->shard, record->scale_offset,
+ (size_t)record->scale_bytes, slot->slab);
+}
+
static void hot_fill_view(ColiTensorView *view,
const V4ExpertRecord *record,
const V4ExpertSlot *slot, int matrix,
@@ -5443,7 +5785,7 @@ static void hot_repin_locked(V4HotPolicy *policy, V4ExpertStoreState *state,
hot_is_pinned(policy, layer, slots[i].expert))
slots[i].used = ++state->clock;
uint64_t decay_interval = policy->repin_interval * 64;
- if (decay_interval &&
+ if (decay_interval && policy->layer_requests[layer] &&
policy->layer_requests[layer] % decay_interval == 0)
for (int expert = 0; expert < state->experts_per_layer; expert++)
usage[expert] = (usage[expert] + 1) / 2;
@@ -5504,12 +5846,14 @@ static int lookup_hot(ColiExpertStore *store, ColiExpertKey key,
return -1;
}
if (!slot->slab) {
- slot->slab = malloc((size_t)state->record_bytes);
- if (!slot->slab) {
+ size_t capacity = (size_t)state->record_bytes + 8192u;
+ if (posix_memalign((void **)&slot->slab, 4096, capacity)) {
+ slot->slab = NULL;
pthread_mutex_unlock(&state->mutex);
memset(view, 0, sizeof(*view));
return -1;
}
+ slot->aligned_slab = 1;
state->stats.resident_bytes += state->record_bytes;
}
policy->packed[hot_slot_index(state, slot)] = 0;
@@ -5517,12 +5861,7 @@ static int lookup_hot(ColiExpertStore *store, ColiExpertKey key,
state->active_leases++;
slot->used = ++state->clock;
pthread_mutex_unlock(&state->mutex);
- int read_result = coli_st_read_at(
- state->index, record->shard, record->scale_offset,
- (size_t)record->scale_bytes, slot->slab);
- if (!read_result) read_result = coli_st_read_at(
- state->index, record->shard, record->weight_offset,
- (size_t)record->weight_bytes, slot->slab + record->scale_bytes);
+ int read_result = v4_read_expert_record(state, record, slot);
pthread_mutex_lock(&state->mutex);
if (read_result) {
slot->references = 0; slot->expert = -1;
@@ -5551,14 +5890,74 @@ static void destroy_hot(ColiExpertStore *store) {
if (policy) *link = policy->next;
pthread_mutex_unlock(&hot_policies_mutex);
if (policy) {
+ hot_usage_save(policy, store ? store->state : NULL);
fprintf(stderr, "v4_rows16 packed_slots=%llu\n",
(unsigned long long)policy->packed_slots);
+ fprintf(stderr,
+ "v4_direct reads=%llu flock_reads=%llu fallbacks=%llu "
+ "payload_bytes=%llu\n",
+ (unsigned long long)__atomic_load_n(
+ &v4_direct_reads, __ATOMIC_RELAXED),
+ (unsigned long long)__atomic_load_n(
+ &v4_direct_flock_reads, __ATOMIC_RELAXED),
+ (unsigned long long)__atomic_load_n(
+ &v4_direct_fallbacks, __ATOMIC_RELAXED),
+ (unsigned long long)__atomic_load_n(
+ &v4_direct_payload_bytes, __ATOMIC_RELAXED));
+ free(policy->history_path);
free(policy->packed); free(policy->pins);
free(policy->layer_requests); free(policy->usage); free(policy);
}
destroy(store);
}
+static int hot_prewarm_history(V4HotPolicy *policy,
+ V4ExpertStoreState *state) {
+ if (!policy || !state || !policy->history_seeded ||
+ policy->pin_count < 1) return 0;
+ size_t capacity = (size_t)state->layers * policy->pin_count;
+ ColiExpertKey *keys = malloc(capacity * sizeof(*keys));
+ if (!keys) return -1;
+ size_t count = 0;
+ for (int layer = 0; layer < state->layers; layer++) {
+ hot_repin_locked(policy, state, layer);
+ const int *pins = policy->pins + (size_t)layer * policy->pin_count;
+ for (int rank = 0; rank < policy->pin_count; rank++)
+ if (pins[rank] >= 0)
+ keys[count++] = (ColiExpertKey){layer, pins[rank]};
+ }
+
+ double began = hot_now();
+ int warmed = 0;
+ uint64_t repin_interval = policy->repin_interval;
+ policy->repin_interval = 0;
+ #pragma omp parallel for schedule(dynamic, 1) reduction(+:warmed)
+ for (size_t i = 0; i < count; i++) {
+ ColiExpertView view;
+ if (!lookup_hot(policy->store, keys[i], &view)) {
+ release(policy->store, &view);
+ warmed++;
+ }
+ }
+ policy->repin_interval = repin_interval;
+ free(keys);
+
+ /* Warmup traffic must not dilute request hit-rate telemetry. */
+ pthread_mutex_lock(&state->mutex);
+ uint64_t resident = state->stats.resident_bytes;
+ uint64_t capacity_bytes = state->stats.capacity_bytes;
+ memset(&state->stats, 0, sizeof(state->stats));
+ state->stats.resident_bytes = resident;
+ state->stats.capacity_bytes = capacity_bytes;
+ memset(policy->layer_requests, 0,
+ (size_t)state->layers * sizeof(*policy->layer_requests));
+ pthread_mutex_unlock(&state->mutex);
+ fprintf(stderr,
+ "v4_autopin warmed=%d requested=%zu bytes=%.3fGiB time=%.3fs\n",
+ warmed, count, resident / 1073741824.0, hot_now() - began);
+ return warmed == (int)count ? 0 : -1;
+}
+
#ifndef COLI_V4_ROWS16_STORE_OPEN
#define COLI_V4_ROWS16_STORE_OPEN coli_deepseek_v4_expert_store_open
#endif
@@ -5573,6 +5972,11 @@ int COLI_V4_ROWS16_STORE_OPEN(
options, output, error, error_size);
if (result) return result;
V4ExpertStoreState *state = (*output)->state;
+ int direct_io = state->layers > 0 && state->experts_per_layer > 0 &&
+ coli_st_streaming_direct_available(
+ state->index, state->records[0].shard);
+ fprintf(stderr, "v4_ssd_io mode=%s fallback=buffered-pread\n",
+ direct_io ? "direct-aligned" : "buffered-pread");
int minimum_slots = state->experts_per_layer < 6
? state->experts_per_layer : 6;
int maximum_pins = state->slots_per_layer - minimum_slots;
@@ -5608,9 +6012,20 @@ int COLI_V4_ROWS16_STORE_OPEN(
}
for (size_t i = 0; i < pins; i++) policy->pins[i] = -1;
policy->store = *output; policy->pin_count = pin_count;
+ policy->history_path = hot_history_path(options->model_dir);
policy->repin_interval = options->repin_interval
? options->repin_interval : (uint64_t)minimum_slots;
if (!policy->repin_interval) policy->repin_interval = 1;
+ const char *autopin = getenv("COLI_V4_AUTOPIN");
+ if (!autopin || atoi(autopin) != 0) {
+ policy->history_total = hot_usage_load(policy, state,
+ options->model_dir);
+ /* A tiny history is less predictive than the adaptive LRU. */
+ policy->history_seeded = policy->history_total >= 5000;
+ if (policy->history_seeded)
+ for (int layer = 0; layer < state->layers; layer++)
+ hot_repin_locked(policy, state, layer);
+ }
pthread_mutex_lock(&hot_policies_mutex);
policy->next = hot_policies; hot_policies = policy;
pthread_mutex_unlock(&hot_policies_mutex);
@@ -5619,6 +6034,10 @@ int COLI_V4_ROWS16_STORE_OPEN(
"v4_hot_policy pin_slots_per_layer=%d repin_interval=%llu "
"mode=resident-ram rows16=hot-pins\n", pin_count,
(unsigned long long)policy->repin_interval);
+ const char *prewarm = getenv("COLI_V4_PREWARM");
+ if (policy->history_seeded && prewarm && atoi(prewarm) != 0 &&
+ hot_prewarm_history(policy, state))
+ fprintf(stderr, "v4_autopin warning=partial-warmup; continuing\n");
return 0;
}
#endif /* COLI_V4_UNIT_EXPERT_STORE_HOT_ROWS16 */
@@ -5853,6 +6272,116 @@ void coli_v4_engine_detach_session(ColiV4Engine *engine) {
engine->active_sessions--;
}
+int coli_v4_full_dspark_wanted;
+
+double coli_v4_dspark_cache_gb(void) {
+ const char *setting = getenv("V4_MTP_GB");
+ double value = setting ? atof(setting) : 0.45;
+ if (value < 0.15) value = 0.15;
+ /* The target cache is reduced by the same reservation, but keep a hard
+ * ceiling so a typo cannot turn a lazy drafter into an OOM request. */
+ if (value > 4.0) value = 4.0;
+ return value;
+}
+
+static int v4_dspark_full_wanted(
+ const ColiV4EngineOpenOptions *options) {
+ if (!options || options->no_dspark) return 0;
+ const char *mtp = getenv("V4_MTP");
+ const char *draft = getenv("V4_DRAFT");
+ return mtp && atoi(mtp) != 0 && draft && atoi(draft) > 0;
+}
+
+static uint64_t v4_dspark_full_reserve_bytes(void) {
+ double cache = coli_v4_dspark_cache_gb() * 1e9;
+ /* The released Flash checkpoint has ~0.56 GiB of resident MTP dense,
+ * projection, Markov and norm tensors. Add head/scratch/tap margin. */
+ double total = cache + 768.0 * 1024.0 * 1024.0;
+ return total >= (double)UINT64_MAX ? UINT64_MAX : (uint64_t)total;
+}
+
+/* Lightweight fallback: keep only the official low-rank Markov head resident
+ * when explicitly requested without the complete three-stage drafter. */
+static int v4_dspark_markov_probe(ColiV4Engine *engine,
+ const ColiSafetensorsTensor **w1_out,
+ const ColiSafetensorsTensor **w2_out) {
+ if (!engine || !engine->target_index || !w1_out || !w2_out) return -1;
+ *w1_out = NULL;
+ *w2_out = NULL;
+ char name[160];
+ for (int stage = 0; stage < 16; stage++) {
+ snprintf(name, sizeof(name),
+ "mtp.%d.markov_head.markov_w1.weight", stage);
+ const ColiSafetensorsTensor *w1 = coli_st_find(engine->target_index,
+ name);
+ snprintf(name, sizeof(name),
+ "mtp.%d.markov_head.markov_w2.weight", stage);
+ const ColiSafetensorsTensor *w2 = coli_st_find(engine->target_index,
+ name);
+ if (!w1 || !w2) continue;
+ int rank = engine->config.dspark_markov_rank;
+ if (w1->dtype != COLI_ST_BF16 || w2->dtype != COLI_ST_BF16 ||
+ w1->rank != 2 || w2->rank != 2 || rank < 1 ||
+ w1->shape[0] != engine->config.vocab_size ||
+ w2->shape[0] != engine->config.vocab_size ||
+ w1->shape[1] != rank || w2->shape[1] != rank ||
+ w1->nbytes <= 0 || w2->nbytes <= 0)
+ return -1;
+ engine->dspark.rank = rank;
+ /* A Markov-only draft is intentionally conservative. Two tokens cap
+ * rollback cost on prose where the low-rank bigram is uncertain; a
+ * benchmarker may raise it up to the checkpoint's trained block. */
+ engine->dspark.block_size = 2;
+ const char *block_setting = getenv("COLI_V4_MARKOV_BLOCK");
+ if (block_setting && atoi(block_setting) > 1)
+ engine->dspark.block_size = atoi(block_setting);
+ if (engine->dspark.block_size > engine->config.dspark_block_size)
+ engine->dspark.block_size = engine->config.dspark_block_size;
+ if (engine->dspark.block_size > 8) engine->dspark.block_size = 8;
+ if (engine->dspark.block_size < 2) return -1;
+ engine->dspark.stage = stage;
+ engine->dspark.bytes = (uint64_t)w1->nbytes + (uint64_t)w2->nbytes;
+ *w1_out = w1;
+ *w2_out = w2;
+ return 0;
+ }
+ return -1;
+}
+
+static int v4_dspark_markov_wanted(const ColiV4EngineOpenOptions *options) {
+ if (!options || options->no_dspark) return 0;
+ const char *setting = getenv("COLI_V4_MARKOV_SPEC");
+ /* Opt-in while real-hardware acceptance is being measured. */
+ return setting && atoi(setting) != 0;
+}
+
+static int v4_dspark_markov_load(ColiV4Engine *engine,
+ const ColiSafetensorsTensor *w1,
+ const ColiSafetensorsTensor *w2) {
+ if (!engine || !w1 || !w2) return -1;
+ engine->dspark.markov_w1 = malloc((size_t)w1->nbytes);
+ engine->dspark.markov_w2 = malloc((size_t)w2->nbytes);
+ if (!engine->dspark.markov_w1 || !engine->dspark.markov_w2 ||
+ coli_st_read_tensor(engine->target_index, w1,
+ engine->dspark.markov_w1) ||
+ coli_st_read_tensor(engine->target_index, w2,
+ engine->dspark.markov_w2)) {
+ free(engine->dspark.markov_w2);
+ free(engine->dspark.markov_w1);
+ engine->dspark.markov_w2 = NULL;
+ engine->dspark.markov_w1 = NULL;
+ engine->dspark.bytes = 0;
+ return -1;
+ }
+ engine->dspark.enabled = 1;
+ fprintf(stderr,
+ "v4_dspark mode=markov-draft stage=%d block=%d rank=%d "
+ "resident=%.3fGiB verification=exact-target\n",
+ engine->dspark.stage, engine->dspark.block_size,
+ engine->dspark.rank, engine->dspark.bytes / 1073741824.0);
+ return 0;
+}
+
#ifdef COLI_V4_TEST_HOOKS
ColiV4Session *coli_v4_test_session_bare_create(ColiV4Engine *engine) {
if (!engine) return NULL;
@@ -5914,6 +6443,11 @@ void coli_v4_engine_destroy(ColiV4Engine *engine) {
engine->dense_resident.index = NULL;
engine->dense_resident.total_bytes = 0;
+ free(engine->dspark.markov_w2);
+ free(engine->dspark.markov_w1);
+ engine->dspark.markov_w2 = NULL;
+ engine->dspark.markov_w1 = NULL;
+ engine->dspark.enabled = 0;
free(engine->head_cache.data);
engine->head_cache.data = NULL;
if (engine->owns_experts && engine->experts && engine->experts->ops &&
@@ -5971,6 +6505,26 @@ int coli_v4_engine_open(ColiV4Engine **output,
error_size))
goto fail;
engine->owns_index = 1;
+ const ColiSafetensorsTensor *dspark_w1 = NULL, *dspark_w2 = NULL;
+ int want_full_dspark = v4_dspark_full_wanted(options);
+ int want_dspark = !want_full_dspark && v4_dspark_markov_wanted(options);
+ if (want_dspark && v4_dspark_markov_probe(engine, &dspark_w1,
+ &dspark_w2)) {
+ fprintf(stderr,
+ "v4_dspark warning=compatible-markov-head-not-found; "
+ "continuing-target-only\n");
+ want_dspark = 0;
+ }
+ engine->runtime.dspark_reserve_bytes = want_full_dspark
+ ? v4_dspark_full_reserve_bytes()
+ : (want_dspark ? engine->dspark.bytes : 0);
+ coli_v4_full_dspark_wanted = want_full_dspark;
+ if (want_full_dspark)
+ fprintf(stderr,
+ "v4_dspark mode=full-3stage cache=%.2fGB reserve=%.2fGiB "
+ "load=lazy verification=exact-target\n",
+ coli_v4_dspark_cache_gb(),
+ engine->runtime.dspark_reserve_bytes / 1073741824.0);
#ifdef COLI_V4_TEST_HOOKS
if (coli_v4_test_fail_expert_store_open) {
if (error && error_size)
@@ -5998,6 +6552,10 @@ int coli_v4_engine_open(ColiV4Engine **output,
&engine->experts, error, error_size))
goto fail;
engine->owns_experts = 1;
+ if (want_dspark && v4_dspark_markov_load(engine, dspark_w1, dspark_w2))
+ fprintf(stderr,
+ "v4_dspark warning=cannot-load-markov-head; "
+ "continuing-target-only\n");
engine->summary.dense_resident = engine->runtime.dense_resident;
engine->summary.head_resident = engine->head_cache.data != NULL;
engine->summary.expert_cache_bytes =
@@ -6096,6 +6654,9 @@ int coli_v4_prompt_build(char **output, size_t *output_length,
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#ifdef __AVX2__
+#include <immintrin.h>
+#endif
#include "deepseek_v4_internal.h"
#include "deepseek_v4_internal.h"
@@ -6170,22 +6731,84 @@ static int final_hidden(float *output, const float *state,
return 0;
}
+static float head_bf16_dot(const uint16_t *weight, const float *hidden,
+ int dimension) {
+ float sum = 0.0f;
+ int column = 0;
+#ifdef __AVX2__
+ for (; column + 8 <= dimension; column += 8) {
+ float products[8];
+ __m128i packed = _mm_loadu_si128(
+ (const __m128i *)(weight + column));
+ __m256i bits = _mm256_slli_epi32(
+ _mm256_cvtepu16_epi32(packed), 16);
+ _mm256_storeu_ps(products, _mm256_mul_ps(
+ _mm256_castsi256_ps(bits),
+ _mm256_loadu_ps(hidden + column)));
+ sum += products[0];
+ sum += products[1];
+ sum += products[2];
+ sum += products[3];
+ sum += products[4];
+ sum += products[5];
+ sum += products[6];
+ sum += products[7];
+ }
+#endif
+ for (; column < dimension; column++)
+ sum += coli_bf16_decode(weight[column]) * hidden[column];
+ return sum;
+}
+
static int head_argmax(ColiV4Engine *engine, const float *hidden,
const ColiSafetensorsIndex *index,
const ColiDeepSeekV4Config *config,
int *best_token, float *best_logit) {
const ColiSafetensorsTensor *head = coli_st_find(index, "head.weight");
int d = config->hidden_size, vocab = config->vocab_size;
+ if (!head || head->dtype != COLI_ST_BF16 || d < 1 || vocab < 1)
+ return -1;
+ int shard = coli_st_tensor_shard(index, head);
+ size_t resident_bytes = (size_t)vocab * (size_t)d * sizeof(uint16_t);
+ const uint16_t *resident = coli_v4_head_cache_data(
+ engine, shard, (uint64_t)head->off, resident_bytes);
+
+ /* The normal V4 memory plan keeps the BF16 head resident. Compute all
+ * rows in one OpenMP team directly from that allocation: the old tiled
+ * path copied the complete ~1 GiB head and created ~2,000 teams per token.
+ * Each row retains the same scalar accumulation order and the final scan
+ * retains vocabulary order, so logits/tie-breaking do not change. */
+ if (resident) {
+ float *scores = malloc((size_t)vocab * sizeof(*scores));
+ if (!scores) return -1;
+ #pragma omp parallel for schedule(static)
+ for (int row = 0; row < vocab; row++) {
+ const uint16_t *weight = resident + (size_t)row * d;
+ scores[row] = head_bf16_dot(weight, hidden, d);
+ }
+ int winner = -1;
+ float maximum = -FLT_MAX;
+ for (int row = 0; row < vocab; row++)
+ if (scores[row] > maximum) {
+ maximum = scores[row];
+ winner = row;
+ }
+ free(scores);
+ *best_token = winner;
+ *best_logit = maximum;
+ return winner < 0 ? -1 : 0;
+ }
+
+ /* Low-memory fallback: stream small row tiles exactly as before. */
enum { ROWS = 64 };
uint16_t *raw = malloc((size_t)ROWS * d * sizeof(*raw));
float *scores = malloc((size_t)ROWS * sizeof(*scores));
- if (!head || head->dtype != COLI_ST_BF16 || !raw || !scores) {
+ if (!raw || !scores) {
free(scores); free(raw);
return -1;
}
int winner = -1;
float maximum = -FLT_MAX;
- int shard = coli_st_tensor_shard(index, head);
for (int start = 0; start < vocab; start += ROWS) {
int rows = vocab - start < ROWS ? vocab - start : ROWS;
size_t bytes = (size_t)rows * d * sizeof(*raw);
@@ -6198,11 +6821,8 @@ static int head_argmax(ColiV4Engine *engine, const float *hidden,
}
#pragma omp parallel for
for (int row = 0; row < rows; row++) {
- float sum = 0.0f;
const uint16_t *weight = raw + (size_t)row * d;
- for (int i = 0; i < d; i++)
- sum += coli_bf16_decode(weight[i]) * hidden[i];
- scores[row] = sum;
+ scores[row] = head_bf16_dot(weight, hidden, d);
}
for (int row = 0; row < rows; row++)
if (scores[row] > maximum) {
@@ -6216,6 +6836,83 @@ static int head_argmax(ColiV4Engine *engine, const float *hidden,
return winner < 0 ? -1 : 0;
}
+static int head_argmax_batch(ColiV4Engine *engine, const float *hidden,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config, int batch,
+ int *best_tokens, float *best_logits) {
+ if (!engine || !hidden || !index || !config || batch < 1 ||
+ !best_tokens || !best_logits) return -1;
+ if (batch == 1)
+ return head_argmax(engine, hidden, index, config, best_tokens,
+ best_logits);
+ const ColiSafetensorsTensor *head = coli_st_find(index, "head.weight");
+ int d = config->hidden_size, vocab = config->vocab_size;
+ if (!head || head->dtype != COLI_ST_BF16) return -1;
+ int shard = coli_st_tensor_shard(index, head);
+ const uint16_t *resident = coli_v4_head_cache_data(
+ engine, shard, (uint64_t)head->off,
+ (size_t)vocab * d * sizeof(uint16_t));
+ if (!resident) {
+ for (int item = 0; item < batch; item++)
+ if (head_argmax(engine, hidden + (size_t)item * d, index, config,
+ &best_tokens[item], &best_logits[item])) return -1;
+ return 0;
+ }
+ float *scores = malloc((size_t)vocab * batch * sizeof(*scores));
+ if (!scores) return -1;
+ #pragma omp parallel for schedule(static)
+ for (int row = 0; row < vocab; row++) {
+ const uint16_t *weight = resident + (size_t)row * d;
+ for (int item = 0; item < batch; item++)
+ scores[(size_t)item * vocab + row] = head_bf16_dot(
+ weight, hidden + (size_t)item * d, d);
+ }
+ for (int item = 0; item < batch; item++) {
+ int winner = -1;
+ float maximum = -FLT_MAX;
+ const float *item_scores = scores + (size_t)item * vocab;
+ for (int row = 0; row < vocab; row++)
+ if (item_scores[row] > maximum) {
+ maximum = item_scores[row];
+ winner = row;
+ }
+ best_tokens[item] = winner;
+ best_logits[item] = maximum;
+ }
+ free(scores);
+ return 0;
+}
+
+static int dspark_markov_argmax(const ColiV4Engine *engine, int token,
+ int *best_token) {
+ if (!engine || !engine->dspark.enabled || !best_token || token < 0 ||
+ token >= engine->config.vocab_size) return -1;
+ int rank = engine->dspark.rank, vocab = engine->config.vocab_size;
+ float *embedding = malloc((size_t)rank * sizeof(*embedding));
+ float *scores = malloc((size_t)vocab * sizeof(*scores));
+ if (!embedding || !scores) {
+ free(scores); free(embedding); return -1;
+ }
+ const uint16_t *source = engine->dspark.markov_w1 + (size_t)token * rank;
+ for (int column = 0; column < rank; column++)
+ embedding[column] = coli_bf16_decode(source[column]);
+ #pragma omp parallel for schedule(static)
+ for (int row = 0; row < vocab; row++)
+ scores[row] = head_bf16_dot(
+ engine->dspark.markov_w2 + (size_t)row * rank,
+ embedding, rank);
+ int winner = -1;
+ float maximum = -FLT_MAX;
+ for (int row = 0; row < vocab; row++)
+ if (scores[row] > maximum) {
+ maximum = scores[row];
+ winner = row;
+ }
+ free(scores); free(embedding);
+ *best_token = winner;
+ return winner < 0 ? -1 : 0;
+}
+
static int has_sentence_end(const char *text, int length) {
for (int i = 0; i < length; i++) {
unsigned char value = (unsigned char)text[i];
@@ -6398,6 +7095,55 @@ static int spec_sentence_end(const char *text, int length) {
return 0;
}
+/* DSpark conditions on the target model's final three layer hiddens. Keep a
+ * short ring only when the full drafter is enabled; target-only runs pay no
+ * allocation and no per-layer reduction. */
+#define V4_MAINH_TARGETS 3
+#define V4_MAINH_RING 128
+static float *g_mainh_ring;
+static int64_t g_mainh_abs[V4_MAINH_RING];
+static int g_mainh_dim;
+
+static void v4_mainh_tap(const ColiDeepSeekV4Config *config, int layer_id,
+ const float *row, int64_t position) {
+ if (!coli_v4_full_dspark_wanted || !config || !row || position < 0) return;
+ int first = config->num_hidden_layers - V4_MAINH_TARGETS;
+ int target = layer_id - first;
+ if (target < 0 || target >= V4_MAINH_TARGETS) return;
+ int dimension = config->hidden_size;
+ int copies = config->hc_mult;
+ if (!g_mainh_ring) {
+ g_mainh_ring = calloc(
+ (size_t)V4_MAINH_RING * V4_MAINH_TARGETS * dimension,
+ sizeof(float));
+ if (!g_mainh_ring) return;
+ for (int item = 0; item < V4_MAINH_RING; item++)
+ g_mainh_abs[item] = -1;
+ g_mainh_dim = dimension;
+ }
+ if (g_mainh_dim != dimension) return;
+ int ring = (int)(position % V4_MAINH_RING);
+ float *slot = g_mainh_ring +
+ ((size_t)ring * V4_MAINH_TARGETS + target) * dimension;
+ for (int column = 0; column < dimension; column++) {
+ float sum = 0.0f;
+ for (int copy = 0; copy < copies; copy++)
+ sum += row[(size_t)copy * dimension + column];
+ slot[column] = coli_bf16_round(sum / copies);
+ }
+ if (target == V4_MAINH_TARGETS - 1) g_mainh_abs[ring] = position;
+}
+
+static const float *v4_mainh_get(int64_t position) {
+ if (!g_mainh_ring || position < 0) return NULL;
+ int slot = (int)(position % V4_MAINH_RING);
+ if (g_mainh_abs[slot] != position) return NULL;
+ return g_mainh_ring +
+ (size_t)slot * V4_MAINH_TARGETS * g_mainh_dim;
+}
+
+#include "deepseek_v4_dspark.inc"
+
static int target_batch(ColiV4Engine *engine, float **state_ptr, float **next_ptr,
ColiDeepSeekV4WindowAttentionState **attention,
const ColiSafetensorsIndex *index,
@@ -6432,6 +7178,9 @@ static int target_batch(ColiV4Engine *engine, float **state_ptr, float **next_pt
coli_v4_layer_free(engine, &layer);
if (result) return -1;
float *swap = state; state = next; next = swap;
+ for (int item = 0; item < batch; item++)
+ v4_mainh_tap(config, layer_id, state + (size_t)item * hd,
+ (int64_t)start + item);
}
*state_ptr = state;
*next_ptr = next;
@@ -6456,12 +7205,48 @@ static int target_token(ColiV4Engine *engine, float **state_ptr, float **next_pt
coli_v4_layer_free(engine, &layer);
if (result) return -1;
float *swap = state; state = next; next = swap;
+ v4_mainh_tap(config, layer_id, state, position);
}
*state_ptr = state;
*next_ptr = next;
return 0;
}
+static ColiV4AttentionSnapshot **spec_attention_save(
+ ColiDeepSeekV4WindowAttentionState **attention, int layers) {
+ if (!attention || layers < 1) return NULL;
+ ColiV4AttentionSnapshot **snapshots = calloc(
+ (size_t)layers, sizeof(*snapshots));
+ if (!snapshots) return NULL;
+ for (int layer = 0; layer < layers; layer++)
+ if (coli_v4_attention_snapshot_create(attention[layer],
+ &snapshots[layer])) {
+ for (int item = 0; item < layers; item++)
+ coli_v4_attention_snapshot_destroy(snapshots[item]);
+ free(snapshots);
+ return NULL;
+ }
+ return snapshots;
+}
+
+static int spec_attention_restore(
+ ColiDeepSeekV4WindowAttentionState **attention,
+ ColiV4AttentionSnapshot **snapshots, int layers) {
+ if (!attention || !snapshots) return -1;
+ for (int layer = 0; layer < layers; layer++)
+ if (coli_v4_attention_snapshot_restore(attention[layer],
+ snapshots[layer])) return -1;
+ return 0;
+}
+
+static void spec_attention_free(ColiV4AttentionSnapshot **snapshots,
+ int layers) {
+ if (!snapshots) return;
+ for (int layer = 0; layer < layers; layer++)
+ coli_v4_attention_snapshot_destroy(snapshots[layer]);
+ free(snapshots);
+}
+
#undef spec_print
#undef COLI_V4_GENERATE_HELPERS_ONLY
#undef COLI_V4_GENERATE_MAIN
@@ -6523,7 +7308,7 @@ static void v4_cli_usage(FILE *stream, const char *program) {
" --thinking enable the official V4 thinking prefix\n"
" --raw-prompt bypass the default V4 chat template\n"
" --stop-sentence stop after the first sentence terminator\n"
- " --no-dspark accepted for compatibility; target-only is always greedy\n"
+ " --no-dspark disable verified speculative drafting\n"
" --oracle FILE validate against an oracle JSON fixture\n"
" --teacher-forcing N oracle: compare top-1 on N prompt positions\n"
" --greedy N oracle: compare N greedy continuation tokens\n"
@@ -6862,6 +7647,50 @@ static int session_emit_token(ColiV4Session *session,
return token == 1;
}
+static int v4_context_token(const int *prompt, int prompt_count,
+ const int *generated, int generated_count,
+ int position) {
+ if (position < 0 || position >= prompt_count + generated_count) return -1;
+ return position < prompt_count ? prompt[position]
+ : generated[position - prompt_count];
+}
+
+/* Prompt-lookup drafting costs no model I/O. Match the context tail against
+ * its most recent earlier 3-gram/2-gram occurrence and propose the following
+ * tokens. Every proposal is still checked by the exact target batch below. */
+static int v4_ngram_draft(const int *prompt, int prompt_count,
+ const int *generated, int generated_count,
+ int *output, int maximum) {
+ int count = prompt_count + generated_count;
+ if (!prompt || !generated || !output || maximum < 1) return 0;
+ for (int gram = 3; gram >= 2; gram--) {
+ if (count < gram + 1) continue;
+ int tail = count - gram;
+ for (int start = count - gram - 1; start >= 0; start--) {
+ int matches = 1;
+ for (int item = 0; item < gram; item++)
+ if (v4_context_token(prompt, prompt_count, generated,
+ generated_count, start + item) !=
+ v4_context_token(prompt, prompt_count, generated,
+ generated_count, tail + item)) {
+ matches = 0;
+ break;
+ }
+ if (!matches) continue;
+ int from = start + gram;
+ int take = count - from;
+ if (take > maximum) take = maximum;
+ if (take < 1) break;
+ for (int item = 0; item < take; item++)
+ output[item] = v4_context_token(
+ prompt, prompt_count, generated, generated_count,
+ from + item);
+ return take;
+ }
+ }
+ return 0;
+}
+
int coli_v4_session_generate(ColiV4Session *session,
const char *prompt, size_t prompt_length,
const ColiV4SessionGenerateOptions *options,
@@ -6881,6 +7710,10 @@ int coli_v4_session_generate(ColiV4Session *session,
session->prompt_count = 0;
session->generated_count = 0;
session->prefix_reused = 0;
+ session->spec_attempts = 0;
+ session->spec_drafted = 0;
+ session->spec_accepted = 0;
+ session->spec_disabled = 0;
int max_new = options->max_new_tokens;
if (max_new > session->max_new_tokens_cap)
@@ -6926,9 +7759,11 @@ int coli_v4_session_generate(ColiV4Session *session,
* would have produced.
* ------------------------------------------------------------------- */
int reuse = kv_prefix_reuse(&session->fed, session->prompt_ids, prompt_count);
- if (!reuse)
+ if (!reuse) {
for (int layer = 0; layer < config->num_hidden_layers; layer++)
coli_v4_window_attention_reset(session->attention[layer]);
+ if (coli_v4_full_dspark_wanted) v4_ds_reset_history();
+ }
session->prefix_reused = reuse;
if (reuse && getenv("V4_PREFIX_LOG"))
fprintf(stderr, "[PREFIX] reusing %d of %d prompt tokens\n",
@@ -6977,7 +7812,220 @@ int coli_v4_session_generate(ColiV4Session *session,
generated_count, options->stop_at_sentence);
double first_at = spec_now();
+ int draft_limit = getenv("V4_DRAFT") ? atoi(getenv("V4_DRAFT")) : 0;
+ if (draft_limit < 0) draft_limit = 0;
+ if (draft_limit > 24) draft_limit = 24;
+ int markov_disabled = 0;
+ int mtp_disabled = 0;
+ int full_mtp_ready = 0;
+
while (!done && generated_count < max_new) {
+ int remaining = max_new - generated_count;
+ if (!options->no_dspark && !session->spec_disabled && remaining >= 3) {
+ int inputs[25] = {0}, drafts[24] = {0};
+ int predictions[25] = {0};
+ float logits[25] = {0};
+ int room = remaining - 1; /* verification emits one exact fallback */
+ int proposals = draft_limit < room ? draft_limit : room;
+ const char *ngram_env = getenv("V4_NGRAM");
+ int ngram_enabled = !ngram_env || atoi(ngram_env) != 0;
+ if (proposals > 1 && ngram_enabled)
+ proposals = v4_ngram_draft(
+ session->prompt_ids, prompt_count, generated,
+ generated_count, drafts, proposals);
+ else
+ proposals = 0;
+ int using_full_mtp = 0;
+ int using_markov = 0;
+ int using_ngram = proposals > 1;
+ inputs[0] = current;
+ int draft_ready = proposals > 1;
+ if (!draft_ready && coli_v4_full_dspark_wanted && full_mtp_ready &&
+ !mtp_disabled) {
+ int mtp_min = getenv("V4_MTP_MIN")
+ ? atoi(getenv("V4_MTP_MIN")) : 3;
+ if (mtp_min < 1) mtp_min = 1;
+ int maximum = draft_limit < room ? draft_limit : room;
+ int mtp_max = getenv("V4_MTP_DRAFT")
+ ? atoi(getenv("V4_MTP_DRAFT")) : 3;
+ if (mtp_max < 1) mtp_max = 1;
+ if (maximum > mtp_max) maximum = mtp_max;
+ if (maximum >= mtp_min) {
+ proposals = v4_dspark_draft(
+ engine, index, config, current, last_processed + 1,
+ drafts, maximum);
+ draft_ready = proposals > 0;
+ using_full_mtp = draft_ready;
+ if (!draft_ready) mtp_disabled = 1;
+ }
+ } else if (!draft_ready && engine->dspark.enabled &&
+ !markov_disabled) {
+ proposals = engine->dspark.block_size;
+ if (proposals > room) proposals = room;
+ draft_ready = proposals > 1;
+ using_markov = draft_ready;
+ for (int item = 0; draft_ready && item < proposals; item++) {
+ if (dspark_markov_argmax(engine, inputs[item],
+ &drafts[item])) {
+ draft_ready = 0;
+ break;
+ }
+ if (item + 1 < proposals) inputs[item + 1] = drafts[item];
+ }
+ }
+ if (!draft_ready) {
+ if (using_markov) markov_disabled = 1;
+ } else {
+ int batch = proposals + 1;
+ for (int item = 1; item < batch; item++)
+ inputs[item] = drafts[item - 1];
+ ColiV4AttentionSnapshot **snapshots = spec_attention_save(
+ attention, config->num_hidden_layers);
+ if (!snapshots) {
+ session->spec_disabled = 1;
+ } else {
+ int old_last = last_processed;
+ for (int item = 0; item < batch; item++)
+ if (load_embedding(state + (size_t)item * hd, index,
+ config, inputs[item])) {
+ spec_attention_free(snapshots,
+ config->num_hidden_layers);
+ kv_prefix_taint(&session->fed);
+ if (error && error_size)
+ snprintf(error, error_size,
+ "cannot load speculative embedding");
+ return -1;
+ }
+ if (target_batch(engine, &state, &next, attention, index,
+ config, experts, inputs, old_last + 1,
+ batch, error, error_size)) {
+ (void)spec_attention_restore(
+ attention, snapshots, config->num_hidden_layers);
+ spec_attention_free(snapshots,
+ config->num_hidden_layers);
+ kv_prefix_taint(&session->fed);
+ return -1;
+ }
+ float *batch_hidden = malloc(
+ (size_t)batch * config->hidden_size * sizeof(float));
+ int heads_ok = batch_hidden != NULL;
+ for (int item = 0; heads_ok && item < batch; item++)
+ if (final_hidden(
+ batch_hidden +
+ (size_t)item * config->hidden_size,
+ state + (size_t)item * hd, index, config,
+ error, error_size)) heads_ok = 0;
+ if (heads_ok && head_argmax_batch(
+ engine, batch_hidden, index, config, batch,
+ predictions, logits)) heads_ok = 0;
+ free(batch_hidden);
+ if (!heads_ok) {
+ (void)spec_attention_restore(
+ attention, snapshots, config->num_hidden_layers);
+ spec_attention_free(snapshots,
+ config->num_hidden_layers);
+ kv_prefix_taint(&session->fed);
+ if (error && error_size && !error[0])
+ snprintf(error, error_size,
+ "speculative target head failed");
+ return -1;
+ }
+
+ int accepted = 0;
+ while (accepted < proposals &&
+ predictions[accepted] == drafts[accepted])
+ accepted++;
+ session->spec_attempts++;
+ session->spec_drafted += (uint64_t)proposals;
+ session->spec_accepted += (uint64_t)accepted;
+ if (using_full_mtp)
+ v4_dspark_feedback(proposals, accepted);
+ /* With recurrent compressed attention, a rejected suffix
+ * cannot be truncated in place: the accepted prefix must
+ * be replayed. Real chat measured 10/24 accepted and
+ * 495 s for 14 visible tokens, so stop full MTP after the
+ * first non-perfect block unless explicitly benchmarking
+ * it. Exact prompt lookup remains available. */
+ if (using_full_mtp && accepted < proposals) {
+ const char *keep = getenv("V4_MTP_PARTIAL_KEEP");
+ if (!keep || atoi(keep) == 0) mtp_disabled = 1;
+ }
+ if (using_ngram && accepted < proposals) {
+ const char *keep = getenv("V4_NGRAM_PARTIAL_KEEP");
+ if (!keep || atoi(keep) == 0)
+ session->spec_disabled = 1;
+ }
+ int available_outputs = accepted + 1;
+ int retained = 0;
+ for (int item = 0;
+ item < available_outputs && !done &&
+ generated_count < max_new; item++) {
+ current = predictions[item];
+ current_logit = logits[item];
+ generated[generated_count++] = current;
+ retained++;
+ done = session_emit_token(
+ session, on_token, user_data, current,
+ current_logit, old_last + 1 + item,
+ generated_count, options->stop_at_sentence);
+ }
+
+ /* A mismatch in the first row (or an early callback stop)
+ * leaves unverified draft inputs in the batched KV state.
+ * Restore the exact snapshot and replay only inputs that
+ * really correspond to emitted outputs. */
+ if (retained < batch) {
+ if (spec_attention_restore(
+ attention, snapshots,
+ config->num_hidden_layers)) {
+ spec_attention_free(
+ snapshots, config->num_hidden_layers);
+ kv_prefix_taint(&session->fed);
+ if (error && error_size)
+ snprintf(error, error_size,
+ "cannot restore speculative KV state");
+ return -1;
+ }
+ if (coli_v4_full_dspark_wanted)
+ v4_ds_invalidate_from(old_last + 1);
+ for (int item = 0; item < retained; item++)
+ if (load_embedding(state + (size_t)item * hd,
+ index, config, inputs[item])) {
+ spec_attention_free(
+ snapshots, config->num_hidden_layers);
+ kv_prefix_taint(&session->fed);
+ if (error && error_size)
+ snprintf(error, error_size,
+ "cannot replay speculative input");
+ return -1;
+ }
+ if (retained > 0 && target_batch(
+ engine, &state, &next, attention, index,
+ config, experts, inputs, old_last + 1,
+ retained, error, error_size)) {
+ spec_attention_free(
+ snapshots, config->num_hidden_layers);
+ kv_prefix_taint(&session->fed);
+ return -1;
+ }
+ }
+ spec_attention_free(snapshots,
+ config->num_hidden_layers);
+ if (retained > 0) {
+ kv_prefix_record(&session->fed, inputs, old_last + 1,
+ retained);
+ last_processed = old_last + retained;
+ full_mtp_ready = 1;
+ }
+ session->state = state;
+ session->next = next;
+ const char *keep = getenv("COLI_V4_MARKOV_KEEP");
+ if (using_markov && accepted == 0 &&
+ (!keep || atoi(keep) == 0)) markov_disabled = 1;
+ continue;
+ }
+ }
+ }
int position = last_processed + 1;
if (target_token(engine, &state, &next, attention, index, config, experts,
current, position, error, error_size)) {
@@ -7001,7 +8049,9 @@ int coli_v4_session_generate(ColiV4Session *session,
current_logit, last_processed,
generated_count,
options->stop_at_sentence);
+ full_mtp_ready = 1;
}
+ if (coli_v4_full_dspark_wanted) v4_dspark_report();
double ended = spec_now();
session->state = state;
session->next = next;
@@ -7027,7 +8077,20 @@ int coli_v4_session_generate(ColiV4Session *session,
generated[generated_count - 1] == 1;
stats_out->time_to_first_token_sec = first_at - setup_done;
stats_out->decode_sec = ended - first_at;
+ stats_out->speculative_drafted = session->spec_drafted;
+ stats_out->speculative_accepted = session->spec_accepted;
}
+ if (session->spec_attempts)
+ fprintf(stderr,
+ "v4_dspark attempts=%llu drafted=%llu accepted=%llu "
+ "acceptance=%.1f%% adaptive_disabled=%d\n",
+ (unsigned long long)session->spec_attempts,
+ (unsigned long long)session->spec_drafted,
+ (unsigned long long)session->spec_accepted,
+ session->spec_drafted
+ ? 100.0 * session->spec_accepted / session->spec_drafted
+ : 0.0,
+ session->spec_disabled);
return 0;
}
@@ -7343,7 +8406,7 @@ static void v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,
&(ColiV4SessionGenerateOptions){
.max_new_tokens = request->max_tokens,
.stop_at_sentence = 0,
- .no_dspark = 1,
+ .no_dspark = 0,
},
v4_serve_token, &stream, &stats, error, sizeof(error));
double elapsed = spec_now() - started;
@@ -7390,7 +8453,7 @@ static int v4_serve_main(void) {
.target_model_dir = model_dir,
.context_tokens = context,
.pin_slots_per_layer = -1,
- .no_dspark = 1,
+ .no_dspark = 0,
};
const char *ram = getenv("RAM_GB");
if (ram && atof(ram) > 0.0)
@@ -7595,13 +8658,13 @@ int main(int argc, char **argv) {
fprintf(stderr, "cannot build DeepSeek V4 prompt\n");
goto cleanup;
}
- if (cli.no_dspark)
- fprintf(stderr, "note: --no-dspark is a compatibility no-op; "
- "this build is target-only\n");
- fprintf(stderr, "v4_cli mode=%s memory=%s target_only=1\n",
+ int target_only = !engine->dspark.enabled || cli.no_dspark;
+ if (cli.no_dspark && engine->dspark.enabled)
+ fprintf(stderr, "note: speculative drafting disabled by CLI\n");
+ fprintf(stderr, "v4_cli mode=%s memory=%s target_only=%d\n",
cli.prompt_mode == COLI_V4_PROMPT_RAW ? "raw" :
cli.prompt_mode == COLI_V4_PROMPT_THINKING ? "thinking" : "chat",
- cli.memory_gib > 0.0 ? "limited" : "auto");
+ cli.memory_gib > 0.0 ? "limited" : "auto", target_only);
int session_context = engine->runtime.context_tokens;
ColiV4SessionCreateOptions session_opts = {
@@ -7639,13 +8702,13 @@ int main(int argc, char **argv) {
experts->ops->stats(experts, &stats_end);
fprintf(stderr, "v4_tokens prompt=%d generated=%d total=%d "
"expert_requests=%llu hits=%llu misses=%llu hit_rate=%.3f "
- "bytes=%llu target_only=1\n",
+ "bytes=%llu target_only=%d\n",
gen_stats.prompt_tokens, gen_stats.generated_tokens,
gen_stats.prompt_tokens + gen_stats.generated_tokens,
(unsigned long long)stats_end.requests,
(unsigned long long)stats_end.hits,
(unsigned long long)stats_end.misses, stats_hit_rate(stats_end),
- (unsigned long long)stats_end.bytes_read);
+ (unsigned long long)stats_end.bytes_read, target_only);
fprintf(stderr, "generated_text=");
if (out_len) fwrite(out_text, 1, out_len, stderr);
fprintf(stderr, "\ntiming time_to_first_token=%.3fs after_first=%.3fs\n",
@@ -8003,7 +9066,9 @@ int coli_v4_shared_expert_forward_ref(float *output,
#ifdef COLI_V4_UNIT_EXPERT_STORE
/* ######## deepseek_v4_expert_store.c ######## */
+#ifndef _GNU_SOURCE
#define _GNU_SOURCE
+#endif
#include "deepseek_v4_internal.h"
#include <assert.h>
@@ -8203,11 +9268,13 @@ static int lookup(ColiExpertStore *store, ColiExpertKey key,
}
/* A short read must never expose a partially overwritten old slot. */
slot->expert = -1;
- if (coli_st_read_at(state->index, record->shard, record->scale_offset,
- (size_t)record->scale_bytes, slot->slab) != 0 ||
- coli_st_read_at(state->index, record->shard, record->weight_offset,
- (size_t)record->weight_bytes,
- slot->slab + record->scale_bytes) != 0) {
+ if (coli_st_read_at_streaming(
+ state->index, record->shard, record->scale_offset,
+ (size_t)record->scale_bytes, slot->slab) != 0 ||
+ coli_st_read_at_streaming(
+ state->index, record->shard, record->weight_offset,
+ (size_t)record->weight_bytes,
+ slot->slab + record->scale_bytes) != 0) {
pthread_mutex_unlock(&state->mutex);
memset(view, 0, sizeof(*view));
return -1;
@@ -8683,6 +9750,16 @@ static int required_int(jval *root, const char *name, int *output,
return 0;
}
+static int optional_int(jval *root, const char *name, int *output,
+ char *error, size_t error_size) {
+ jval *value = json_get(root, name);
+ if (!value) return 0;
+ if (json_int_value(value, output) != 0)
+ return set_error(error, error_size,
+ "invalid integer config field: %s", name);
+ return 0;
+}
+
static int required_float(jval *root, const char *name, float *output,
char *error, size_t error_size) {
jval *value = json_get(root, name);
@@ -8752,6 +9829,16 @@ int coli_v4_config_parse(ColiDeepSeekV4Config *config, const char *json,
free(arena);
return -1;
}
+ if (optional_int(root, "dspark_block_size", &config->dspark_block_size,
+ error, error_size) ||
+ optional_int(root, "dspark_noise_token_id",
+ &config->dspark_noise_token_id, error, error_size) ||
+ optional_int(root, "dspark_markov_rank",
+ &config->dspark_markov_rank, error, error_size)) {
+ json_free(root);
+ free(arena);
+ return -1;
+ }
jval *rope = json_get(root, "rope_scaling");
if (!rope || rope->t != J_OBJ ||
required_int(rope, "original_max_position_embeddings",
@@ -8863,6 +9950,9 @@ int coli_v4_config_load(ColiDeepSeekV4Config *config, const char *model_dir,
#include <math.h>
#include <stdlib.h>
#include <string.h>
+#ifdef __AVX2__
+#include <immintrin.h>
+#endif
float coli_e8m0_decode(uint8_t value) {
if (value == 0xff) return NAN;
@@ -9088,7 +10178,8 @@ int coli_fp8_matvec_ref(float *output, const ColiTensorView *weight,
weight->scale_format != COLI_SCALE_F32 ||
!weight->data || !weight->scales || weight->rows < 1 ||
weight->columns < 1 || weight->columns % 128 ||
- weight->block_rows != 128 || weight->block_columns != 128)
+ (weight->block_rows != 128 && weight->block_rows != 8) ||
+ weight->block_columns != 128)
return -1;
size_t rows = (size_t)weight->rows;
size_t columns = (size_t)weight->columns;
@@ -9110,6 +10201,39 @@ int coli_fp8_matvec_ref(float *output, const ColiTensorView *weight,
free(activation_scales);
return -1;
}
+#ifdef __AVX2__
+ if (weight->block_rows == 8) {
+ if (rows % 8) {
+ free(activation_scales); free(activation); return -1;
+ }
+ float fp8[256];
+ for (int code = 0; code < 256; code++)
+ fp8[code] = coli_e4m3fn_decode((uint8_t)code);
+ const uint8_t *data = weight->data;
+ const float *scales = weight->scales;
+ #pragma omp parallel for schedule(static)
+ for (int64_t tile = 0; tile < weight->rows / 8; tile++) {
+ __m256 sum = _mm256_setzero_ps();
+ size_t scale_row = ((size_t)tile * 8) / 128;
+ for (size_t base = 0; base < columns; base += 128) {
+ __m256 scale = _mm256_set1_ps(
+ scales[scale_row * scale_columns + base / 128]);
+ for (size_t offset = 0; offset < 128; offset++) {
+ size_t column = base + offset;
+ __m128i bytes = _mm_loadl_epi64((const __m128i *)(data +
+ ((size_t)tile * columns + column) * 8));
+ __m256i codes = _mm256_cvtepu8_epi32(bytes);
+ __m256 values = _mm256_i32gather_ps(fp8, codes, 4);
+ __m256 x = _mm256_set1_ps(activation[column]);
+ sum = _mm256_add_ps(sum, _mm256_mul_ps(
+ _mm256_mul_ps(x, values), scale));
+ }
+ }
+ _mm256_storeu_ps(output + (size_t)tile * 8, sum);
+ }
+ free(activation_scales); free(activation); return 0;
+ }
+#endif
matmul_fp8(output, activation, weight->data, weight->scales,
1, (int)columns, (int)rows);
free(activation_scales);
@@ -9124,6 +10248,9 @@ int coli_fp8_matvec_ref(float *output, const ColiTensorView *weight,
#include <stdint.h>
#include <stdlib.h>
+#ifdef __AVX2__
+#include <immintrin.h>
+#endif
#include "native_quant.h"
#if defined(__GNUC__)
@@ -9185,7 +10312,8 @@ int coli_fp8_dual_matvec_ref(float *output_a, float *output_b,
a->scale_format != COLI_SCALE_F32 ||
b->scale_format != COLI_SCALE_F32 || !a->data || !b->data ||
!a->scales || !b->scales || a->rows < 1 || a->columns < 1 ||
- a->columns % 128 || a->block_rows != 128 ||
+ a->columns % 128 ||
+ (a->block_rows != 128 && a->block_rows != 8) ||
a->block_columns != 128) return -1;
size_t rows = (size_t)a->rows, columns = (size_t)a->columns;
size_t scale_rows = (rows + 127) / 128, scale_columns = columns / 128;
@@ -9201,6 +10329,47 @@ int coli_fp8_dual_matvec_ref(float *output_a, float *output_b,
input, columns, 128) != 0) {
free(activation_scales); free(activation); return -1;
}
+#ifdef __AVX2__
+ if (a->block_rows == 8) {
+ if (rows % 8) {
+ free(activation_scales); free(activation); return -1;
+ }
+ float fp8[256];
+ for (int code = 0; code < 256; code++)
+ fp8[code] = coli_e4m3fn_decode((uint8_t)code);
+ const uint8_t *data_a = a->data, *data_b = b->data;
+ const float *scales_a = a->scales, *scales_b = b->scales;
+ #pragma omp parallel for schedule(static)
+ for (int64_t tile = 0; tile < a->rows / 8; tile++) {
+ __m256 sum_a = _mm256_setzero_ps();
+ __m256 sum_b = _mm256_setzero_ps();
+ size_t scale_row = ((size_t)tile * 8) / 128;
+ for (size_t base = 0; base < columns; base += 128) {
+ size_t scale_index = scale_row * scale_columns + base / 128;
+ __m256 scale_a = _mm256_set1_ps(scales_a[scale_index]);
+ __m256 scale_b = _mm256_set1_ps(scales_b[scale_index]);
+ for (size_t offset = 0; offset < 128; offset++) {
+ size_t column = base + offset;
+ size_t packed = ((size_t)tile * columns + column) * 8;
+ __m256i codes_a = _mm256_cvtepu8_epi32(_mm_loadl_epi64(
+ (const __m128i *)(data_a + packed)));
+ __m256i codes_b = _mm256_cvtepu8_epi32(_mm_loadl_epi64(
+ (const __m128i *)(data_b + packed)));
+ __m256 values_a = _mm256_i32gather_ps(fp8, codes_a, 4);
+ __m256 values_b = _mm256_i32gather_ps(fp8, codes_b, 4);
+ __m256 x = _mm256_set1_ps(activation[column]);
+ sum_a = _mm256_add_ps(sum_a, _mm256_mul_ps(
+ _mm256_mul_ps(x, values_a), scale_a));
+ sum_b = _mm256_add_ps(sum_b, _mm256_mul_ps(
+ _mm256_mul_ps(x, values_b), scale_b));
+ }
+ }
+ _mm256_storeu_ps(output_a + (size_t)tile * 8, sum_a);
+ _mm256_storeu_ps(output_b + (size_t)tile * 8, sum_b);
+ }
+ free(activation_scales); free(activation); return 0;
+ }
+#endif
matmul_fp8(output_a, activation, a->data, a->scales,
1, (int)columns, (int)rows);
matmul_fp8(output_b, activation, b->data, b->scales,
@@ -9215,6 +10384,9 @@ int coli_fp8_dual_matvec_ref(float *output_a, float *output_b,
#include <stdint.h>
#include <stdlib.h>
+#ifdef __AVX2__
+#include <immintrin.h>
+#endif
#include "native_quant.h"
#if defined(__GNUC__)
@@ -9233,7 +10405,8 @@ int coli_fp8_matmul_batch_ref(float *outputs, const ColiTensorView *weight,
weight->scale_format != COLI_SCALE_F32 ||
!weight->data || !weight->scales || weight->rows < 1 ||
weight->columns < 1 || weight->columns % 128 ||
- weight->block_rows != 128 || weight->block_columns != 128) return -1;
+ (weight->block_rows != 128 && weight->block_rows != 8) ||
+ weight->block_columns != 128) return -1;
size_t rows = (size_t)weight->rows, columns = (size_t)weight->columns;
size_t scale_rows = (rows + 127) / 128;
size_t scale_columns = columns / 128;
@@ -9251,6 +10424,47 @@ int coli_fp8_matmul_batch_ref(float *outputs, const ColiTensorView *weight,
inputs + (size_t)item * columns, columns, 128) != 0) {
free(activation_scales); free(activations); return -1;
}
+#ifdef __AVX2__
+ if (weight->block_rows == 8) {
+ if (rows % 8) {
+ free(activation_scales); free(activations); return -1;
+ }
+ float fp8[256];
+ for (int code = 0; code < 256; code++)
+ fp8[code] = coli_e4m3fn_decode((uint8_t)code);
+ const uint8_t *data = weight->data;
+ const float *scales = weight->scales;
+ #pragma omp parallel for schedule(static)
+ for (int64_t tile = 0; tile < weight->rows / 8; tile++) {
+ __m256 sums[64];
+ for (int item = 0; item < batch; item++)
+ sums[item] = _mm256_setzero_ps();
+ size_t scale_row = ((size_t)tile * 8) / 128;
+ for (size_t base = 0; base < columns; base += 128) {
+ __m256 scale = _mm256_set1_ps(
+ scales[scale_row * scale_columns + base / 128]);
+ for (size_t offset = 0; offset < 128; offset++) {
+ size_t column = base + offset;
+ __m128i bytes = _mm_loadl_epi64((const __m128i *)(data +
+ ((size_t)tile * columns + column) * 8));
+ __m256i codes = _mm256_cvtepu8_epi32(bytes);
+ __m256 values = _mm256_mul_ps(
+ _mm256_i32gather_ps(fp8, codes, 4), scale);
+ for (int item = 0; item < batch; item++) {
+ __m256 x = _mm256_set1_ps(
+ activations[(size_t)item * columns + column]);
+ sums[item] = _mm256_add_ps(
+ sums[item], _mm256_mul_ps(x, values));
+ }
+ }
+ }
+ for (int item = 0; item < batch; item++)
+ _mm256_storeu_ps(outputs + (size_t)item * rows +
+ (size_t)tile * 8, sums[item]);
+ }
+ free(activation_scales); free(activations); return 0;
+ }
+#endif
matmul_fp8(outputs, activations, weight->data, weight->scales,
batch, (int)columns, (int)rows);
free(activation_scales); free(activations); return 0;
@@ -9295,7 +10509,7 @@ int coli_fp4_matmul_batch_ref(float *outputs, const ColiTensorView *weight,
* shared layer has no resident-cache rows16 layout yet. Migrate and delete it
* when that performance API lands upstream. */
-#ifdef __AVX512F__
+#if defined(__AVX512F__) || defined(__AVX2__)
#include <immintrin.h>
#elif defined(__aarch64__)
#include <arm_neon.h>
@@ -9368,6 +10582,46 @@ static __m512 decode_scales_rows16(const unsigned char *scales,
_mm_loadu_si128((const __m128i *)scales));
return _mm512_i32gather_ps(codes, e8, 4);
}
+#elif defined(__AVX2__)
+typedef struct Avx2Rows16Tables {
+ float fp4[16];
+ float e8[256];
+} Avx2Rows16Tables;
+
+static void avx2_rows16_tables(Avx2Rows16Tables *tables) {
+ for (int i = 0; i < 16; i++)
+ tables->fp4[i] = coli_e2m1_decode((uint8_t)i);
+ for (int i = 0; i < 256; i++)
+ tables->e8[i] = coli_e8m0_decode((uint8_t)i);
+}
+
+static inline void avx2_decode_rows16(__m256 values[2],
+ const unsigned char *data, int high,
+ const Avx2Rows16Tables *tables) {
+ __m128i bytes = _mm_loadu_si128((const __m128i *)data);
+ __m256i lo = _mm256_cvtepu8_epi32(bytes);
+ __m256i hi = _mm256_cvtepu8_epi32(_mm_srli_si128(bytes, 8));
+ __m256i mask = _mm256_set1_epi32(15);
+ if (high) {
+ lo = _mm256_srli_epi32(lo, 4);
+ hi = _mm256_srli_epi32(hi, 4);
+ } else {
+ lo = _mm256_and_si256(lo, mask);
+ hi = _mm256_and_si256(hi, mask);
+ }
+ values[0] = _mm256_i32gather_ps(tables->fp4, lo, 4);
+ values[1] = _mm256_i32gather_ps(tables->fp4, hi, 4);
+}
+
+static inline void avx2_decode_scales(__m256 values[2],
+ const unsigned char *codes,
+ const Avx2Rows16Tables *tables) {
+ __m128i bytes = _mm_loadu_si128((const __m128i *)codes);
+ __m256i lo = _mm256_cvtepu8_epi32(bytes);
+ __m256i hi = _mm256_cvtepu8_epi32(_mm_srli_si128(bytes, 8));
+ values[0] = _mm256_i32gather_ps(tables->e8, lo, 4);
+ values[1] = _mm256_i32gather_ps(tables->e8, hi, 4);
+}
#elif defined(__aarch64__)
/* The 16 packed rows map onto four float32x4 accumulators. Per-row work is
* (activation * value) * scale added once per column, columns ascending —
@@ -9470,6 +10724,46 @@ int coli_fp4_matvec_rows16_v10(float *output,
_mm512_storeu_ps(output + (size_t)tile * 16, sum);
}
free(activation_scales); free(activation); return 0;
+#elif defined(__AVX2__)
+ if (!output || !input || !packed_valid(weight)) return -1;
+ size_t columns = (size_t)weight->columns;
+ size_t data_stride = columns / 2, scale_stride = columns / 32;
+ float *activation = malloc(columns * sizeof(*activation));
+ uint8_t *activation_scales = malloc(columns / 128);
+ if (!activation || !activation_scales) {
+ free(activation_scales); free(activation); return -1;
+ }
+ if (coli_fp8_activation_qdq_ref(activation, activation_scales,
+ input, columns, 128)) {
+ free(activation_scales); free(activation); return -1;
+ }
+ Avx2Rows16Tables tables; avx2_rows16_tables(&tables);
+ const unsigned char *data = weight->data, *scales = weight->scales;
+ #pragma omp parallel for schedule(static)
+ for (int64_t tile = 0; tile < weight->rows / 16; tile++) {
+ __m256 sum[2] = {_mm256_setzero_ps(), _mm256_setzero_ps()};
+ for (size_t base = 0; base < columns; base += 32) {
+ __m256 scale[2];
+ avx2_decode_scales(
+ scale,
+ scales + ((size_t)tile * scale_stride + base / 32) * 16,
+ &tables);
+ for (size_t offset = 0; offset < 32; offset++) {
+ size_t column = base + offset;
+ const unsigned char *codes = data +
+ ((size_t)tile * data_stride + column / 2) * 16;
+ __m256 values[2];
+ avx2_decode_rows16(values, codes, column & 1, &tables);
+ __m256 x = _mm256_set1_ps(activation[column]);
+ for (int half = 0; half < 2; half++)
+ sum[half] = _mm256_add_ps(sum[half], _mm256_mul_ps(
+ _mm256_mul_ps(x, values[half]), scale[half]));
+ }
+ }
+ _mm256_storeu_ps(output + (size_t)tile * 16, sum[0]);
+ _mm256_storeu_ps(output + (size_t)tile * 16 + 8, sum[1]);
+ }
+ free(activation_scales); free(activation); return 0;
#else /* __aarch64__ */
if (!output || !input || !packed_valid(weight)) return -1;
size_t columns = (size_t)weight->columns;
@@ -9568,6 +10862,58 @@ int coli_fp4_dual_matvec_rows16_v10(float *output_a, float *output_b,
_mm512_storeu_ps(output_b + (size_t)tile * 16, sum_b);
}
free(activation_scales); free(activation); return 0;
+#elif defined(__AVX2__)
+ if (!output_a || !output_b || !input || !packed_valid(a) ||
+ !packed_valid(b) || a->rows != b->rows ||
+ a->columns != b->columns) return -1;
+ size_t columns = (size_t)a->columns;
+ size_t data_stride = columns / 2, scale_stride = columns / 32;
+ float *activation = malloc(columns * sizeof(*activation));
+ uint8_t *activation_scales = malloc(columns / 128);
+ if (!activation || !activation_scales) {
+ free(activation_scales); free(activation); return -1;
+ }
+ if (coli_fp8_activation_qdq_ref(activation, activation_scales,
+ input, columns, 128)) {
+ free(activation_scales); free(activation); return -1;
+ }
+ Avx2Rows16Tables tables; avx2_rows16_tables(&tables);
+ const unsigned char *data_a = a->data, *data_b = b->data;
+ const unsigned char *scales_a = a->scales, *scales_b = b->scales;
+ #pragma omp parallel for schedule(static)
+ for (int64_t tile = 0; tile < a->rows / 16; tile++) {
+ __m256 sum_a[2] = {_mm256_setzero_ps(), _mm256_setzero_ps()};
+ __m256 sum_b[2] = {_mm256_setzero_ps(), _mm256_setzero_ps()};
+ for (size_t base = 0; base < columns; base += 32) {
+ size_t scale_offset =
+ ((size_t)tile * scale_stride + base / 32) * 16;
+ __m256 scale_a[2], scale_b[2];
+ avx2_decode_scales(scale_a, scales_a + scale_offset, &tables);
+ avx2_decode_scales(scale_b, scales_b + scale_offset, &tables);
+ for (size_t offset = 0; offset < 32; offset++) {
+ size_t column = base + offset;
+ size_t packed_offset =
+ ((size_t)tile * data_stride + column / 2) * 16;
+ __m256 values_a[2], values_b[2];
+ avx2_decode_rows16(values_a, data_a + packed_offset,
+ column & 1, &tables);
+ avx2_decode_rows16(values_b, data_b + packed_offset,
+ column & 1, &tables);
+ __m256 x = _mm256_set1_ps(activation[column]);
+ for (int half = 0; half < 2; half++) {
+ sum_a[half] = _mm256_add_ps(sum_a[half], _mm256_mul_ps(
+ _mm256_mul_ps(x, values_a[half]), scale_a[half]));
+ sum_b[half] = _mm256_add_ps(sum_b[half], _mm256_mul_ps(
+ _mm256_mul_ps(x, values_b[half]), scale_b[half]));
+ }
+ }
+ }
+ _mm256_storeu_ps(output_a + (size_t)tile * 16, sum_a[0]);
+ _mm256_storeu_ps(output_a + (size_t)tile * 16 + 8, sum_a[1]);
+ _mm256_storeu_ps(output_b + (size_t)tile * 16, sum_b[0]);
+ _mm256_storeu_ps(output_b + (size_t)tile * 16 + 8, sum_b[1]);
+ }
+ free(activation_scales); free(activation); return 0;
#else /* __aarch64__ */
if (!output_a || !output_b || !input || !packed_valid(a) ||
!packed_valid(b) || a->rows != b->rows ||
diff --git a/c/deepseek_v4.h b/c/deepseek_v4.h
index 107a0653..f38d8be3 100644
--- a/c/deepseek_v4.h
+++ b/c/deepseek_v4.h
@@ -31,6 +31,9 @@ typedef struct {
int moe_intermediate_size;
int num_hash_layers;
int num_nextn_predict_layers;
+ int dspark_block_size;
+ int dspark_noise_token_id;
+ int dspark_markov_rank;
int hc_mult;
int hc_sinkhorn_iters;
int vocab_size;
@@ -81,7 +84,7 @@ typedef struct {
int context_tokens; /* 0 => 4096 */
int pin_slots_per_layer; /* -1 => auto */
uint64_t repin_interval; /* 0 => auto */
- int no_dspark; /* compatibility no-op in target-only engine */
+ int no_dspark; /* disable speculative draft/verification */
} ColiV4EngineOpenOptions;
typedef struct {
@@ -119,7 +122,7 @@ typedef struct {
typedef struct {
int max_new_tokens; /* required; clamped by session cap */
int stop_at_sentence;
- int no_dspark; /* compatibility no-op in target-only engine */
+ int no_dspark; /* disable speculative draft/verification */
} ColiV4SessionGenerateOptions;
typedef struct {
@@ -128,6 +131,8 @@ typedef struct {
int eos_stopped;
double time_to_first_token_sec;
double decode_sec;
+ uint64_t speculative_drafted;
+ uint64_t speculative_accepted;
} ColiV4SessionGenerateStats;
/* Return non-zero to stop generation. */
diff --git a/c/deepseek_v4_dspark.inc b/c/deepseek_v4_dspark.inc
new file mode 100644
index 00000000..628db053
--- /dev/null
+++ b/c/deepseek_v4_dspark.inc
@@ -0,0 +1,1208 @@
+/* ================= Colibri DSpark-SSD: complete native drafter =============
+ *
+ * This is an original C implementation of the architecture described by the
+ * DeepSeek-V4 checkpoint's own inference/model.py. It intentionally does not
+ * depend on another inference runtime. The invariants that matter here are:
+ *
+ * - all three mtp.* transformer stages are evaluated;
+ * - the five draft positions attend bidirectionally inside their block;
+ * - mtp.2's HC head, Markov logit bias and confidence head are used;
+ * - routed MTP experts live in a separate bounded LRU, so drafting cannot
+ * evict the target model's learned expert cache;
+ * - candidates are never committed without exact target verification.
+ *
+ * DSpark is opt-in while the real-checkpoint A/B gate is being calibrated:
+ * V4_DRAFT=5 V4_MTP=1 V4_MTP_DRAFT=3. V4_MTP_GB is a TOTAL cache budget
+ * across the three stages, and V4_MTP_MISS caps cold expert records loaded by
+ * one draft round. Prompt lookup may still use all five draft slots; the
+ * trained MTP path defaults to three to avoid rejected-suffix replay.
+ * ========================================================================== */
+
+#define V4_DSPARK_STAGES 3
+#define V4_DSPARK_BLOCK 5
+#define V4_DSPARK_WIN 128
+#define V4_DSPARK_RANK 256
+
+typedef struct {
+ int id;
+ uint8_t *slab;
+ uint64_t used;
+} V4DSparkSlot;
+
+typedef struct {
+ int ready;
+ int stage;
+
+ /* FP8 weights stay E4M3. Expand the tiny UE8M0 sidecars once at load:
+ * the current native kernel deliberately has one branch-free F32-scale
+ * hot path (the target model uses the same representation). */
+ uint8_t *wq_a_w, *wq_b_w, *wkv_w, *wo_a_w, *wo_b_w;
+ uint8_t *sh1_w, *sh2_w, *sh3_w;
+ float *wq_a_s, *wq_b_s, *wkv_s, *wo_a_s, *wo_b_s;
+ float *sh1_s, *sh2_s, *sh3_s;
+
+ ColiFloatTensor q_norm, kv_norm, attn_norm, ffn_norm, attn_sink;
+ ColiFloatTensor hc_attn_fn, hc_attn_base, hc_attn_scale;
+ ColiFloatTensor hc_ffn_fn, hc_ffn_base, hc_ffn_scale;
+ ColiFloatTensor gate_w, gate_b;
+
+ /* Context injection KV is independent for every MTP stage. */
+ float *kv; /* [window][head_dim] */
+ int64_t kv_abs[V4_DSPARK_WIN];
+
+ /* FP4 expert cache. One slab is scales+payload for w1,w2,w3. */
+ V4DSparkSlot *slots;
+ int nslots, cap_slots;
+ int64_t slab_bytes;
+ uint64_t clock;
+ const ColiSafetensorsTensor *ew[256][3], *es[256][3];
+ int64_t hits, misses, aborted;
+} V4DSparkStage;
+
+static V4DSparkStage g_v4ds[V4_DSPARK_STAGES];
+
+static struct {
+ int ready;
+ uint8_t *main_proj_w;
+ float *main_proj_s;
+ int64_t main_proj_rows, main_proj_cols;
+ ColiFloatTensor main_norm;
+
+ ColiFloatTensor head_fn, head_base, head_scale, norm;
+ ColiFloatTensor confidence_proj;
+ uint16_t *markov_w1, *markov_w2; /* [vocab][256], BF16 */
+ size_t markov_w1_bytes, markov_w2_bytes;
+
+ float *main_x; /* projected context ring */
+ int64_t main_x_abs[V4_DSPARK_WIN];
+
+ uint64_t rounds, drafted, accepted;
+ double accept_ewma[V4_DSPARK_BLOCK];
+} g_v4ds_core;
+
+static void v4_ds_bf16(float *values, size_t count) {
+ coli_bf16_round_array(values, count);
+}
+
+static void v4_ds_rmsnorm(float *out, const float *in, const float *weight,
+ int n, float eps) {
+ float square = 0.0f;
+ for (int i = 0; i < n; i++) square += in[i] * in[i];
+ float scale = 1.0f / sqrtf(square / n + eps);
+ for (int i = 0; i < n; i++) out[i] = in[i] * scale * weight[i];
+ v4_ds_bf16(out, (size_t)n);
+}
+
+static int v4_ds_read_raw(const ColiSafetensorsIndex *index, const char *name,
+ int dtype, void **output, size_t *bytes) {
+ const ColiSafetensorsTensor *tensor = coli_st_find(index, name);
+ if (!tensor || tensor->dtype != dtype || !tensor->nbytes) return -1;
+ void *data = malloc((size_t)tensor->nbytes);
+ if (!data || coli_st_read_tensor(index, tensor, data)) {
+ free(data);
+ return -1;
+ }
+ *output = data;
+ if (bytes) *bytes = (size_t)tensor->nbytes;
+ return 0;
+}
+
+/* DSpark's resident FP8 matrices use the same column-major rows8 tile consumed
+ * by the AVX2 target kernel. Packing is local to this translation unit because
+ * the split build keeps the target layer loader in a different object. */
+static int v4_ds_pack_rows8(uint8_t *data, int64_t rows, int64_t columns) {
+#ifndef __AVX2__
+ (void)data; (void)rows; (void)columns;
+ return 0;
+#else
+ if (!data || rows < 8 || rows % 8 || columns < 128 || columns % 128)
+ return -1;
+ size_t tile_bytes = (size_t)8 * (size_t)columns;
+ uint8_t *scratch = malloc(tile_bytes);
+ if (!scratch) return -1;
+ for (int64_t tile = 0; tile < rows / 8; tile++) {
+ uint8_t *target = data + (size_t)tile * tile_bytes;
+ memcpy(scratch, target, tile_bytes);
+ for (int64_t column = 0; column < columns; column++)
+ for (int lane = 0; lane < 8; lane++)
+ target[(size_t)column * 8 + lane] =
+ scratch[(size_t)lane * columns + column];
+ }
+ free(scratch);
+ return 1;
+#endif
+}
+
+static int v4_ds_load_fp8(const ColiSafetensorsIndex *index,
+ const char *weight_name, const char *scale_name,
+ uint8_t **weight, float **scale) {
+ const ColiSafetensorsTensor *weight_tensor = coli_st_find(index, weight_name);
+ const ColiSafetensorsTensor *scale_tensor = coli_st_find(index, scale_name);
+ size_t wb = 0;
+ if (!weight_tensor || weight_tensor->dtype != COLI_ST_F8_E4M3 ||
+ weight_tensor->rank != 2 ||
+ weight_tensor->nbytes != weight_tensor->shape[0] *
+ weight_tensor->shape[1] ||
+ !scale_tensor || scale_tensor->rank != 2 ||
+ scale_tensor->dtype != COLI_ST_F8_E8M0 ||
+ scale_tensor->shape[0] != (weight_tensor->shape[0] + 127) / 128 ||
+ scale_tensor->shape[1] != (weight_tensor->shape[1] + 127) / 128 ||
+ scale_tensor->numel < 1)
+ return -1;
+ if (v4_ds_read_raw(index, weight_name, COLI_ST_F8_E4M3,
+ (void **)weight, &wb))
+ return -1;
+ *scale = malloc((size_t)scale_tensor->numel * sizeof(**scale));
+ if (!*scale ||
+ st_read_scale_f32((ColiSafetensorsIndex *)index, scale_name, *scale,
+ scale_tensor->numel, 0) != scale_tensor->numel) {
+ free(*weight);
+ free(*scale);
+ *weight = NULL;
+ *scale = NULL;
+ return -1;
+ }
+ if (v4_ds_pack_rows8(*weight, weight_tensor->shape[0],
+ weight_tensor->shape[1]) < 0) {
+ free(*weight);
+ free(*scale);
+ *weight = NULL;
+ *scale = NULL;
+ return -1;
+ }
+ return wb ? 0 : -1;
+}
+
+static int v4_ds_name(char *out, size_t size, int stage, const char *suffix) {
+ int n = snprintf(out, size, "mtp.%d.%s", stage, suffix);
+ return n < 0 || (size_t)n >= size ? -1 : 0;
+}
+
+static int v4_ds_load_fp8_suffix(const ColiSafetensorsIndex *index, int stage,
+ const char *stem,
+ uint8_t **weight, float **scale) {
+ char wn[160], sn[160];
+ if (snprintf(wn, sizeof(wn), "mtp.%d.%s.weight", stage, stem) >= (int)sizeof(wn) ||
+ snprintf(sn, sizeof(sn), "mtp.%d.%s.scale", stage, stem) >= (int)sizeof(sn))
+ return -1;
+ return v4_ds_load_fp8(index, wn, sn, weight, scale);
+}
+
+static int v4_ds_load_f32_suffix(ColiFloatTensor *output,
+ const ColiSafetensorsIndex *index, int stage,
+ const char *suffix) {
+ char name[160], error[256];
+ if (v4_ds_name(name, sizeof(name), stage, suffix)) return -1;
+ return coli_tensor_load_f32(output, index, name, error, sizeof(error));
+}
+
+static void v4_ds_fp8_view(ColiTensorView *view,
+ const uint8_t *weight, const float *scale,
+ int64_t rows, int64_t columns) {
+ memset(view, 0, sizeof(*view));
+ view->format = COLI_TENSOR_FP8_E4M3_BLOCK;
+ view->scale_format = COLI_SCALE_F32;
+ view->data = weight;
+ view->scales = scale;
+ view->data_bytes = (size_t)(rows * columns);
+ view->scale_bytes = (size_t)(((rows + 127) / 128) *
+ ((columns + 127) / 128)) * sizeof(float);
+ view->rows = rows;
+ view->columns = columns;
+#ifdef __AVX2__
+ view->block_rows = 8;
+#else
+ view->block_rows = 128;
+#endif
+ view->block_columns = 128;
+}
+
+/* Official FP8 linears quantize activations to E4M3/UE8M0 first. Reusing the
+ * engine's native reference kernel is both more faithful and considerably
+ * faster than the scalar, unquantized prototype. */
+static int v4_ds_mm(float *out, const uint8_t *weight, const float *scale,
+ int64_t rows, int64_t columns, const float *input) {
+ ColiTensorView view;
+ v4_ds_fp8_view(&view, weight, scale, rows, columns);
+ if (coli_fp8_matvec_ref(out, &view, input)) return -1;
+ v4_ds_bf16(out, (size_t)rows);
+ return 0;
+}
+
+static int v4_ds_core_load(const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config) {
+ if (g_v4ds_core.ready) return g_v4ds_core.ready > 0;
+ g_v4ds_core.ready = -1;
+ if (config->hc_mult < 1 || config->hc_mult > 16 ||
+ config->n_routed_experts < 1 || config->n_routed_experts > 256 ||
+ config->num_experts_per_tok < 1 ||
+ config->num_experts_per_tok > 16)
+ return 0;
+
+ const ColiSafetensorsTensor *mp =
+ coli_st_find(index, "mtp.0.main_proj.weight");
+ const ColiSafetensorsTensor *ms =
+ coli_st_find(index, "mtp.0.main_proj.scale");
+ if (!mp || !ms || mp->dtype != COLI_ST_F8_E4M3 ||
+ ms->dtype != COLI_ST_F8_E8M0 || mp->rank != 2)
+ return 0;
+ g_v4ds_core.main_proj_rows = config->hidden_size;
+ g_v4ds_core.main_proj_cols =
+ (int64_t)config->hidden_size * V4_MAINH_TARGETS;
+ if (mp->shape[0] != g_v4ds_core.main_proj_rows ||
+ mp->shape[1] != g_v4ds_core.main_proj_cols ||
+ v4_ds_load_fp8(index, "mtp.0.main_proj.weight",
+ "mtp.0.main_proj.scale",
+ &g_v4ds_core.main_proj_w,
+ &g_v4ds_core.main_proj_s) ||
+ v4_ds_load_f32_suffix(&g_v4ds_core.main_norm, index, 0,
+ "main_norm.weight") ||
+ v4_ds_load_f32_suffix(&g_v4ds_core.head_fn, index, 2,
+ "hc_head_fn") ||
+ v4_ds_load_f32_suffix(&g_v4ds_core.head_base, index, 2,
+ "hc_head_base") ||
+ v4_ds_load_f32_suffix(&g_v4ds_core.head_scale, index, 2,
+ "hc_head_scale") ||
+ v4_ds_load_f32_suffix(&g_v4ds_core.norm, index, 2,
+ "norm.weight") ||
+ v4_ds_load_f32_suffix(&g_v4ds_core.confidence_proj, index, 2,
+ "confidence_head.proj.weight") ||
+ v4_ds_read_raw(index, "mtp.2.markov_head.markov_w1.weight",
+ COLI_ST_BF16, (void **)&g_v4ds_core.markov_w1,
+ &g_v4ds_core.markov_w1_bytes) ||
+ v4_ds_read_raw(index, "mtp.2.markov_head.markov_w2.weight",
+ COLI_ST_BF16, (void **)&g_v4ds_core.markov_w2,
+ &g_v4ds_core.markov_w2_bytes))
+ return 0;
+
+ size_t markov_bytes = (size_t)config->vocab_size * V4_DSPARK_RANK *
+ sizeof(uint16_t);
+ if (g_v4ds_core.markov_w1_bytes != markov_bytes ||
+ g_v4ds_core.markov_w2_bytes != markov_bytes)
+ return 0;
+
+ g_v4ds_core.main_x = calloc((size_t)V4_DSPARK_WIN * config->hidden_size,
+ sizeof(float));
+ if (!g_v4ds_core.main_x) return 0;
+ for (int i = 0; i < V4_DSPARK_WIN; i++)
+ g_v4ds_core.main_x_abs[i] = -1;
+ for (int i = 0; i < V4_DSPARK_BLOCK; i++)
+ g_v4ds_core.accept_ewma[i] = 0.5;
+
+ g_v4ds_core.ready = 1;
+ fprintf(stderr,
+ "[MTP] DSpark core ready: stages=%d block=%d main_proj=%lldx%lld "
+ "markov_rank=%d\n",
+ V4_DSPARK_STAGES, V4_DSPARK_BLOCK,
+ (long long)g_v4ds_core.main_proj_rows,
+ (long long)g_v4ds_core.main_proj_cols, V4_DSPARK_RANK);
+ return 1;
+}
+
+static int v4_ds_stage_load(const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config, int stage_id) {
+ V4DSparkStage *stage = &g_v4ds[stage_id];
+ if (stage->ready) return stage->ready > 0;
+ stage->ready = -1;
+ stage->stage = stage_id;
+
+#define V4_DS_LOAD_FP8(field, stem) \
+ do { if (v4_ds_load_fp8_suffix(index, stage_id, stem, \
+ &stage->field##_w, &stage->field##_s)) \
+ return 0; } while (0)
+ V4_DS_LOAD_FP8(wq_a, "attn.wq_a");
+ V4_DS_LOAD_FP8(wq_b, "attn.wq_b");
+ V4_DS_LOAD_FP8(wkv, "attn.wkv");
+ V4_DS_LOAD_FP8(wo_a, "attn.wo_a");
+ V4_DS_LOAD_FP8(wo_b, "attn.wo_b");
+ V4_DS_LOAD_FP8(sh1, "ffn.shared_experts.w1");
+ V4_DS_LOAD_FP8(sh2, "ffn.shared_experts.w2");
+ V4_DS_LOAD_FP8(sh3, "ffn.shared_experts.w3");
+#undef V4_DS_LOAD_FP8
+
+#define V4_DS_LOAD_F32(field, suffix) \
+ do { if (v4_ds_load_f32_suffix(&stage->field, index, stage_id, suffix)) \
+ return 0; } while (0)
+ V4_DS_LOAD_F32(q_norm, "attn.q_norm.weight");
+ V4_DS_LOAD_F32(kv_norm, "attn.kv_norm.weight");
+ V4_DS_LOAD_F32(attn_norm, "attn_norm.weight");
+ V4_DS_LOAD_F32(ffn_norm, "ffn_norm.weight");
+ V4_DS_LOAD_F32(attn_sink, "attn.attn_sink");
+ V4_DS_LOAD_F32(hc_attn_fn, "hc_attn_fn");
+ V4_DS_LOAD_F32(hc_attn_base, "hc_attn_base");
+ V4_DS_LOAD_F32(hc_attn_scale,"hc_attn_scale");
+ V4_DS_LOAD_F32(hc_ffn_fn, "hc_ffn_fn");
+ V4_DS_LOAD_F32(hc_ffn_base, "hc_ffn_base");
+ V4_DS_LOAD_F32(hc_ffn_scale, "hc_ffn_scale");
+ V4_DS_LOAD_F32(gate_w, "ffn.gate.weight");
+ V4_DS_LOAD_F32(gate_b, "ffn.gate.bias");
+#undef V4_DS_LOAD_F32
+
+ static const char *matrices[3] = {"w1", "w2", "w3"};
+ stage->slab_bytes = 0;
+ for (int expert = 0; expert < config->n_routed_experts; expert++) {
+ for (int matrix = 0; matrix < 3; matrix++) {
+ char name[192];
+ snprintf(name, sizeof(name), "mtp.%d.ffn.experts.%d.%s.weight",
+ stage_id, expert, matrices[matrix]);
+ stage->ew[expert][matrix] = coli_st_find(index, name);
+ snprintf(name, sizeof(name), "mtp.%d.ffn.experts.%d.%s.scale",
+ stage_id, expert, matrices[matrix]);
+ stage->es[expert][matrix] = coli_st_find(index, name);
+ if (!stage->ew[expert][matrix] || !stage->es[expert][matrix])
+ return 0;
+ if (!expert)
+ stage->slab_bytes += stage->ew[expert][matrix]->nbytes +
+ stage->es[expert][matrix]->nbytes;
+ }
+ }
+
+ /* V4_MTP_GB is intentionally total, not per stage. */
+ double total_gb = coli_v4_dspark_cache_gb();
+ double stage_bytes = total_gb * 1e9 / V4_DSPARK_STAGES;
+ stage->cap_slots = (int)(stage_bytes / (double)stage->slab_bytes);
+ if (stage->cap_slots < 4) stage->cap_slots = 4;
+ if (stage->cap_slots > config->n_routed_experts)
+ stage->cap_slots = config->n_routed_experts;
+ stage->slots = calloc((size_t)stage->cap_slots, sizeof(*stage->slots));
+ stage->kv = calloc((size_t)V4_DSPARK_WIN * config->head_dim, sizeof(float));
+ if (!stage->slots || !stage->kv) return 0;
+ for (int i = 0; i < stage->cap_slots; i++) stage->slots[i].id = -1;
+ for (int i = 0; i < V4_DSPARK_WIN; i++) stage->kv_abs[i] = -1;
+
+ stage->ready = 1;
+ fprintf(stderr,
+ "[MTP] stage %d ready: expert_cache=%d/%d %.2fGB\n",
+ stage_id, stage->cap_slots, config->n_routed_experts,
+ stage->cap_slots * (double)stage->slab_bytes / 1e9);
+ return 1;
+}
+
+static int v4_ds_load_all(const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config) {
+ if (!v4_ds_core_load(index, config)) return 0;
+ for (int stage = 0; stage < V4_DSPARK_STAGES; stage++)
+ if (!v4_ds_stage_load(index, config, stage)) return 0;
+ return 1;
+}
+
+static int v4_ds_expert(const ColiSafetensorsIndex *index,
+ V4DSparkStage *stage, int expert,
+ ColiExpertView *view, int *miss_budget) {
+ int slot = -1;
+ int lru = 0;
+ for (int i = 0; i < stage->nslots; i++) {
+ if (stage->slots[i].id == expert) {
+ slot = i;
+ stage->hits++;
+ break;
+ }
+ if (stage->slots[i].used < stage->slots[lru].used) lru = i;
+ }
+ if (slot < 0) {
+ if (*miss_budget <= 0) {
+ stage->aborted++;
+ return 0;
+ }
+ (*miss_budget)--;
+ stage->misses++;
+ slot = stage->nslots < stage->cap_slots ? stage->nslots++ : lru;
+ if (!stage->slots[slot].slab) {
+ stage->slots[slot].slab = malloc((size_t)stage->slab_bytes);
+ if (!stage->slots[slot].slab) return 0;
+ }
+ stage->slots[slot].id = -1;
+ uint8_t *cursor = stage->slots[slot].slab;
+ for (int matrix = 0; matrix < 3; matrix++) {
+ const ColiSafetensorsTensor *scale = stage->es[expert][matrix];
+ const ColiSafetensorsTensor *weight = stage->ew[expert][matrix];
+ if (coli_st_read_tensor(index, scale, cursor))
+ return 0;
+ cursor += scale->nbytes;
+ int shard = coli_st_tensor_shard(index, weight);
+ if (shard < 0 || weight->off < 0 ||
+ coli_st_read_at_streaming(
+ index, shard, (uint64_t)weight->off,
+ (size_t)weight->nbytes, cursor))
+ return 0;
+ cursor += weight->nbytes;
+ }
+ stage->slots[slot].id = expert;
+ }
+ stage->slots[slot].used = ++stage->clock;
+
+ uint8_t *cursor = stage->slots[slot].slab;
+ ColiTensorView *views[3] = {&view->gate, &view->down, &view->up};
+ for (int matrix = 0; matrix < 3; matrix++) {
+ const ColiSafetensorsTensor *scale = stage->es[expert][matrix];
+ const ColiSafetensorsTensor *weight = stage->ew[expert][matrix];
+ ColiTensorView *tensor = views[matrix];
+ memset(tensor, 0, sizeof(*tensor));
+ tensor->format = COLI_TENSOR_FP4_NATIVE_BLOCK;
+ tensor->scale_format = COLI_SCALE_UE8M0;
+ tensor->scales = cursor;
+ cursor += scale->nbytes;
+ tensor->data = cursor;
+ cursor += weight->nbytes;
+ tensor->scale_bytes = (size_t)scale->nbytes;
+ tensor->data_bytes = (size_t)weight->nbytes;
+ tensor->rows = weight->shape[0];
+ tensor->columns = weight->shape[1] * 2;
+ tensor->block_rows = 1;
+ tensor->block_columns = 32;
+ }
+ view->lease = NULL;
+ return 1;
+}
+
+/* Hyper-Connections: exact ordering of the checkpoint's split-Sinkhorn. */
+static void v4_ds_sinkhorn(const float *mix, const float *scale,
+ const float *base, int hc, int iterations,
+ float eps, float *pre, float *post, float *comb) {
+ for (int j = 0; j < hc; j++) {
+ float z = mix[j] * scale[0] + base[j];
+ pre[j] = 1.0f / (1.0f + expf(-z)) + eps;
+ z = mix[j + hc] * scale[1] + base[j + hc];
+ post[j] = 2.0f / (1.0f + expf(-z));
+ }
+ for (int j = 0; j < hc; j++) {
+ float maximum = -FLT_MAX;
+ for (int k = 0; k < hc; k++) {
+ float value = mix[2 * hc + j * hc + k] * scale[2] +
+ base[2 * hc + j * hc + k];
+ comb[j * hc + k] = value;
+ if (value > maximum) maximum = value;
+ }
+ float sum = 0.0f;
+ for (int k = 0; k < hc; k++) {
+ comb[j * hc + k] = expf(comb[j * hc + k] - maximum);
+ sum += comb[j * hc + k];
+ }
+ for (int k = 0; k < hc; k++)
+ comb[j * hc + k] = comb[j * hc + k] / sum + eps;
+ }
+
+ /* Initial column normalisation, then (row,column) x (iters-1). */
+ for (int k = 0; k < hc; k++) {
+ float sum = 0.0f;
+ for (int j = 0; j < hc; j++) sum += comb[j * hc + k];
+ for (int j = 0; j < hc; j++) comb[j * hc + k] /= sum + eps;
+ }
+ for (int iteration = 0; iteration < iterations - 1; iteration++) {
+ for (int j = 0; j < hc; j++) {
+ float sum = 0.0f;
+ for (int k = 0; k < hc; k++) sum += comb[j * hc + k];
+ for (int k = 0; k < hc; k++) comb[j * hc + k] /= sum + eps;
+ }
+ for (int k = 0; k < hc; k++) {
+ float sum = 0.0f;
+ for (int j = 0; j < hc; j++) sum += comb[j * hc + k];
+ for (int j = 0; j < hc; j++) comb[j * hc + k] /= sum + eps;
+ }
+ }
+}
+
+static void v4_ds_hc_pre(float *out, const float *state,
+ const ColiFloatTensor *function,
+ const ColiFloatTensor *base,
+ const ColiFloatTensor *scale,
+ const ColiDeepSeekV4Config *config,
+ float *post, float *comb) {
+ int hc = config->hc_mult;
+ int hidden = config->hidden_size;
+ int flat = hc * hidden;
+ int mixes = (2 + hc) * hc;
+ float square = 0.0f;
+ for (int i = 0; i < flat; i++) square += state[i] * state[i];
+ float inverse_rms =
+ 1.0f / sqrtf(square / flat + config->rms_norm_eps);
+ float mixed[64], pre[16];
+ for (int row = 0; row < mixes; row++) {
+ float sum = 0.0f;
+ const float *weight = function->data + (size_t)row * flat;
+ for (int i = 0; i < flat; i++) sum += weight[i] * state[i];
+ mixed[row] = sum * inverse_rms;
+ }
+ v4_ds_sinkhorn(mixed, scale->data, base->data, hc,
+ config->hc_sinkhorn_iters, config->hc_eps,
+ pre, post, comb);
+ for (int i = 0; i < hidden; i++) {
+ float sum = 0.0f;
+ for (int copy = 0; copy < hc; copy++)
+ sum += pre[copy] * state[(size_t)copy * hidden + i];
+ out[i] = coli_bf16_round(sum);
+ }
+}
+
+static void v4_ds_hc_post(float *out, const float *branch,
+ const float *residual, const float *post,
+ const float *comb,
+ const ColiDeepSeekV4Config *config) {
+ int hc = config->hc_mult;
+ int hidden = config->hidden_size;
+ for (int dst = 0; dst < hc; dst++) {
+ for (int i = 0; i < hidden; i++) {
+ float sum = post[dst] * branch[i];
+ for (int src = 0; src < hc; src++)
+ sum += comb[src * hc + dst] *
+ residual[(size_t)src * hidden + i];
+ out[(size_t)dst * hidden + i] = coli_bf16_round(sum);
+ }
+ }
+}
+
+static int v4_ds_main_x(float *out, int64_t position,
+ const ColiDeepSeekV4Config *config) {
+ if (position < 0 || !g_v4ds_core.main_x) return 0;
+ int slot = (int)(position % V4_DSPARK_WIN);
+ float *cached = g_v4ds_core.main_x +
+ (size_t)slot * config->hidden_size;
+ if (g_v4ds_core.main_x_abs[slot] == position) {
+ memcpy(out, cached, (size_t)config->hidden_size * sizeof(float));
+ return 1;
+ }
+ const float *main_hidden = v4_mainh_get(position);
+ if (!main_hidden) return 0;
+ if (v4_ds_mm(cached, g_v4ds_core.main_proj_w,
+ g_v4ds_core.main_proj_s,
+ g_v4ds_core.main_proj_rows,
+ g_v4ds_core.main_proj_cols, main_hidden))
+ return 0;
+ v4_ds_rmsnorm(cached, cached, g_v4ds_core.main_norm.data,
+ config->hidden_size, config->rms_norm_eps);
+ g_v4ds_core.main_x_abs[slot] = position;
+ memcpy(out, cached, (size_t)config->hidden_size * sizeof(float));
+ return 1;
+}
+
+/* Speculative target batches can write hidden taps for a rejected suffix.
+ * Purge them before refeeding the accepted prefix; otherwise future drafts
+ * would condition on a history that never happened. */
+static void v4_ds_invalidate_from(int64_t position) {
+ for (int slot = 0; slot < V4_MAINH_RING; slot++) {
+ if (g_mainh_abs[slot] >= position) g_mainh_abs[slot] = -1;
+ }
+ for (int slot = 0; slot < V4_DSPARK_WIN; slot++) {
+ if (g_v4ds_core.main_x_abs[slot] >= position)
+ g_v4ds_core.main_x_abs[slot] = -1;
+ for (int stage = 0; stage < V4_DSPARK_STAGES; stage++)
+ if (g_v4ds[stage].kv_abs[slot] >= position)
+ g_v4ds[stage].kv_abs[slot] = -1;
+ }
+}
+
+static void v4_ds_reset_history(void) {
+ v4_ds_invalidate_from(0);
+}
+
+/* DSpark QAT keeps the RoPE tail in BF16 and simulates FP8 only for the
+ * non-positional 448 dimensions, in blocks of 64. */
+static int v4_ds_kv_qdq(float *kv, int head_dim, int rope_dim) {
+ int nope = head_dim - rope_dim;
+ uint8_t scales[16];
+ if (nope < 1 || nope > 1024 ||
+ coli_fp8_activation_qdq_ref(kv, scales, kv, (size_t)nope, 64))
+ return -1;
+ v4_ds_bf16(kv, (size_t)head_dim);
+ return 0;
+}
+
+static int v4_ds_backfill_context(const ColiDeepSeekV4Config *config,
+ int64_t position,
+ const float *cosines, const float *sines) {
+ int hidden = config->hidden_size;
+ int head_dim = config->head_dim;
+ int rope_dim = config->qk_rope_head_dim;
+ int pairs = rope_dim / 2;
+ float *main_x = malloc((size_t)hidden * sizeof(float));
+ float *kv = malloc((size_t)head_dim * sizeof(float));
+ if (!main_x || !kv) {
+ free(kv); free(main_x); return -1;
+ }
+ int64_t first = position > V4_DSPARK_WIN
+ ? position - V4_DSPARK_WIN : 0;
+ for (int64_t absolute = first; absolute < position; absolute++) {
+ int slot = (int)(absolute % V4_DSPARK_WIN);
+ int needed = 0;
+ for (int stage = 0; stage < V4_DSPARK_STAGES; stage++)
+ if (g_v4ds[stage].kv_abs[slot] != absolute) needed = 1;
+ if (!needed || !v4_ds_main_x(main_x, absolute, config)) continue;
+ for (int stage_id = 0; stage_id < V4_DSPARK_STAGES; stage_id++) {
+ V4DSparkStage *stage = &g_v4ds[stage_id];
+ if (stage->kv_abs[slot] == absolute) continue;
+ if (v4_ds_mm(kv, stage->wkv_w, stage->wkv_s,
+ head_dim, hidden, main_x)) {
+ free(kv); free(main_x); return -1;
+ }
+ v4_ds_rmsnorm(kv, kv, stage->kv_norm.data, head_dim,
+ config->rms_norm_eps);
+ coli_v4_rope_apply(kv + head_dim - rope_dim, 1, rope_dim,
+ cosines + (size_t)absolute * pairs,
+ sines + (size_t)absolute * pairs, 0);
+ v4_ds_bf16(kv + head_dim - rope_dim, (size_t)rope_dim);
+ if (v4_ds_kv_qdq(kv, head_dim, rope_dim)) {
+ free(kv); free(main_x); return -1;
+ }
+ memcpy(stage->kv + (size_t)slot * head_dim, kv,
+ (size_t)head_dim * sizeof(float));
+ stage->kv_abs[slot] = absolute;
+ }
+ }
+ free(kv);
+ free(main_x);
+ return 0;
+}
+
+static int v4_ds_attention(float *out, V4DSparkStage *stage,
+ const float *input, int64_t query_position,
+ const float *block_kv, int block_count,
+ const ColiDeepSeekV4Config *config,
+ const float *cosines, const float *sines) {
+ int hidden = config->hidden_size;
+ int head_dim = config->head_dim;
+ int rope_dim = config->qk_rope_head_dim;
+ int heads = config->num_attention_heads;
+ int groups = config->o_groups;
+ int q_rank = config->q_lora_rank;
+ int o_rank = config->o_lora_rank;
+ int pairs = rope_dim / 2;
+
+ float *qa = malloc((size_t)q_rank * sizeof(float));
+ float *query = malloc((size_t)heads * head_dim * sizeof(float));
+ float *attended = calloc((size_t)heads * head_dim, sizeof(float));
+ float *oa = malloc((size_t)groups * o_rank * sizeof(float));
+ if (!qa || !query || !attended || !oa) {
+ free(oa); free(attended); free(query); free(qa); return -1;
+ }
+ if (v4_ds_mm(qa, stage->wq_a_w, stage->wq_a_s,
+ q_rank, hidden, input))
+ goto fail;
+ v4_ds_rmsnorm(qa, qa, stage->q_norm.data, q_rank,
+ config->rms_norm_eps);
+ if (v4_ds_mm(query, stage->wq_b_w, stage->wq_b_s,
+ (int64_t)heads * head_dim, q_rank, qa))
+ goto fail;
+ for (int head = 0; head < heads; head++) {
+ float *row = query + (size_t)head * head_dim;
+ float square = 0.0f;
+ for (int d = 0; d < head_dim; d++) square += row[d] * row[d];
+ float inverse = 1.0f /
+ sqrtf(square / head_dim + config->rms_norm_eps);
+ for (int d = 0; d < head_dim; d++)
+ row[d] = coli_bf16_round(row[d] * inverse);
+ coli_v4_rope_apply(row + head_dim - rope_dim, 1, rope_dim,
+ cosines + (size_t)query_position * pairs,
+ sines + (size_t)query_position * pairs, 0);
+ v4_ds_bf16(row + head_dim - rope_dim, (size_t)rope_dim);
+ }
+
+ float softmax_scale = 1.0f / sqrtf((float)head_dim);
+ for (int head = 0; head < heads; head++) {
+ const float *q = query + (size_t)head * head_dim;
+ const float *keys[V4_DSPARK_WIN + V4_DSPARK_BLOCK];
+ float scores[V4_DSPARK_WIN + V4_DSPARK_BLOCK];
+ int count = 0;
+ float maximum = stage->attn_sink.data[head];
+ for (int slot = 0; slot < V4_DSPARK_WIN; slot++) {
+ if (stage->kv_abs[slot] < 0 ||
+ stage->kv_abs[slot] >= query_position)
+ continue;
+ keys[count] = stage->kv + (size_t)slot * head_dim;
+ float dot = 0.0f;
+ for (int d = 0; d < head_dim; d++) dot += q[d] * keys[count][d];
+ scores[count] = dot * softmax_scale;
+ if (scores[count] > maximum) maximum = scores[count];
+ count++;
+ }
+ /* The parallel backbone is deliberately bidirectional in-block. */
+ for (int item = 0; item < block_count; item++) {
+ keys[count] = block_kv + (size_t)item * head_dim;
+ float dot = 0.0f;
+ for (int d = 0; d < head_dim; d++) dot += q[d] * keys[count][d];
+ scores[count] = dot * softmax_scale;
+ if (scores[count] > maximum) maximum = scores[count];
+ count++;
+ }
+ float denominator = expf(stage->attn_sink.data[head] - maximum);
+ for (int item = 0; item < count; item++) {
+ scores[item] = expf(scores[item] - maximum);
+ denominator += scores[item];
+ }
+ float *result = attended + (size_t)head * head_dim;
+ for (int item = 0; item < count; item++) {
+ float probability = scores[item] / denominator;
+ for (int d = 0; d < head_dim; d++)
+ result[d] += probability * keys[item][d];
+ }
+ v4_ds_bf16(result, (size_t)head_dim);
+ coli_v4_rope_apply(result + head_dim - rope_dim, 1, rope_dim,
+ cosines + (size_t)query_position * pairs,
+ sines + (size_t)query_position * pairs, 1);
+ v4_ds_bf16(result + head_dim - rope_dim, (size_t)rope_dim);
+ }
+
+ int heads_per_group = heads / groups;
+ int group_width = heads_per_group * head_dim;
+ int scale_columns = (group_width + 127) / 128;
+ int scale_rows = (o_rank + 127) / 128;
+ for (int group = 0; group < groups; group++) {
+ ColiTensorView view;
+ v4_ds_fp8_view(&view,
+ stage->wo_a_w + (size_t)group * o_rank * group_width,
+ stage->wo_a_s + (size_t)group * scale_rows * scale_columns,
+ o_rank, group_width);
+ if (coli_fp8_matvec_ref(oa + (size_t)group * o_rank, &view,
+ attended + (size_t)group * group_width))
+ goto fail;
+ v4_ds_bf16(oa + (size_t)group * o_rank, (size_t)o_rank);
+ }
+ if (v4_ds_mm(out, stage->wo_b_w, stage->wo_b_s,
+ hidden, (int64_t)groups * o_rank, oa))
+ goto fail;
+ free(oa); free(attended); free(query); free(qa);
+ return 0;
+
+fail:
+ free(oa); free(attended); free(query); free(qa);
+ return -1;
+}
+
+static int v4_ds_moe(float *out, V4DSparkStage *stage,
+ const float *input,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config,
+ int *miss_budget) {
+ int hidden = config->hidden_size;
+ int intermediate = config->moe_intermediate_size;
+ int topk = config->num_experts_per_tok;
+ float weights[16];
+ int experts[16];
+ if (topk > 16 || coli_v4_route(weights, experts, input,
+ stage->gate_w.data, stage->gate_b.data, NULL,
+ config->n_routed_experts, hidden, topk,
+ config->routed_scaling_factor))
+ return -1;
+
+ ColiTensorView shared_gate, shared_down, shared_up;
+ v4_ds_fp8_view(&shared_gate, stage->sh1_w, stage->sh1_s,
+ intermediate, hidden);
+ v4_ds_fp8_view(&shared_down, stage->sh2_w, stage->sh2_s,
+ hidden, intermediate);
+ v4_ds_fp8_view(&shared_up, stage->sh3_w, stage->sh3_s,
+ intermediate, hidden);
+ if (coli_v4_shared_expert_forward_ref(out, &shared_gate, &shared_down,
+ &shared_up, input,
+ config->swiglu_limit))
+ return -1;
+
+ float *branch = malloc((size_t)hidden * sizeof(float));
+ if (!branch) return -1;
+ for (int rank = 0; rank < topk; rank++) {
+ ColiExpertView expert;
+ if (!v4_ds_expert(index, stage, experts[rank], &expert, miss_budget)) {
+ free(branch);
+ return 1; /* bounded cold-start abort */
+ }
+ if (coli_v4_expert_forward_ref(branch, &expert, input,
+ weights[rank],
+ config->swiglu_limit)) {
+ free(branch);
+ return -1;
+ }
+ for (int i = 0; i < hidden; i++) out[i] += branch[i];
+ }
+ v4_ds_bf16(out, (size_t)hidden);
+ free(branch);
+ return 0;
+}
+
+/* One complete MTP transformer stage over all five positions. Attention is
+ * split into prepare/all-KV then query phases so every position can see the
+ * entire block, exactly as the parallel reference does. */
+static int v4_ds_stage_forward(float *state, V4DSparkStage *stage,
+ int64_t first_position,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config,
+ const float *cosines, const float *sines,
+ int *miss_budget) {
+ int hidden = config->hidden_size;
+ int hc = config->hc_mult;
+ int head_dim = config->head_dim;
+ int rope_dim = config->qk_rope_head_dim;
+ int pairs = rope_dim / 2;
+ size_t flat = (size_t)hc * hidden;
+
+ float *residual = malloc((size_t)V4_DSPARK_BLOCK * flat * sizeof(float));
+ float *normed = malloc((size_t)V4_DSPARK_BLOCK * hidden * sizeof(float));
+ float *block_kv = malloc((size_t)V4_DSPARK_BLOCK * head_dim * sizeof(float));
+ float *posts = malloc((size_t)V4_DSPARK_BLOCK * hc * sizeof(float));
+ float *combs = malloc((size_t)V4_DSPARK_BLOCK * hc * hc * sizeof(float));
+ float *branch = malloc((size_t)config->num_attention_heads * head_dim *
+ sizeof(float));
+ if (!residual || !normed || !block_kv || !posts || !combs || !branch) {
+ free(branch); free(combs); free(posts); free(block_kv);
+ free(normed); free(residual); return -1;
+ }
+
+ memcpy(residual, state,
+ (size_t)V4_DSPARK_BLOCK * flat * sizeof(float));
+ for (int item = 0; item < V4_DSPARK_BLOCK; item++) {
+ float *x = state + (size_t)item * flat;
+ float *xn = normed + (size_t)item * hidden;
+ float *post = posts + (size_t)item * hc;
+ float *comb = combs + (size_t)item * hc * hc;
+ v4_ds_hc_pre(xn, x, &stage->hc_attn_fn,
+ &stage->hc_attn_base, &stage->hc_attn_scale,
+ config, post, comb);
+ v4_ds_rmsnorm(xn, xn, stage->attn_norm.data, hidden,
+ config->rms_norm_eps);
+ float *kv = block_kv + (size_t)item * head_dim;
+ if (v4_ds_mm(kv, stage->wkv_w, stage->wkv_s,
+ head_dim, hidden, xn))
+ goto fail;
+ v4_ds_rmsnorm(kv, kv, stage->kv_norm.data, head_dim,
+ config->rms_norm_eps);
+ int64_t absolute = first_position + item;
+ coli_v4_rope_apply(kv + head_dim - rope_dim, 1, rope_dim,
+ cosines + (size_t)absolute * pairs,
+ sines + (size_t)absolute * pairs, 0);
+ v4_ds_bf16(kv + head_dim - rope_dim, (size_t)rope_dim);
+ if (v4_ds_kv_qdq(kv, head_dim, rope_dim)) goto fail;
+ }
+ for (int item = 0; item < V4_DSPARK_BLOCK; item++) {
+ float *x = state + (size_t)item * flat;
+ const float *xn = normed + (size_t)item * hidden;
+ if (v4_ds_attention(branch, stage, xn, first_position + item,
+ block_kv, V4_DSPARK_BLOCK, config,
+ cosines, sines))
+ goto fail;
+ v4_ds_hc_post(x, branch, residual + (size_t)item * flat,
+ posts + (size_t)item * hc,
+ combs + (size_t)item * hc * hc, config);
+ }
+
+ memcpy(residual, state,
+ (size_t)V4_DSPARK_BLOCK * flat * sizeof(float));
+ for (int item = 0; item < V4_DSPARK_BLOCK; item++) {
+ float *x = state + (size_t)item * flat;
+ float *xn = normed + (size_t)item * hidden;
+ float *post = posts + (size_t)item * hc;
+ float *comb = combs + (size_t)item * hc * hc;
+ v4_ds_hc_pre(xn, x, &stage->hc_ffn_fn,
+ &stage->hc_ffn_base, &stage->hc_ffn_scale,
+ config, post, comb);
+ v4_ds_rmsnorm(xn, xn, stage->ffn_norm.data, hidden,
+ config->rms_norm_eps);
+ int result = v4_ds_moe(branch, stage, xn, index, config, miss_budget);
+ if (result < 0) goto fail;
+ if (result > 0) {
+ free(branch); free(combs); free(posts); free(block_kv);
+ free(normed); free(residual); return 1;
+ }
+ v4_ds_hc_post(x, branch, residual + (size_t)item * flat,
+ post, comb, config);
+ }
+
+ free(branch); free(combs); free(posts); free(block_kv);
+ free(normed); free(residual);
+ return 0;
+
+fail:
+ free(branch); free(combs); free(posts); free(block_kv);
+ free(normed); free(residual);
+ return -1;
+}
+
+static int v4_ds_final_hidden(float *out, const float *state,
+ const ColiDeepSeekV4Config *config) {
+ int hidden = config->hidden_size;
+ int hc = config->hc_mult;
+ int flat = hidden * hc;
+ float square = 0.0f;
+ for (int i = 0; i < flat; i++) square += state[i] * state[i];
+ float inverse = 1.0f /
+ sqrtf(square / flat + config->rms_norm_eps);
+ float pre[16];
+ for (int copy = 0; copy < hc; copy++) {
+ float mix = 0.0f;
+ const float *weight = g_v4ds_core.head_fn.data +
+ (size_t)copy * flat;
+ for (int i = 0; i < flat; i++) mix += weight[i] * state[i];
+ float z = mix * inverse * g_v4ds_core.head_scale.data[0] +
+ g_v4ds_core.head_base.data[copy];
+ pre[copy] = 1.0f / (1.0f + expf(-z)) + config->hc_eps;
+ }
+ for (int i = 0; i < hidden; i++) {
+ float sum = 0.0f;
+ for (int copy = 0; copy < hc; copy++)
+ sum += pre[copy] * state[(size_t)copy * hidden + i];
+ out[i] = coli_bf16_round(sum);
+ }
+ v4_ds_rmsnorm(out, out, g_v4ds_core.norm.data, hidden,
+ config->rms_norm_eps);
+ return 0;
+}
+
+static void v4_ds_markov_embed(float out[V4_DSPARK_RANK], int token) {
+ const uint16_t *row = g_v4ds_core.markov_w1 +
+ (size_t)token * V4_DSPARK_RANK;
+ for (int i = 0; i < V4_DSPARK_RANK; i++) out[i] = coli_bf16_decode(row[i]);
+}
+
+/* Compute all backbone logits in one vocabulary sweep so head.weight is read
+ * once for the five positions. The small Markov correction remains serial,
+ * which is the defining semi-autoregressive part of DSpark. */
+static int v4_ds_heads(ColiV4Engine *engine,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config,
+ const float *state, int anchor_token,
+ int *draft, float *confidence) {
+ const ColiSafetensorsTensor *head = coli_st_find(index, "head.weight");
+ if (!head || head->dtype != COLI_ST_BF16) return -1;
+ int head_shard = coli_st_tensor_shard(index, head);
+ if (head_shard < 0 || head->off < 0) return -1;
+ uint64_t head_offset = (uint64_t)head->off;
+ int hidden = config->hidden_size;
+ int vocab = config->vocab_size;
+ int hc = config->hc_mult;
+ size_t flat = (size_t)hc * hidden;
+ float *hiddens = malloc((size_t)V4_DSPARK_BLOCK * hidden * sizeof(float));
+ float *logits = malloc((size_t)V4_DSPARK_BLOCK * vocab * sizeof(float));
+ if (!hiddens || !logits) {
+ free(logits); free(hiddens); return -1;
+ }
+ for (int item = 0; item < V4_DSPARK_BLOCK; item++)
+ if (v4_ds_final_hidden(hiddens + (size_t)item * hidden,
+ state + (size_t)item * flat, config)) {
+ free(logits); free(hiddens); return -1;
+ }
+
+ const uint16_t *resident = coli_v4_head_cache_data(
+ engine, head_shard, head_offset, (size_t)head->nbytes);
+ enum { ROWS = 64 };
+ uint16_t *raw = resident ? NULL :
+ malloc((size_t)ROWS * hidden * sizeof(uint16_t));
+ if (!resident && !raw) {
+ free(logits); free(hiddens); return -1;
+ }
+ for (int start = 0; start < vocab; start += ROWS) {
+ int rows = vocab - start < ROWS ? vocab - start : ROWS;
+ const uint16_t *weights = resident
+ ? resident + (size_t)start * hidden : raw;
+ if (!resident) {
+ if (coli_st_read_at_engine(engine, index, head_shard,
+ head_offset + (uint64_t)start * hidden * sizeof(uint16_t),
+ (size_t)rows * hidden * sizeof(uint16_t), raw)) {
+ free(raw); free(logits); free(hiddens); return -1;
+ }
+ weights = raw;
+ }
+ #pragma omp parallel for
+ for (int row = 0; row < rows; row++) {
+ const uint16_t *weight = weights + (size_t)row * hidden;
+ float sums[V4_DSPARK_BLOCK] = {0};
+ for (int d = 0; d < hidden; d++) {
+ float w = coli_bf16_decode(weight[d]);
+ for (int item = 0; item < V4_DSPARK_BLOCK; item++)
+ sums[item] += w * hiddens[(size_t)item * hidden + d];
+ }
+ for (int item = 0; item < V4_DSPARK_BLOCK; item++)
+ logits[(size_t)item * vocab + start + row] = sums[item];
+ }
+ }
+ free(raw);
+
+ int previous = anchor_token;
+ for (int item = 0; item < V4_DSPARK_BLOCK; item++) {
+ float embedding[V4_DSPARK_RANK];
+ v4_ds_markov_embed(embedding, previous);
+ int winner = -1;
+ float maximum = -FLT_MAX;
+ #pragma omp parallel
+ {
+ int local_winner = -1;
+ float local_maximum = -FLT_MAX;
+ #pragma omp for nowait
+ for (int token = 0; token < vocab; token++) {
+ const uint16_t *weight = g_v4ds_core.markov_w2 +
+ (size_t)token * V4_DSPARK_RANK;
+ float score = logits[(size_t)item * vocab + token];
+ for (int rank = 0; rank < V4_DSPARK_RANK; rank++)
+ score += coli_bf16_decode(weight[rank]) * embedding[rank];
+ if (score > local_maximum) {
+ local_maximum = score;
+ local_winner = token;
+ }
+ }
+ #pragma omp critical
+ if (local_maximum > maximum) {
+ maximum = local_maximum;
+ winner = local_winner;
+ }
+ }
+ if (winner < 0) {
+ free(logits); free(hiddens); return -1;
+ }
+ draft[item] = winner;
+ previous = winner;
+
+ float score = 0.0f;
+ const float *hidden_row = hiddens + (size_t)item * hidden;
+ for (int d = 0; d < hidden; d++)
+ score += g_v4ds_core.confidence_proj.data[d] * hidden_row[d];
+ for (int rank = 0; rank < V4_DSPARK_RANK; rank++)
+ score += g_v4ds_core.confidence_proj.data[hidden + rank] *
+ embedding[rank];
+ confidence[item] = score >= 0.0f
+ ? 1.0f / (1.0f + expf(-score))
+ : expf(score) / (1.0f + expf(score));
+ }
+ free(logits);
+ free(hiddens);
+ return 0;
+}
+
+static int v4_dspark_draft(ColiV4Engine *engine,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config,
+ int anchor_token, int64_t first_position,
+ int *draft, int max_out) {
+ if (!engine || !index || !config || !draft || max_out < 1) return 0;
+ double began = spec_now();
+ if (!v4_ds_load_all(index, config)) return 0;
+ double loaded = spec_now();
+
+ int hidden = config->hidden_size;
+ int hc = config->hc_mult;
+ int rope_dim = config->qk_rope_head_dim;
+ int pairs = rope_dim / 2;
+ int64_t table_rows = first_position + V4_DSPARK_BLOCK;
+ if (table_rows < V4_DSPARK_BLOCK || table_rows > INT_MAX) return 0;
+ float *cosines = malloc((size_t)table_rows * pairs * sizeof(float));
+ float *sines = malloc((size_t)table_rows * pairs * sizeof(float));
+ float *state = malloc((size_t)V4_DSPARK_BLOCK * hc * hidden *
+ sizeof(float));
+ if (!cosines || !sines || !state) {
+ free(state); free(sines); free(cosines); return 0;
+ }
+
+ /* MTP stages have compress_ratio=0, hence base RoPE without YaRN. */
+ if (coli_v4_rope_precompute(cosines, sines, rope_dim, (int)table_rows,
+ 0, config->rope_theta, config->rope_factor,
+ config->rope_beta_fast, config->rope_beta_slow) ||
+ v4_ds_backfill_context(config, first_position, cosines, sines)) {
+ free(state); free(sines); free(cosines); return 0;
+ }
+
+ int noise_token = getenv("V4_MTP_NOISE")
+ ? atoi(getenv("V4_MTP_NOISE")) : 128799;
+ size_t flat = (size_t)hc * hidden;
+ for (int item = 0; item < V4_DSPARK_BLOCK; item++) {
+ int token = item == 0 ? anchor_token : noise_token;
+ if (load_embedding(state + (size_t)item * flat,
+ index, config, token)) {
+ free(state); free(sines); free(cosines); return 0;
+ }
+ }
+
+ /* A cold first block can route to at most 5*6*3 distinct records. The
+ * cache normally makes subsequent rounds zero-I/O; users can lower this
+ * cap to warm gradually on slower or memory-constrained machines. */
+ int miss_budget = getenv("V4_MTP_MISS")
+ ? atoi(getenv("V4_MTP_MISS")) : 96;
+ if (miss_budget < 0) miss_budget = 0;
+ double prepared = spec_now();
+ double stage_seconds[V4_DSPARK_STAGES] = {0};
+ for (int stage_id = 0; stage_id < V4_DSPARK_STAGES; stage_id++) {
+ double stage_began = spec_now();
+ int result = v4_ds_stage_forward(state, &g_v4ds[stage_id], first_position,
+ index, config, cosines, sines,
+ &miss_budget);
+ stage_seconds[stage_id] = spec_now() - stage_began;
+ if (result != 0) {
+ free(state); free(sines); free(cosines); return 0;
+ }
+ }
+
+ int candidates[V4_DSPARK_BLOCK];
+ float confidence[V4_DSPARK_BLOCK];
+ if (v4_ds_heads(engine, index, config, state, anchor_token,
+ candidates, confidence)) {
+ free(state); free(sines); free(cosines); return 0;
+ }
+ free(state);
+ free(sines);
+ free(cosines);
+ double headed = spec_now();
+
+ int count = max_out < V4_DSPARK_BLOCK ? max_out : V4_DSPARK_BLOCK;
+ double threshold = getenv("V4_MTP_CONF")
+ ? atof(getenv("V4_MTP_CONF")) : 0.55;
+ if (threshold > 0.0) {
+ for (int item = 0; item < count; item++) {
+ /* Blend the trained confidence with observed survival on this
+ * exact machine/workload. SSD verification is kept only while
+ * its estimated prefix survival clears the configured price. */
+ double survival = 0.70 * confidence[item] +
+ 0.30 * g_v4ds_core.accept_ewma[item];
+ if (survival < threshold) {
+ count = item;
+ break;
+ }
+ }
+ }
+ if (count < 1) return 0;
+ memcpy(draft, candidates, (size_t)count * sizeof(int));
+ g_v4ds_core.rounds++;
+ g_v4ds_core.drafted += (uint64_t)count;
+
+ if (getenv("V4_MTP_LOG") && atoi(getenv("V4_MTP_LOG"))) {
+ fprintf(stderr,
+ "[MTP] draft n=%d conf=%.3f,%.3f,%.3f,%.3f,%.3f "
+ "cold_left=%d time=load:%.3f prep:%.3f "
+ "stages:%.3f/%.3f/%.3f head:%.3f total:%.3f\n",
+ count, confidence[0], confidence[1], confidence[2],
+ confidence[3], confidence[4], miss_budget,
+ loaded - began, prepared - loaded,
+ stage_seconds[0], stage_seconds[1], stage_seconds[2],
+ headed - prepared - stage_seconds[0] - stage_seconds[1] -
+ stage_seconds[2],
+ headed - began);
+ }
+ return count;
+}
+
+static void v4_dspark_feedback(int drafted, int accepted) {
+ if (drafted < 0) return;
+ if (accepted < 0) accepted = 0;
+ if (accepted > drafted) accepted = drafted;
+ g_v4ds_core.accepted += (uint64_t)accepted;
+ for (int item = 0; item < drafted && item < V4_DSPARK_BLOCK; item++) {
+ double sample = item < accepted ? 1.0 : 0.0;
+ g_v4ds_core.accept_ewma[item] =
+ 0.90 * g_v4ds_core.accept_ewma[item] + 0.10 * sample;
+ }
+}
+
+static void v4_dspark_report(void) {
+ if (!g_v4ds_core.rounds) return;
+ fprintf(stderr,
+ "[MTP] rounds=%llu drafted=%llu accepted=%llu rate=%.1f%% "
+ "ewma=%.2f,%.2f,%.2f,%.2f,%.2f\n",
+ (unsigned long long)g_v4ds_core.rounds,
+ (unsigned long long)g_v4ds_core.drafted,
+ (unsigned long long)g_v4ds_core.accepted,
+ g_v4ds_core.drafted
+ ? 100.0 * g_v4ds_core.accepted / g_v4ds_core.drafted : 0.0,
+ g_v4ds_core.accept_ewma[0], g_v4ds_core.accept_ewma[1],
+ g_v4ds_core.accept_ewma[2], g_v4ds_core.accept_ewma[3],
+ g_v4ds_core.accept_ewma[4]);
+ for (int stage_id = 0; stage_id < V4_DSPARK_STAGES; stage_id++) {
+ V4DSparkStage *stage = &g_v4ds[stage_id];
+ fprintf(stderr,
+ "[MTP] stage=%d cache=%d/%d hits=%lld misses=%lld aborts=%lld\n",
+ stage_id, stage->nslots, stage->cap_slots,
+ (long long)stage->hits, (long long)stage->misses,
+ (long long)stage->aborted);
+ }
+}
diff --git a/c/deepseek_v4_internal.h b/c/deepseek_v4_internal.h
index b62e24da..63b8a8fc 100644
--- a/c/deepseek_v4_internal.h
+++ b/c/deepseek_v4_internal.h
@@ -56,6 +56,13 @@ int coli_st_read_tensor(const ColiSafetensorsIndex *index,
const ColiSafetensorsTensor *tensor, void *destination);
int coli_st_read_at(const ColiSafetensorsIndex *index, int shard,
uint64_t offset, size_t length, void *destination);
+/* Large, transient SSD read: prefer the index's O_DIRECT twin and use an
+ * aligned bounce buffer, falling back to the ordinary buffered path. */
+int coli_st_read_at_streaming(const ColiSafetensorsIndex *index, int shard,
+ uint64_t offset, size_t length,
+ void *destination);
+int coli_st_streaming_direct_available(const ColiSafetensorsIndex *index,
+ int shard);
int coli_st_prefetch_at(const ColiSafetensorsIndex *index, int shard,
uint64_t offset, size_t length);
const char *coli_st_dtype_name(ColiSafetensorsDType dtype);
@@ -71,6 +78,11 @@ void coli_float_tensor_free(ColiFloatTensor *tensor);
typedef struct ColiV4Engine ColiV4Engine;
+/* Runtime-selected full DSpark profile, shared with the separately compiled
+ * generation unit. */
+extern int coli_v4_full_dspark_wanted;
+double coli_v4_dspark_cache_gb(void);
+
/* ==== begin deepseek_v4_math.h ==== */
#ifdef __cplusplus
@@ -135,6 +147,10 @@ typedef struct {
ColiSafetensorsDType dtype;
int rank;
int64_t shape[COLI_ST_MAX_RANK];
+ /* Dense FP8 weights may be transposed inside each 8-row tile after load.
+ * This is an in-memory execution layout only; checkpoint bytes and scales
+ * remain unchanged. */
+ int packed_rows8;
} ColiDeepSeekV4TensorSpec;
typedef struct {
@@ -606,6 +622,7 @@ typedef struct {
uint64_t target_expert_cache_bytes;
int pin_slots_per_layer;
uint64_t repin_interval;
+ uint64_t dspark_reserve_bytes;
} ColiDeepSeekV4RuntimeOptions;
enum { COLI_V4_RESIDENT_MAX_LAYERS = 128 };
@@ -628,6 +645,15 @@ struct ColiV4Engine {
const ColiSafetensorsIndex *index;
uint64_t total_bytes;
} dense_resident;
+ struct {
+ uint16_t *markov_w1;
+ uint16_t *markov_w2;
+ uint64_t bytes;
+ int rank;
+ int block_size;
+ int stage;
+ int enabled;
+ } dspark;
char *owned_target_model_dir;
int owns_experts;
int owns_index;
@@ -664,6 +690,10 @@ struct ColiV4Session {
* from that position instead of re-prefilling it. */
kv_prefix fed;
int prefix_reused; /* reuse length of the request in flight, for stats */
+ uint64_t spec_attempts;
+ uint64_t spec_drafted;
+ uint64_t spec_accepted;
+ int spec_disabled;
};
/* RAM-tiered expert open used by coli_v4_engine_open (replaces ld --wrap). */
diff --git a/c/native_quant_fp4_rows16.h b/c/native_quant_fp4_rows16.h
index dbe855b1..ed73fbf4 100644
--- a/c/native_quant_fp4_rows16.h
+++ b/c/native_quant_fp4_rows16.h
@@ -6,7 +6,7 @@
/* Targets with a rows16 fast kernel: the matvec entry points below return 0
* and the engine may pack hot experts into the 16-row interleaved layout.
* Elsewhere they return -1 and callers keep the flat reference path. */
-#if defined(__AVX512F__) || defined(__aarch64__)
+#if defined(__AVX512F__) || defined(__AVX2__) || defined(__aarch64__)
#define COLI_FP4_ROWS16_KERNEL 1
#endif
diff --git a/c/openai_server.py b/c/openai_server.py
index 28903240..7c312c7a 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -1415,6 +1415,35 @@ def cap_for_arch(arch, cap):
return 0 if arch == "glm" else 8
+def tune_child_env(env, arch):
+ """Apply the engine-local defaults that a direct server launch otherwise misses.
+
+ ``coli chat`` already supplies these values, but users also launch this file
+ directly. Keep setdefault semantics so every explicit operator setting wins.
+ """
+ if arch != "deepseek_v4":
+ return env
+ if not env.get("COLI_NO_OMP_TUNE"):
+ from resource_plan import physical_cpu_count
+ env.setdefault("OMP_NUM_THREADS", str(physical_cpu_count()))
+ env.setdefault("OMP_WAIT_POLICY", "active")
+ env.setdefault("GOMP_SPINCOUNT", "200000")
+ env.setdefault("OMP_DYNAMIC", "FALSE")
+ if sys.platform != "win32":
+ env.setdefault("OMP_PROC_BIND", "close")
+ env.setdefault("OMP_PLACES", "cores")
+ # All speculative paths stay opt-in: partial acceptance requires expensive
+ # recurrent-attention replay on this engine.
+ env.setdefault("V4_DRAFT", "0")
+ env.setdefault("V4_MTP", "0")
+ env.setdefault("V4_MTP_DRAFT", "3")
+ env.setdefault("V4_MTP_GB", "0.45")
+ env.setdefault("V4_MTP_MISS", "96")
+ env.setdefault("V4_MTP_MIN", "3")
+ env.setdefault("V4_MTP_CONF", "0.55")
+ return env
+
+
class Engine:
# cap=None = "not explicitly set": a glm-arch model's engine resolves the
# 0 sentinel (8 historically, 1 on Metal+darwin+fast SSD -- colibri.c
@@ -1423,10 +1452,12 @@ class Engine:
# main() below, so programmatic callers that never pass cap get the same
# auto behavior as the CLI; an explicit int (0 included) is verbatim.
def __init__(self, executable, model, cap=None, max_tokens=1024, env=None, kv_slots=1):
+ arch = model_arch(model)
child_env = dict(env or os.environ, SNAP=str(model), SERVE="1", SERVE_BATCH="1",
NGEN=str(max_tokens), KV_SLOTS=str(kv_slots))
+ tune_child_env(child_env, arch)
self.process = subprocess.Popen(
- [str(executable), str(cap_for_arch(model_arch(model), cap))], env=child_env,
+ [str(executable), str(cap_for_arch(arch, cap))], env=child_env,
stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=0,
)
self.write_lock = threading.Lock()
diff --git a/c/st.h b/c/st.h
index 35cbd6c2..7261adf4 100644
--- a/c/st.h
+++ b/c/st.h
@@ -6,7 +6,9 @@
* - converte sempre in float32 in uscita (BF16/F16/F32 supportati). */
#ifndef ST_H
#define ST_H
+#ifndef _GNU_SOURCE
#define _GNU_SOURCE
+#endif
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
diff --git a/c/tests/test_cli_output.py b/c/tests/test_cli_output.py
index 0c5a3f68..9659e940 100644
--- a/c/tests/test_cli_output.py
+++ b/c/tests/test_cli_output.py
@@ -227,5 +227,54 @@ class BannerModelLineTest(unittest.TestCase):
self.coli.banner("run", True)
+class OmpThreadsForEveryEngineTest(unittest.TestCase):
+ """#805 set OMP_NUM_THREADS from physical cores -- for glm only.
+
+ env_for_engine() forwarded to env_for() when arch was "glm" and built its
+ own environment otherwise, so inkling, kimi_k3, olmoe and deepseek_v4 kept
+ libgomp's nproc default: logical cores, a 2x over-subscription of a
+ memory-bound int4 GEMV on any SMT host.
+ """
+
+ @classmethod
+ def setUpClass(cls):
+ loader = SourceFileLoader("coli_omp_under_test", str(CLI))
+ spec = importlib.util.spec_from_loader(loader.name, loader)
+ cls.coli = importlib.util.module_from_spec(spec)
+ loader.exec_module(cls.coli)
+
+ def args(self):
+ return types.SimpleNamespace(model="/x", ram=None, ctx=None, ngen=None,
+ temp=None, cap=None)
+
+ def test_every_engine_gets_physical_cores(self):
+ with mock.patch("resource_plan.physical_cpu_count", return_value=6):
+ for arch in ("inkling", "kimi", "olmoe", "deepseek_v4"):
+ with self.subTest(arch=arch):
+ env = self.coli.env_for_engine(self.args(), arch)
+ self.assertEqual(env.get("OMP_NUM_THREADS"), "6")
+
+ def test_v4_gets_memory_bound_affinity_defaults(self):
+ with mock.patch.object(self.coli.sys, "platform", "linux"), \
+ mock.patch("resource_plan.physical_cpu_count", return_value=6):
+ env = self.coli.env_for_engine(self.args(), "deepseek_v4")
+ self.assertEqual(env.get("OMP_PROC_BIND"), "close")
+ self.assertEqual(env.get("OMP_PLACES"), "cores")
+ self.assertEqual(env.get("OMP_WAIT_POLICY"), "active")
+ self.assertEqual(env.get("OMP_DYNAMIC"), "FALSE")
+
+ def test_explicit_setting_still_wins(self):
+ with mock.patch.dict(os.environ, {"OMP_NUM_THREADS": "3"}), \
+ mock.patch("resource_plan.physical_cpu_count", return_value=6):
+ env = self.coli.env_for_engine(self.args(), "deepseek_v4")
+ self.assertEqual(env["OMP_NUM_THREADS"], "3")
+
+ def test_kill_switch_is_honoured(self):
+ with mock.patch.dict(os.environ, {"COLI_NO_OMP_TUNE": "1"}, clear=False):
+ os.environ.pop("OMP_NUM_THREADS", None)
+ env = self.coli.env_for_engine(self.args(), "deepseek_v4")
+ self.assertNotIn("OMP_NUM_THREADS", env)
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/c/tests/test_deepseek_v4.c b/c/tests/test_deepseek_v4.c
index b29690e5..dd87043c 100644
--- a/c/tests/test_deepseek_v4.c
+++ b/c/tests/test_deepseek_v4.c
@@ -84,6 +84,8 @@ static int test_config(int argc, char **argv) {
"\"index_topk\":8,\"n_routed_experts\":8,\"num_experts_per_tok\":2,"
"\"n_shared_experts\":1,\"moe_intermediate_size\":32,"
"\"num_hash_layers\":1,\"num_nextn_predict_layers\":1,"
+ "\"dspark_block_size\":5,\"dspark_noise_token_id\":255,"
+ "\"dspark_markov_rank\":16,"
"\"hc_mult\":4,\"hc_sinkhorn_iters\":5,\"vocab_size\":256,"
"\"max_position_embeddings\":4096,\"rms_norm_eps\":1e-6,"
"\"hc_eps\":1e-6,\"routed_scaling_factor\":1.5,\"swiglu_limit\":10,"
@@ -100,6 +102,8 @@ static int test_config(int argc, char **argv) {
}
if (config.hidden_size != 128 || config.num_hidden_layers != 3 ||
config.n_routed_experts != 8 || config.num_experts_per_tok != 2 ||
+ config.dspark_block_size != 5 || config.dspark_noise_token_id != 255 ||
+ config.dspark_markov_rank != 16 ||
config.compress_ratio_count != 4 || config.compress_ratios[1] != 4 ||
config.compress_ratios[2] != 128 || config.rope_factor != 4.0f)
return 1;
diff --git a/c/tests/test_deepseek_v4_dspark_source.py b/c/tests/test_deepseek_v4_dspark_source.py
new file mode 100644
index 00000000..57fb4663
--- /dev/null
+++ b/c/tests/test_deepseek_v4_dspark_source.py
@@ -0,0 +1,79 @@
+import pathlib
+import unittest
+
+
+ROOT = pathlib.Path(__file__).resolve().parents[1]
+
+
+class DeepSeekV4DSparkSourceTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.engine = (ROOT / "deepseek_v4.c").read_text(encoding="utf-8")
+ cls.drafter = (ROOT / "deepseek_v4_dspark.inc").read_text(
+ encoding="utf-8"
+ )
+ cls.launcher = (ROOT / "coli").read_text(encoding="utf-8")
+
+ def test_complete_three_stage_drafter_is_built(self):
+ self.assertEqual(self.drafter.count("static int v4_dspark_draft("), 1)
+ self.assertIn("#define V4_DSPARK_STAGES 3", self.drafter)
+ self.assertIn('"mtp.2.markov_head.markov_w1.weight"', self.drafter)
+ self.assertIn('"confidence_head.proj.weight"', self.drafter)
+
+ def test_drafter_is_lazy_and_budgeted_before_target_cache(self):
+ reserve = self.engine.index("engine->runtime.dspark_reserve_bytes =")
+ store = self.engine.index("coli_v4_expert_store_open_planned(", reserve)
+ self.assertLess(reserve, store)
+ self.assertIn("load=lazy verification=exact-target", self.engine)
+ self.assertIn("double total_gb = coli_v4_dspark_cache_gb()", self.drafter)
+
+ def test_exact_target_hidden_precedes_full_mtp(self):
+ self.assertIn("int full_mtp_ready = 0;", self.engine)
+ target = self.engine.index("if (target_token(engine, &state, &next")
+ ready = self.engine.index("full_mtp_ready = 1;", target)
+ draft = self.engine.index("proposals = v4_dspark_draft(")
+ self.assertLess(draft, target)
+ self.assertLess(target, ready)
+
+ def test_rejected_suffix_invalidates_hidden_taps(self):
+ restore = self.engine.index("if (spec_attention_restore(")
+ invalidate = self.engine.index("v4_ds_invalidate_from(old_last + 1)")
+ replay = self.engine.index("if (retained > 0 && target_batch(", invalidate)
+ self.assertLess(restore, invalidate)
+ self.assertLess(invalidate, replay)
+
+ def test_prompt_lookup_and_full_mtp_are_exactly_verified(self):
+ self.assertIn("static int v4_ngram_draft(", self.engine)
+ self.assertIn('getenv("V4_NGRAM")', self.engine)
+ self.assertIn("int batch = proposals + 1;", self.engine)
+ self.assertIn("predictions[accepted] == drafts[accepted]", self.engine)
+
+ def test_full_mtp_expands_ue8m0_scales_once(self):
+ self.assertIn("float *wq_a_s, *wq_b_s", self.drafter)
+ self.assertIn("st_read_scale_f32(", self.drafter)
+ self.assertIn("view->scale_format = COLI_SCALE_F32", self.drafter)
+ self.assertIn("* sizeof(float);", self.drafter)
+ self.assertIn("static int v4_ds_pack_rows8(", self.drafter)
+ self.assertIn("view->block_rows = 8", self.drafter)
+
+ def test_chat_keeps_all_speculation_opt_in(self):
+ for setting in (
+ 'env.setdefault("V4_DRAFT", "0")',
+ 'env.setdefault("V4_MTP", "0")',
+ 'env.setdefault("V4_MTP_DRAFT", "3")',
+ 'env.setdefault("V4_MTP_GB", "0.45")',
+ ):
+ self.assertIn(setting, self.launcher)
+ self.assertIn(".no_dspark = 0,", self.engine)
+ self.assertIn("accepted < proposals", self.engine)
+ self.assertIn('getenv("V4_MTP_PARTIAL_KEEP")', self.engine)
+ self.assertIn('getenv("V4_NGRAM_PARTIAL_KEEP")', self.engine)
+
+ def test_target_ssd_reader_uses_aligned_final_slots(self):
+ self.assertIn("v4_read_expert_record(state, record, slot)", self.engine)
+ self.assertIn("posix_memalign((void **)&slot->slab, 4096", self.engine)
+ self.assertIn("payload_bytes=%llu", self.engine)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tests/test_deepseek_v4_tiny.py b/c/tests/test_deepseek_v4_tiny.py
index 9fae7c2c..e07de41b 100644
--- a/c/tests/test_deepseek_v4_tiny.py
+++ b/c/tests/test_deepseek_v4_tiny.py
@@ -161,8 +161,20 @@ def check_session(
f"session {name}: length mismatch: "
f"prompt={prompt_count} generated={generated_count}"
)
- if compatibility_flag and "compatibility no-op" not in result.stderr:
- raise AssertionError("--no-dspark compatibility notice was not emitted")
+ if compatibility_flag:
+ # --no-dspark used to be a no-op that only printed a notice, because the
+ # engine was target-only and had nothing to disable. It now disables
+ # verified speculative drafting for real, so asserting the old notice
+ # would require the engine to keep claiming it does nothing.
+ #
+ # What matters either way is that the run stays token-exact, which the
+ # checks above already prove, and that the flag actually suppresses
+ # speculation: with drafting off no attempt is ever made, so the
+ # counters the engine prints at exit must be zero.
+ attempts = re.search(r"v4_dspark attempts=(\d+)", result.stderr)
+ if attempts and int(attempts.group(1)) != 0:
+ raise AssertionError(
+ f"--no-dspark still attempted {attempts.group(1)} speculations")
print(f"PASS target session {name}: exact IDs and exact length")
return actual["full_ids"]
diff --git a/c/tests/test_native_quant.c b/c/tests/test_native_quant.c
index 480e87da..8e7ed4c7 100644
--- a/c/tests/test_native_quant.c
+++ b/c/tests/test_native_quant.c
@@ -97,6 +97,32 @@ int main(void) {
return 1;
for (int i = 0; i < 128; i++)
if (!close_enough(fp8_output[i], 128.0f)) return 1;
+
+ /* Flash layout: eight rows are interleaved by column. It must be
+ * numerically equivalent to the guarded row-major reference path while
+ * exercising different values in every lane. */
+ uint8_t row_major[8 * 128], rows8[8 * 128];
+ float row_major_out[8], rows8_out[8];
+ for (int row = 0; row < 8; row++)
+ for (int column = 0; column < 128; column++)
+ row_major[row * 128 + column] =
+ (uint8_t)(0x30 + ((row + column) & 15));
+ for (int column = 0; column < 128; column++)
+ for (int row = 0; row < 8; row++)
+ rows8[column * 8 + row] = row_major[row * 128 + column];
+ ColiTensorView row_major_view = {
+ COLI_TENSOR_FP8_E4M3_BLOCK, COLI_SCALE_F32,
+ row_major, fp8_scales, sizeof(row_major), sizeof(fp8_scales),
+ 8, 128, 128, 128
+ };
+ ColiTensorView rows8_view = row_major_view;
+ rows8_view.data = rows8;
+ rows8_view.block_rows = 8;
+ if (coli_fp8_matvec_ref(row_major_out, &row_major_view, input) != 0 ||
+ coli_fp8_matvec_ref(rows8_out, &rows8_view, input) != 0)
+ return 1;
+ for (int row = 0; row < 8; row++)
+ if (!close_enough(rows8_out[row], row_major_out[row])) return 1;
puts("native quant tests: ok");
return 0;
}
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index 5465499d..796bba02 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -20,7 +20,7 @@ from openai_server import (APIError, APIHandler, APIServer, ClientCancelled,
_engine_error, cap_for_arch, conversation_cache_slot, model_arch,
generation_options, parse_tool_calls, read_engine_turn,
render_chat, render_chat_kimi, serve,
- split_thinking_reply, stop_policy)
+ split_thinking_reply, stop_policy, tune_child_env)
class FakeEngine:
@@ -670,8 +670,22 @@ class CapSentinelShimTest(unittest.TestCase):
self.assertEqual(model_arch(self._model("glm_moe_dsa")), "glm")
self.assertEqual(model_arch(self._model("inkling")), "inkling")
self.assertEqual(model_arch(self._model("kimi_k3")), "kimi")
+ self.assertEqual(model_arch(self._model("deepseek_v4")), "deepseek_v4")
self.assertEqual(model_arch("/nonexistent"), "glm")
+ def test_direct_v4_server_gets_bounded_dspark_defaults(self):
+ env = {"V4_MTP_CONF": "0.7"}
+ with patch("resource_plan.physical_cpu_count", return_value=6), \
+ patch("openai_server.sys.platform", "linux"):
+ tune_child_env(env, "deepseek_v4")
+ self.assertEqual(env["OMP_NUM_THREADS"], "6")
+ self.assertEqual(env["OMP_PROC_BIND"], "close")
+ self.assertEqual(env["V4_DRAFT"], "0")
+ self.assertEqual(env["V4_MTP"], "0")
+ self.assertEqual(env["V4_MTP_DRAFT"], "3")
+ self.assertEqual(env["V4_MTP_GB"], "0.45")
+ self.assertEqual(env["V4_MTP_CONF"], "0.7") # explicit override wins
+
class HTTPTest(unittest.TestCase):
@classmethod
diff --git a/c/tok.h b/c/tok.h
index 66853f68..54c5c699 100644
--- a/c/tok.h
+++ b/c/tok.h
@@ -12,7 +12,9 @@
*/
#ifndef TOK_H
#define TOK_H
+#ifndef _GNU_SOURCE
#define _GNU_SOURCE
+#endif
#include <stdio.h>
#include <stdlib.h>
#include <string.h>