Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

perf(dsv4): share expert cache across prefill layers

JustVugga0ba858237fa
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.c
index 8d39135d..03b7eddb 100644
--- a/c/deepseek_v4.c
+++ b/c/deepseek_v4.c
@@ -7019,6 +7019,7 @@ typedef struct {
} V4ExpertRecord;
typedef struct {
+ int owner_layer;
int expert;
int loading_expert;
unsigned references;
@@ -7042,9 +7043,15 @@ typedef struct {
* mutates slot identity must maintain this index */
int *allocated_per_layer; /* shared by the embedded base and hot StoreOps:
* every slab allocation must advance this cursor */
+ int *resident_per_layer; /* logical owners, including in-flight loads */
int *lru_head; /* every recency update in either StoreOps must touch the
* intrusive list while state->mutex is held */
int *lru_tail;
+ /* Batched prefill is layer-major. While pool_layer >= 0, misses for that
+ * layer may borrow any physical cache partition; decode leaves this at -1
+ * and keeps the ordinary per-layer allocation policy. */
+ int pool_layer;
+ int pool_reserve_per_layer;
uint64_t clock;
unsigned active_leases;
ColiExpertStoreStats stats;
@@ -7143,6 +7150,11 @@ static V4ExpertSlot *layer_slots(V4ExpertStoreState *state, int layer) {
return state->slots + (size_t)layer * state->slots_per_layer;
}
+static int slot_partition(const V4ExpertStoreState *state,
+ const V4ExpertSlot *slot) {
+ return (int)(slot - state->slots) / state->slots_per_layer;
+}
+
#ifdef COLI_V4_TEST_HOOKS
uint64_t coli_v4_test_expert_victim_probes;
#define V4_COUNT_VICTIM_PROBE() (coli_v4_test_expert_victim_probes++)
@@ -7152,34 +7164,36 @@ uint64_t coli_v4_test_expert_victim_probes;
/* All LRU helpers are called with state->mutex held. Slabs are allocated in
* slot order and never freed before store destruction, so the next empty slot
- * is O(1). Allocated slots form one exact LRU list per layer: moving a slot to
- * the tail is equivalent to assigning the old monotonically increasing used
- * timestamp, without searching the whole cache for its minimum on a miss. */
-static void touch_lru_slot(V4ExpertStoreState *state, int layer,
- V4ExpertSlot *slot) {
+ * is O(1). Allocated slots form one exact LRU list per physical partition:
+ * moving a slot to the tail is equivalent to assigning the old monotonically
+ * increasing used timestamp, without searching the whole cache for its
+ * minimum on a miss. A pooled prefill may change a slot's logical owner, but
+ * its physical partition (and therefore list membership) never changes. */
+static void touch_lru_slot(V4ExpertStoreState *state, V4ExpertSlot *slot) {
int index = (int)(slot - state->slots);
- int layer_begin = layer * state->slots_per_layer;
- int layer_end = layer_begin + state->slots_per_layer;
- assert(index >= layer_begin && index < layer_end);
+ int partition = slot_partition(state, slot);
+ int partition_begin = partition * state->slots_per_layer;
+ int partition_end = partition_begin + state->slots_per_layer;
+ assert(index >= partition_begin && index < partition_end);
if (slot->in_lru) {
if (slot->lru_previous >= 0)
state->slots[slot->lru_previous].lru_next = slot->lru_next;
else
- state->lru_head[layer] = slot->lru_next;
+ state->lru_head[partition] = slot->lru_next;
if (slot->lru_next >= 0)
state->slots[slot->lru_next].lru_previous = slot->lru_previous;
else
- state->lru_tail[layer] = slot->lru_previous;
+ state->lru_tail[partition] = slot->lru_previous;
} else {
slot->in_lru = 1;
}
- slot->lru_previous = state->lru_tail[layer];
+ slot->lru_previous = state->lru_tail[partition];
slot->lru_next = -1;
- if (state->lru_tail[layer] >= 0)
- state->slots[state->lru_tail[layer]].lru_next = index;
+ if (state->lru_tail[partition] >= 0)
+ state->slots[state->lru_tail[partition]].lru_next = index;
else
- state->lru_head[layer] = index;
- state->lru_tail[layer] = index;
+ state->lru_head[partition] = index;
+ state->lru_tail[partition] = index;
}
static V4ExpertSlot *next_empty_slot(V4ExpertStoreState *state, int layer) {
@@ -7193,7 +7207,20 @@ static void mark_slot_allocated(V4ExpertStoreState *state, int layer,
assert(slot == layer_slots(state, layer) +
state->allocated_per_layer[layer]);
state->allocated_per_layer[layer]++;
- touch_lru_slot(state, layer, slot);
+ touch_lru_slot(state, slot);
+}
+
+/* Prefer the current layer's physical partition, then borrow unused slabs
+ * from the remaining partitions. The scan is bounded by model depth, not by
+ * cache capacity, and advances only when a whole partition is full. */
+static V4ExpertSlot *next_pooled_empty_slot(V4ExpertStoreState *state,
+ int layer) {
+ for (int offset = 0; offset < state->layers; offset++) {
+ int partition = (layer + offset) % state->layers;
+ V4ExpertSlot *slot = next_empty_slot(state, partition);
+ if (slot) return slot;
+ }
+ return NULL;
}
/* Only referenced (in-use/in-flight) slots can precede the victim. Therefore
@@ -7223,12 +7250,11 @@ static V4ExpertSlot *indexed_expert_slot(V4ExpertStoreState *state,
ColiExpertKey key) {
int slot_index = state->slot_by_expert[expert_cell(
state, key.layer, key.expert)];
- size_t layer_begin = (size_t)key.layer * state->slots_per_layer;
- size_t layer_end = layer_begin + state->slots_per_layer;
- if (slot_index < 0 || (size_t)slot_index < layer_begin ||
- (size_t)slot_index >= layer_end) return NULL;
+ size_t slot_count = (size_t)state->layers * state->slots_per_layer;
+ if (slot_index < 0 || (size_t)slot_index >= slot_count) return NULL;
V4ExpertSlot *slot = &state->slots[slot_index];
- if (slot->expert != key.expert && slot->loading_expert != key.expert)
+ if (slot->owner_layer != key.layer ||
+ (slot->expert != key.expert && slot->loading_expert != key.expert))
return NULL;
return slot;
}
@@ -7237,19 +7263,30 @@ static void unindex_expert_slot(V4ExpertStoreState *state,
V4ExpertSlot *slot) {
int slot_index = (int)(slot - state->slots);
int experts[2] = {slot->expert, slot->loading_expert};
+ int removed = 0;
for (int i = 0; i < 2; i++) {
int expert = experts[i];
- if (expert < 0 || expert >= state->experts_per_layer) continue;
- int layer = slot_index / state->slots_per_layer;
- int *entry = &state->slot_by_expert[expert_cell(state, layer, expert)];
- if (*entry == slot_index) *entry = -1;
+ if (slot->owner_layer < 0 || slot->owner_layer >= state->layers ||
+ expert < 0 || expert >= state->experts_per_layer) continue;
+ int *entry = &state->slot_by_expert[expert_cell(
+ state, slot->owner_layer, expert)];
+ if (*entry == slot_index) {
+ *entry = -1;
+ removed = 1;
+ }
}
+ if (removed && state->resident_per_layer[slot->owner_layer] > 0)
+ state->resident_per_layer[slot->owner_layer]--;
}
static void index_expert_slot(V4ExpertStoreState *state, int layer,
int expert, V4ExpertSlot *slot) {
- state->slot_by_expert[expert_cell(state, layer, expert)] =
- (int)(slot - state->slots);
+ int slot_index = (int)(slot - state->slots);
+ int *entry = &state->slot_by_expert[expert_cell(state, layer, expert)];
+ assert(*entry < 0 || *entry == slot_index);
+ if (*entry != slot_index) state->resident_per_layer[layer]++;
+ slot->owner_layer = layer;
+ *entry = slot_index;
}
static void fill_tensor_view(ColiTensorView *view,
@@ -7346,7 +7383,7 @@ static int lookup(ColiExpertStore *store, ColiExpertKey key,
slot->references++;
state->active_leases++;
slot->used = ++state->clock;
- touch_lru_slot(state, key.layer, slot);
+ touch_lru_slot(state, slot);
if (state->ehit) {
size_t expert_index =
(size_t)key.layer * state->experts_per_layer + key.expert;
@@ -7459,6 +7496,7 @@ static void destroy(ColiExpertStore *store) {
free(state->slots);
free(state->slot_by_expert);
free(state->allocated_per_layer);
+ free(state->resident_per_layer);
free(state->lru_head);
free(state->lru_tail);
free(state->ehit);
@@ -7495,6 +7533,7 @@ int coli_deepseek_v4_expert_store_open(
}
state->layers = options->layers;
state->experts_per_layer = options->experts_per_layer;
+ state->pool_layer = -1;
if (coli_st_index_open(&state->index, options->model_dir, error, error_size) != 0)
goto fail;
/* DUAL-SSD: register COLI_MODEL_MIRROR copies and derive the read split
@@ -7526,6 +7565,7 @@ int coli_deepseek_v4_expert_store_open(
((uint64_t)state->layers * state->record_bytes));
int minimum_slots = state->experts_per_layer < 6
? state->experts_per_layer : 6;
+ state->pool_reserve_per_layer = minimum_slots;
if (state->slots_per_layer < minimum_slots) {
set_error(error, error_size,
"cache budget cannot hold %d active experts per layer "
@@ -7540,10 +7580,12 @@ int coli_deepseek_v4_expert_store_open(
sizeof(*state->slots));
state->allocated_per_layer = calloc(
(size_t)state->layers, sizeof(*state->allocated_per_layer));
+ state->resident_per_layer = calloc(
+ (size_t)state->layers, sizeof(*state->resident_per_layer));
state->lru_head = malloc((size_t)state->layers * sizeof(*state->lru_head));
state->lru_tail = malloc((size_t)state->layers * sizeof(*state->lru_tail));
- if (!state->slots || !state->allocated_per_layer || !state->lru_head ||
- !state->lru_tail) {
+ if (!state->slots || !state->allocated_per_layer ||
+ !state->resident_per_layer || !state->lru_head || !state->lru_tail) {
set_error(error, error_size, "out of memory creating expert cache slots");
goto fail;
}
@@ -7552,6 +7594,7 @@ int coli_deepseek_v4_expert_store_open(
state->lru_tail[layer] = -1;
}
for (int i = 0; i < state->layers * state->slots_per_layer; i++) {
+ state->slots[i].owner_layer = -1;
state->slots[i].expert = -1;
state->slots[i].loading_expert = -1;
state->slots[i].lru_previous = -1;
@@ -7581,6 +7624,7 @@ fail:
free(state->records);
free(state->slot_by_expert);
free(state->allocated_per_layer);
+ free(state->resident_per_layer);
free(state->lru_head);
free(state->lru_tail);
free(state->ehit);
@@ -7705,7 +7749,7 @@ static V4HotPolicy *hot_find(ColiExpertStore *store) {
#endif
static int hot_is_pinned(const V4HotPolicy *policy, int layer, int expert) {
- if (!policy || policy->pin_count < 1) return 0;
+ if (!policy || policy->pin_count < 1 || layer < 0 || expert < 0) return 0;
int active = policy->pin_count;
#if COLI_V4_PIN_RAMP_REQUESTS > 0
if (!policy->history_seeded && active > 4) {
@@ -8000,7 +8044,7 @@ static void hot_repin_locked(V4HotPolicy *policy, V4ExpertStoreState *state,
state, (ColiExpertKey){layer, expert});
if (slot && slot->slab && slot->expert == expert) {
slot->used = ++state->clock;
- touch_lru_slot(state, layer, slot);
+ touch_lru_slot(state, slot);
}
}
uint64_t decay_interval = policy->repin_interval * 64;
@@ -8024,13 +8068,65 @@ static V4ExpertSlot *hot_oldest_victim(V4ExpertStoreState *state,
V4_COUNT_VICTIM_PROBE();
if (!slot->references) {
if (!fallback) fallback = slot;
- if (!hot_is_pinned(policy, layer, slot->expert)) return slot;
+ if (!hot_is_pinned(policy, slot->owner_layer, slot->expert))
+ return slot;
}
index = slot->lru_next;
}
return fallback;
}
+/* Pool-mode victim selection merges the heads of the physical-partition LRU
+ * lists. It skips only active leases, the bounded pin set and the fixed
+ * six-expert decode reserve, so adding passive RAM slots does not make a miss
+ * more expensive. Pinned experts survive unless every available slot is
+ * pinned, matching the ordinary cache policy. */
+static V4ExpertSlot *hot_oldest_pool_victim(
+ V4ExpertStoreState *state, const V4HotPolicy *policy) {
+ V4ExpertSlot *best = NULL;
+ V4ExpertSlot *reserved = NULL;
+ V4ExpertSlot *pinned = NULL;
+ V4ExpertSlot *last_resort = NULL;
+ for (int partition = 0; partition < state->layers; partition++) {
+ int index = state->lru_head[partition];
+ while (index >= 0) {
+ V4ExpertSlot *slot = &state->slots[index];
+ V4_COUNT_VICTIM_PROBE();
+ if (!slot->references) {
+ int is_pinned = hot_is_pinned(
+ policy, slot->owner_layer, slot->expert);
+ int may_lend = slot->owner_layer < 0 ||
+ slot->owner_layer == state->pool_layer ||
+ state->resident_per_layer[slot->owner_layer] >
+ state->pool_reserve_per_layer;
+ if (!last_resort || slot->used < last_resort->used)
+ last_resort = slot;
+ if (!is_pinned &&
+ (!reserved || slot->used < reserved->used))
+ reserved = slot;
+ if (may_lend) {
+ if (is_pinned) {
+ if (!pinned || slot->used < pinned->used)
+ pinned = slot;
+ } else {
+ if (!best || slot->used < best->used) best = slot;
+ break;
+ }
+ }
+ }
+ index = slot->lru_next;
+ }
+ }
+ /* Preserve both the decode reserve and pins whenever possible. If active
+ * leases or an extremely small cache make that impossible, correctness
+ * wins: relax the reserve before evicting a pin, then use the exact oldest
+ * unreferenced slot as the final fallback. */
+ if (best) return best;
+ if (reserved) return reserved;
+ if (pinned) return pinned;
+ return last_resort;
+}
+
static int lookup_hot(ColiExpertStore *store, ColiExpertKey key,
ColiExpertView *view) {
if (!store || !store->state || !view) {
@@ -8067,7 +8163,7 @@ retry_lookup:
slot->references++;
state->active_leases++;
slot->used = ++state->clock; state->stats.hits++;
- touch_lru_slot(state, key.layer, slot);
+ touch_lru_slot(state, slot);
if (hot_is_pinned(policy, key.layer, key.expert) && !store->gpu)
hot_pack_slot_locked(policy, state, record, slot);
memset(view, 0, sizeof(*view)); view->key = key;
@@ -8093,8 +8189,12 @@ retry_lookup:
}
goto retry_lookup;
}
- slot = next_empty_slot(state, key.layer);
- if (!slot) slot = hot_oldest_victim(state, policy, key.layer);
+ int pooled = state->pool_layer == key.layer;
+ slot = pooled ? next_pooled_empty_slot(state, key.layer)
+ : next_empty_slot(state, key.layer);
+ if (!slot)
+ slot = pooled ? hot_oldest_pool_victim(state, policy)
+ : hot_oldest_victim(state, policy, key.layer);
if (!slot) {
pthread_mutex_unlock(&state->mutex);
memset(view, 0, sizeof(*view));
@@ -8109,7 +8209,7 @@ retry_lookup:
return -1;
}
slot->aligned_slab = 1;
- mark_slot_allocated(state, key.layer, slot);
+ mark_slot_allocated(state, slot_partition(state, slot), slot);
state->stats.resident_bytes += state->record_bytes;
}
policy->packed[hot_slot_index(state, slot)] = 0;
@@ -8119,7 +8219,7 @@ retry_lookup:
slot->references = 1;
state->active_leases++;
slot->used = ++state->clock;
- touch_lru_slot(state, key.layer, slot);
+ touch_lru_slot(state, slot);
pthread_mutex_unlock(&state->mutex);
int rep = coli_st_expert_route(key.layer, key.expert);
struct timespec disk_t0;
@@ -8146,7 +8246,7 @@ retry_lookup:
(disk_t1.tv_nsec - disk_t0.tv_nsec) * 1e-9;
if (read_result) {
unindex_expert_slot(state, slot);
- slot->references = 0; slot->expert = -1;
+ slot->references = 0; slot->owner_layer = -1; slot->expert = -1;
slot->loading_expert = -1;
if (state->active_leases) state->active_leases--;
free(v4_pack_buf);
@@ -8157,7 +8257,7 @@ retry_lookup:
}
slot->expert = key.expert; slot->loading_expert = -1;
slot->used = ++state->clock;
- touch_lru_slot(state, key.layer, slot);
+ touch_lru_slot(state, slot);
state->stats.misses++; state->stats.bytes_read += record->record_bytes;
hot_pack_slot_commit(policy, state, record, slot, v4_pack_buf);
v4_pack_buf = NULL;
@@ -8184,6 +8284,24 @@ int coli_v4_test_expert_slot_index(ColiExpertStore *store, ColiExpertKey key) {
}
#endif
+/* Let the layer currently sweeping a batched CPU prefill borrow the complete
+ * cache capacity. Slots retain an explicit logical owner and stay indexed, so
+ * changing layers does not blindly flush warm entries: an entry is displaced
+ * only when the active layer actually needs its slab. Pinned entries remain
+ * protected by hot_oldest_pool_victim().
+ *
+ * The capability is intentionally private to the production hot store. A
+ * registered alternative ExpertStore backend receives a safe no-op rather
+ * than having its opaque state reinterpreted here. */
+void coli_v4_expert_store_prefill_pool(ColiExpertStore *store, int layer) {
+ if (!store || !store->state || store->gpu || !hot_find(store)) return;
+ V4ExpertStoreState *state = store->state;
+ if (layer < 0 || layer >= state->layers) layer = -1;
+ pthread_mutex_lock(&state->mutex);
+ state->pool_layer = layer;
+ pthread_mutex_unlock(&state->mutex);
+}
+
static void destroy_hot(ColiExpertStore *store) {
pthread_mutex_lock(&hot_policies_mutex);
V4HotPolicy **link = &hot_policies;
@@ -8391,12 +8509,10 @@ void coli_v4_expert_store_emit_emap(ColiExpertStore *store) {
if (!hex) return;
pthread_mutex_lock(&state->mutex);
for (size_t i = 0; i < cells; i++) {
- int tier = 0;
int layer = (int)(i / (size_t)cols), expert = (int)(i % (size_t)cols);
- V4ExpertSlot *slots = state->slots +
- (size_t)layer * state->slots_per_layer;
- for (int z = 0; z < state->slots_per_layer; z++)
- if (slots[z].slab && slots[z].expert == expert) { tier = 1; break; }
+ V4ExpertSlot *slot = indexed_expert_slot(
+ state, (ColiExpertKey){layer, expert});
+ int tier = slot && slot->slab && slot->expert == expert;
int heat = state->eheat ? state->eheat[i] : 0;
if (heat > 63) heat = 63;
int b = (tier << 6) | heat;
@@ -11298,12 +11414,21 @@ static const float *v4_mainh_get(int64_t position) {
#include "deepseek_v4_dspark.inc"
+static int v4_prefill_pool_enabled(void) {
+ static int enabled = -1;
+ if (enabled < 0) {
+ const char *setting = getenv("COLI_V4_PREFILL_POOL");
+ enabled = !setting || atoi(setting) != 0;
+ }
+ return enabled;
+}
+
static int target_batch(ColiV4Engine *engine, float **state_ptr, float **next_ptr,
ColiDeepSeekV4WindowAttentionState **attention,
const ColiSafetensorsIndex *index,
const ColiDeepSeekV4Config *config,
ColiExpertStore *experts, const int *tokens,
- int start, int batch,
+ int start, int batch, int use_prefill_pool,
ColiV4SessionAbortFn should_abort, void *abort_ctx,
char *error, size_t error_size) {
if (!state_ptr || !next_ptr || !*state_ptr || !*next_ptr || !attention ||
@@ -11314,10 +11439,25 @@ static int target_batch(ColiV4Engine *engine, float **state_ptr, float **next_pt
}
float *state = *state_ptr, *next = *next_ptr;
size_t hd = (size_t)config->hc_mult * config->hidden_size;
+ /* Keep a kill switch for checkpoint A/B and unusual storage backends. It
+ * does not change the caller's semantic distinction: speculative decode
+ * always passes use_prefill_pool=0. */
+ int pool_experts = use_prefill_pool && v4_prefill_pool_enabled();
for (int layer_id = 0; layer_id < config->num_hidden_layers; layer_id++) {
ColiDeepSeekV4LayerWeights layer;
if (coli_v4_layer_load(engine, &layer, config, index, layer_id,
- error, error_size)) return -1;
+ error, error_size)) {
+ if (pool_experts)
+ coli_v4_expert_store_prefill_pool(experts, -1);
+ return -1;
+ }
+ /* A true prefill is layer-major: every prompt position crosses this
+ * layer before the next one begins. Let it borrow otherwise idle
+ * partitions so its expert union remains resident across chunks.
+ * Speculative decode also calls target_batch, but passes 0 and keeps
+ * ordinary per-layer allocation, as does target_token. */
+ if (pool_experts)
+ coli_v4_expert_store_prefill_pool(experts, layer_id);
int result = 0;
/* Chunk width caps every batch-scaled buffer in the block AND bounds
* the expert union. The batch kernels' contract is 128 (the CPU
@@ -11357,12 +11497,17 @@ static int target_batch(ColiV4Engine *engine, float **state_ptr, float **next_pt
layer_id, offset, chunk);
}
coli_v4_layer_free(engine, &layer);
- if (result) return -1;
+ if (result) {
+ if (pool_experts)
+ coli_v4_expert_store_prefill_pool(experts, -1);
+ return -1;
+ }
float *swap = state; state = next; next = swap;
for (int item = 0; item < batch; item++)
v4_mainh_tap(config, layer_id, state + (size_t)item * hd,
(int64_t)start + item);
}
+ if (pool_experts) coli_v4_expert_store_prefill_pool(experts, -1);
*state_ptr = state;
*next_ptr = next;
return 0;
@@ -12357,7 +12502,7 @@ int coli_v4_session_generate(ColiV4Session *session,
}
if (target_batch(engine, &state, &next, attention, index, config,
experts, session->prompt_ids + done_upto, done_upto,
- seg, NULL, NULL, error, error_size)) {
+ seg, 1, NULL, NULL, error, error_size)) {
/* A real failure leaves the segment half-applied across layers;
* only then is the record discarded. Cancels never land here —
* segments are atomic and the poll runs between them. */
@@ -12502,7 +12647,7 @@ int coli_v4_session_generate(ColiV4Session *session,
}
if (target_batch(engine, &state, &next, attention, index,
config, experts, inputs, old_last + 1,
- batch, NULL, NULL, error, error_size)) {
+ batch, 0, NULL, NULL, error, error_size)) {
(void)spec_attention_restore(
attention, snapshots, config->num_hidden_layers);
spec_attention_free(snapshots,
@@ -12606,7 +12751,7 @@ int coli_v4_session_generate(ColiV4Session *session,
if (retained > 0 && target_batch(
engine, &state, &next, attention, index,
config, experts, inputs, old_last + 1,
- retained, NULL, NULL, error, error_size)) {
+ retained, 0, NULL, NULL, error, error_size)) {
spec_attention_free(
snapshots, config->num_hidden_layers);
kv_prefix_taint(&session->fed);
@@ -12719,7 +12864,8 @@ static int v4_oracle_teacher_forcing(
return -1;
}
if (target_batch(NULL, &state, &next, attention, index, config, experts,
- full_ids, 0, full_count, NULL, NULL, error, error_size)) {
+ full_ids, 0, full_count, 1, NULL, NULL,
+ error, error_size)) {
free(state); free(next); free(hidden);
return -1;
}
@@ -12765,7 +12911,7 @@ static int v4_oracle_greedy_from_prompt(
return -1;
}
if (target_batch(NULL, &state, &next, attention, index, config, experts,
- prompt_ids, 0, prompt_count, NULL, NULL,
+ prompt_ids, 0, prompt_count, 1, NULL, NULL,
error, error_size)) {
free(state); free(next); free(hidden);
return -1;
@@ -13592,7 +13738,7 @@ int main(int argc, char **argv) {
if (load_embedding(tf_state + (size_t)item * hd_tf, index, &config,
full_ids[item])) goto cleanup;
if (target_batch(engine, &tf_state, &tf_next, attention, index, &config,
- experts, full_ids, 0, full_count, NULL, NULL,
+ experts, full_ids, 0, full_count, 1, NULL, NULL,
error, sizeof(error))) {
fprintf(stderr, "%s\n", error); goto cleanup;
}
diff --git a/c/deepseek_v4_internal.h b/c/deepseek_v4_internal.h
index f5c70087..6873fe8c 100644
--- a/c/deepseek_v4_internal.h
+++ b/c/deepseek_v4_internal.h
@@ -611,6 +611,11 @@ int coli_deepseek_v4_expert_store_open(
char *error,
size_t error_size);
+/* Batched CPU prefill only: `layer` borrows the complete expert-cache pool;
+ * a negative value restores ordinary per-layer miss allocation for decode.
+ * Alternative registered ExpertStore backends safely ignore the request. */
+void coli_v4_expert_store_prefill_pool(ColiExpertStore *store, int layer);
+
#ifdef __cplusplus
}
#endif
diff --git a/c/tests/test_deepseek_v4.c b/c/tests/test_deepseek_v4.c
index fc083271..7d2eb999 100644
--- a/c/tests/test_deepseek_v4.c
+++ b/c/tests/test_deepseek_v4.c
@@ -384,12 +384,17 @@ static int write_all(int fd, const void *data, size_t length) {
return 0;
}
-static int write_fixture_experts(const char *path, int expert_count) {
+static int write_fixture_layers(const char *path, int layer_count,
+ int expert_count) {
static const char *matrix_names[3] = {"w1", "w2", "w3"};
- if (expert_count < 1 || (size_t)expert_count > SIZE_MAX / 1024) return -1;
- size_t header_capacity = 256 + (size_t)expert_count * 1024;
+ if (layer_count < 1 || expert_count < 1 ||
+ (size_t)layer_count > SIZE_MAX / (size_t)expert_count)
+ return -1;
+ size_t cells = (size_t)layer_count * expert_count;
+ if (cells > (SIZE_MAX - 256) / 1024) return -1;
+ size_t header_capacity = 256 + cells * 1024;
char *header = malloc(header_capacity);
- size_t payload_size = 4 + (size_t)expert_count * 51;
+ size_t payload_size = 4 + cells * 51;
unsigned char *payload = malloc(payload_size);
if (!header || !payload) {
free(payload);
@@ -400,34 +405,37 @@ static int write_fixture_experts(const char *path, int expert_count) {
header, header_capacity,
"{\"resident\":{\"dtype\":\"F32\",\"shape\":[1],"
"\"data_offsets\":[3,7]}");
- for (int expert = 0; expert < expert_count; expert++) {
- size_t scale = expert ? 4 + (size_t)expert * 51 : 0;
- size_t weight = scale + 3 + (expert ? 0 : 4);
- for (int matrix = 0; matrix < 3; matrix++) {
- int count = snprintf(
- header + used, header_capacity - used,
- ",\"layers.0.ffn.experts.%d.%s.scale\":{"
- "\"dtype\":\"F8_E8M0\",\"shape\":[1,1],"
- "\"data_offsets\":[%zu,%zu]}",
- expert, matrix_names[matrix], scale + matrix,
- scale + matrix + 1);
- if (count < 0 || (size_t)count >= header_capacity - used) {
- free(payload); free(header); return -1;
+ for (int layer = 0; layer < layer_count; layer++) {
+ for (int expert = 0; expert < expert_count; expert++) {
+ size_t cell = (size_t)layer * expert_count + expert;
+ size_t scale = cell ? 4 + cell * 51 : 0;
+ size_t weight = scale + 3 + (cell ? 0 : 4);
+ for (int matrix = 0; matrix < 3; matrix++) {
+ int count = snprintf(
+ header + used, header_capacity - used,
+ ",\"layers.%d.ffn.experts.%d.%s.scale\":{"
+ "\"dtype\":\"F8_E8M0\",\"shape\":[1,1],"
+ "\"data_offsets\":[%zu,%zu]}",
+ layer, expert, matrix_names[matrix], scale + matrix,
+ scale + matrix + 1);
+ if (count < 0 || (size_t)count >= header_capacity - used) {
+ free(payload); free(header); return -1;
+ }
+ used += (size_t)count;
}
- used += (size_t)count;
- }
- for (int matrix = 0; matrix < 3; matrix++) {
- int count = snprintf(
- header + used, header_capacity - used,
- ",\"layers.0.ffn.experts.%d.%s.weight\":{"
- "\"dtype\":\"I8\",\"shape\":[1,16],"
- "\"data_offsets\":[%zu,%zu]}",
- expert, matrix_names[matrix], weight + matrix * 16,
- weight + (matrix + 1) * 16);
- if (count < 0 || (size_t)count >= header_capacity - used) {
- free(payload); free(header); return -1;
+ for (int matrix = 0; matrix < 3; matrix++) {
+ int count = snprintf(
+ header + used, header_capacity - used,
+ ",\"layers.%d.ffn.experts.%d.%s.weight\":{"
+ "\"dtype\":\"I8\",\"shape\":[1,16],"
+ "\"data_offsets\":[%zu,%zu]}",
+ layer, expert, matrix_names[matrix], weight + matrix * 16,
+ weight + (matrix + 1) * 16);
+ if (count < 0 || (size_t)count >= header_capacity - used) {
+ free(payload); free(header); return -1;
+ }
+ used += (size_t)count;
}
- used += (size_t)count;
}
}
if (used + 1 >= header_capacity) {
@@ -451,6 +459,10 @@ static int write_fixture_experts(const char *path, int expert_count) {
return result ? -1 : 0;
}
+static int write_fixture_experts(const char *path, int expert_count) {
+ return write_fixture_layers(path, 1, expert_count);
+}
+
static int write_fixture(const char *path) {
return write_fixture_experts(path, 7);
}
@@ -471,6 +483,24 @@ static int expect_fixture_expert(const ColiExpertView *view, int expert) {
(unsigned char)(weight + 32);
}
+static int expect_fixture_expert_at(const ColiExpertView *view, int layer,
+ int expert, int experts_per_layer) {
+ int cell = layer * experts_per_layer + expert;
+ int scale = cell ? 4 + cell * 51 : 0;
+ int weight = scale + 3 + (cell ? 0 : 4);
+ return view->gate.format == COLI_TENSOR_FP4_NATIVE_BLOCK &&
+ view->gate.rows == 1 && view->gate.columns == 32 &&
+ view->gate.data_bytes == 16 && view->gate.scale_bytes == 1 &&
+ ((const unsigned char *)view->gate.scales)[0] ==
+ (unsigned char)scale &&
+ ((const unsigned char *)view->gate.data)[0] ==
+ (unsigned char)weight &&
+ ((const unsigned char *)view->down.data)[0] ==
+ (unsigned char)(weight + 16) &&
+ ((const unsigned char *)view->up.data)[0] ==
+ (unsigned char)(weight + 32);
+}
+
enum {
TEST_READ_SAME_EXPERT,
TEST_READ_DISTINCT_EXPERTS
@@ -756,6 +786,184 @@ static int test_expert_store(void) {
return 0;
}
+static int lookup_fixture_at(ColiExpertStore *store, int layer, int expert,
+ int experts_per_layer) {
+ ColiExpertView view;
+ if (coli_expert_lookup(store, (ColiExpertKey){layer, expert}, &view) ||
+ !expect_fixture_expert_at(&view, layer, expert, experts_per_layer)) {
+ fprintf(stderr, "pooled fixture mismatch: layer=%d expert=%d\n",
+ layer, expert);
+ return -1;
+ }
+ coli_expert_release(store, &view);
+ return 0;
+}
+
+/* Six slots per physical layer cannot retain an eight-expert union. Pooling
+ * two layers provides twelve slots: the first sweep reads each layer-0 expert
+ * once, the second sweep must read zero bytes. The test also proves that a
+ * borrowed slot remains indexed after pool close and that switching pool owner
+ * does not blindly flush an already-warm expert from another layer. */
+static int test_expert_store_prefill_pool(void) {
+ enum { LAYERS = 2, EXPERTS = 8, SLOTS_PER_LAYER = 6 };
+ char directory[] = "colibri-v4-pool-XXXXXX";
+ char path[256], error[256];
+ setenv("COLI_V4_AUTOPIN", "0", 1);
+ setenv("COLI_V4_SAVE_USAGE", "0", 1);
+ setenv("COLI_V4_ROWS16", "0", 1);
+ if (!mkdtemp(directory)) { perror("mkdtemp pool"); return 1; }
+ snprintf(path, sizeof(path), "%s/model.safetensors", directory);
+ if (write_fixture_layers(path, LAYERS, EXPERTS)) {
+ perror("write pool fixture");
+ rmdir(directory);
+ return 1;
+ }
+ ColiDeepSeekV4ExpertStoreOptions options = {
+ directory, LAYERS, EXPERTS,
+ (uint64_t)LAYERS * SLOTS_PER_LAYER * 51, -1, UINT64_MAX
+ };
+ /* Deterministic A/B. With the ordinary six-slot partition, iterating an
+ * eight-expert union in the same order twice is a cyclic 0%%-hit workload:
+ * 16 reads. The pooled twelve-slot capacity must turn the second sweep
+ * into eight hits: 8 reads, exactly half the bytes. */
+ ColiExpertStore *baseline = NULL;
+ if (coli_deepseek_v4_expert_store_open(
+ &options, &baseline, error, sizeof(error))) {
+ fprintf(stderr, "baseline pool store open failed: %s\n", error);
+ unlink(path); rmdir(directory);
+ return 1;
+ }
+ int failed = 0;
+ for (int sweep = 0; sweep < 2 && !failed; sweep++)
+ for (int expert = 0; expert < EXPERTS; expert++)
+ if (lookup_fixture_at(baseline, 0, expert, EXPERTS)) {
+ failed = 1;
+ break;
+ }
+ ColiExpertStoreStats baseline_stats;
+ baseline->ops->stats(baseline, &baseline_stats);
+ baseline->ops->destroy(baseline);
+ if (baseline_stats.requests != 16 || baseline_stats.hits != 0 ||
+ baseline_stats.misses != 16 || baseline_stats.bytes_read != 16 * 51) {
+ fprintf(stderr,
+ "prefill baseline mismatch: requests=%llu hits=%llu "
+ "misses=%llu bytes=%llu\n",
+ (unsigned long long)baseline_stats.requests,
+ (unsigned long long)baseline_stats.hits,
+ (unsigned long long)baseline_stats.misses,
+ (unsigned long long)baseline_stats.bytes_read);
+ failed = 1;
+ }
+
+ ColiExpertStore *store = NULL;
+ if (coli_deepseek_v4_expert_store_open(
+ &options, &store, error, sizeof(error))) {
+ fprintf(stderr, "pool store open failed: %s\n", error);
+ unlink(path); rmdir(directory);
+ return 1;
+ }
+ /* A warm entry owned by the other layer must survive while unused slabs
+ * are still available to the pooled layer. */
+ failed |= lookup_fixture_at(store, 1, 7, EXPERTS);
+ coli_v4_expert_store_prefill_pool(store, 0);
+ for (int sweep = 0; sweep < 2 && !failed; sweep++)
+ for (int expert = 0; expert < EXPERTS; expert++)
+ if (lookup_fixture_at(store, 0, expert, EXPERTS)) {
+ failed = 1;
+ break;
+ }
+ ColiExpertStoreStats pooled_sweeps;
+ store->ops->stats(store, &pooled_sweeps);
+ /* Subtract the one layer-1 warmup read above. */
+ uint64_t pooled_requests = pooled_sweeps.requests - 1;
+ uint64_t pooled_hits = pooled_sweeps.hits;
+ uint64_t pooled_misses = pooled_sweeps.misses - 1;
+ uint64_t pooled_bytes = pooled_sweeps.bytes_read - 51;
+ if (pooled_requests != 16 || pooled_hits != 8 || pooled_misses != 8 ||
+ pooled_bytes != 8 * 51 ||
+ baseline_stats.bytes_read != pooled_bytes * 2) {
+ fprintf(stderr,
+ "prefill pooled A/B mismatch: requests=%llu hits=%llu "
+ "misses=%llu bytes=%llu baseline_bytes=%llu\n",
+ (unsigned long long)pooled_requests,
+ (unsigned long long)pooled_hits,
+ (unsigned long long)pooled_misses,
+ (unsigned long long)pooled_bytes,
+ (unsigned long long)baseline_stats.bytes_read);
+ failed = 1;
+ }
+ int borrowed = coli_v4_test_expert_slot_index(
+ store, (ColiExpertKey){0, EXPERTS - 1});
+ if (borrowed < SLOTS_PER_LAYER) {
+ fprintf(stderr, "prefill did not borrow a slot: index=%d\n", borrowed);
+ failed = 1;
+ }
+
+ coli_v4_expert_store_prefill_pool(store, 1);
+ failed |= lookup_fixture_at(store, 1, 7, EXPERTS);
+ coli_v4_expert_store_prefill_pool(store, -1);
+ failed |= lookup_fixture_at(store, 0, 7, EXPERTS);
+ failed |= lookup_fixture_at(store, 1, 7, EXPERTS);
+
+ ColiExpertStoreStats stats;
+ store->ops->stats(store, &stats);
+ if (stats.requests != 20 || stats.hits != 11 || stats.misses != 9 ||
+ stats.bytes_read != 9 * 51 || stats.resident_bytes != 9 * 51 ||
+ stats.capacity_bytes != LAYERS * SLOTS_PER_LAYER * 51) {
+ fprintf(stderr,
+ "prefill pool stats mismatch: requests=%llu hits=%llu "
+ "misses=%llu bytes=%llu resident=%llu capacity=%llu\n",
+ (unsigned long long)stats.requests,
+ (unsigned long long)stats.hits,
+ (unsigned long long)stats.misses,
+ (unsigned long long)stats.bytes_read,
+ (unsigned long long)stats.resident_bytes,
+ (unsigned long long)stats.capacity_bytes);
+ failed = 1;
+ }
+
+ /* Fill the remaining slabs from layer 1, then force four global victims.
+ * Layer 0 started with eight pooled residents: its six-expert decode
+ * reserve must survive while layer 1 replaces its own older entries. */
+ coli_v4_expert_store_prefill_pool(store, 1);
+ for (int expert = 0; expert < 7; expert++)
+ failed |= lookup_fixture_at(store, 1, expert, EXPERTS);
+ coli_v4_expert_store_prefill_pool(store, -1);
+ int layer0_resident = 0;
+ for (int expert = 0; expert < EXPERTS; expert++)
+ if (coli_v4_test_expert_slot_index(
+ store, (ColiExpertKey){0, expert}) >= 0)
+ layer0_resident++;
+ if (layer0_resident != SLOTS_PER_LAYER) {
+ fprintf(stderr, "prefill pool lost decode reserve: layer0=%d\n",
+ layer0_resident);
+ failed = 1;
+ }
+ store->ops->stats(store, &stats);
+ if (stats.requests != 27 || stats.hits != 11 || stats.misses != 16 ||
+ stats.bytes_read != 16 * 51 ||
+ stats.resident_bytes != LAYERS * SLOTS_PER_LAYER * 51 ||
+ stats.capacity_bytes != LAYERS * SLOTS_PER_LAYER * 51) {
+ fprintf(stderr,
+ "prefill reserve stats mismatch: requests=%llu hits=%llu "
+ "misses=%llu bytes=%llu resident=%llu capacity=%llu\n",
+ (unsigned long long)stats.requests,
+ (unsigned long long)stats.hits,
+ (unsigned long long)stats.misses,
+ (unsigned long long)stats.bytes_read,
+ (unsigned long long)stats.resident_bytes,
+ (unsigned long long)stats.capacity_bytes);
+ failed = 1;
+ }
+ store->ops->destroy(store);
+ unlink(path);
+ rmdir(directory);
+ if (failed) return 1;
+ puts("DeepSeek-V4 ExpertStore prefill pool: ok "
+ "(A/B bytes 816->408, second sweep=0 reads, warm entries + decode reserve retained)");
+ return 0;
+}
+
static int run_expert_miss_scaling_case(const char *directory, int experts,
int slots) {
char error[256];
@@ -1302,6 +1510,10 @@ int main(int argc, char **argv) {
fprintf(stderr, "FAIL: test_expert_store\n");
return 1;
}
+ if (test_expert_store_prefill_pool() != 0) {
+ fprintf(stderr, "FAIL: test_expert_store_prefill_pool\n");
+ return 1;
+ }
if (test_expert_store_miss_scaling() != 0) {
fprintf(stderr, "FAIL: test_expert_store_miss_scaling\n");
return 1;