Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(v4): use direct reads for per_matrix REAP expert records (#1615)

Frank_zhu62702f2c8ca0
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.c
index e2a8bf27..d57f6e5b 100644
--- a/c/deepseek_v4.c
+++ b/c/deepseek_v4.c
@@ -8223,8 +8223,9 @@ static size_t hot_slot_index(const V4ExpertStoreState *state,
*
* FLOCK-packed checkpoints store [scales][weights] contiguously and need one
* request. Standard HF checkpoints keep the ranges apart: weights use direct
- * I/O while the much smaller scales use buffered pread. Any direct-I/O error
- * falls back to the exact buffered path. */
+ * I/O while the much smaller scales use buffered pread. REAP-style
+ * per_matrix records issue one window per scale/weight segment. Any
+ * direct-I/O error falls back to the exact buffered path. */
static uint64_t v4_direct_reads;
static uint64_t v4_direct_flock_reads;
static uint64_t v4_direct_payload_bytes;
@@ -8280,33 +8281,87 @@ static int v4_read_direct_window(const V4ExpertStoreState *state, int shard,
return 0;
}
+/* Direct window into an interior slab offset. v4_read_direct_window bounces
+ * at slab[0], so a second per_matrix segment would clobber earlier bytes. */
+static int v4_read_direct_copy(const V4ExpertStoreState *state, int shard,
+ int rep, unsigned char *destination,
+ uint64_t offset, size_t length) {
+ if (!destination) return -1;
+ if (!length) return 0;
+ if (length > SIZE_MAX - 8192u) return -1;
+ unsigned char *bounce = NULL;
+ if (posix_memalign((void **)&bounce, 4096, length + 8192u)) return -1;
+ int result = v4_read_direct_window(state, shard, rep, bounce, offset,
+ length, 0);
+ if (!result) memcpy(destination, bounce, length);
+ compat_aligned_free(bounce);
+ return result;
+}
+
+static int v4_try_direct_segment(V4ExpertStoreState *state, int shard, int rep,
+ V4ExpertSlot *slot, uint64_t dest,
+ uint64_t offset, uint64_t bytes) {
+ if (!slot->aligned_slab ||
+ !coli_st_streaming_direct_available_rep(state->index, shard, rep))
+ return -1;
+ size_t length = (size_t)bytes;
+ if (dest == 0)
+ return v4_read_direct_window(state, shard, rep, slot->slab, offset,
+ length, 0);
+ return v4_read_direct_copy(state, shard, rep, slot->slab + dest, offset,
+ length);
+}
+
+static int v4_read_per_matrix_segment(V4ExpertStoreState *state, int shard,
+ int rep, V4ExpertSlot *slot,
+ uint64_t dest, uint64_t offset,
+ uint64_t bytes, int *used_direct,
+ int *used_fallback) {
+ if (!v4_try_direct_segment(state, shard, rep, slot, dest, offset, bytes)) {
+ *used_direct = 1;
+ return 0;
+ }
+ if (slot->aligned_slab &&
+ coli_st_streaming_direct_available_rep(state->index, shard, rep))
+ *used_fallback = 1;
+ return coli_st_read_at_rep(state->index, shard, rep, offset, (size_t)bytes,
+ slot->slab + dest);
+}
+
static int v4_read_expert_record(V4ExpertStoreState *state,
const V4ExpertRecord *record,
V4ExpertSlot *slot, int rep) {
if (record->per_matrix) {
- int direct_available = slot->aligned_slab &&
- coli_st_streaming_direct_available_rep(state->index, record->m_scale_shard[0], rep);
- if (direct_available)
- __atomic_fetch_add(&v4_direct_fallbacks, UINT64_C(1),
- __ATOMIC_RELAXED);
+ int used_direct = 0;
+ int used_fallback = 0;
uint64_t scale_cursor = 0;
for (int matrix = 0; matrix < V4_MATRIX_COUNT; matrix++) {
- if (coli_st_read_at_rep(state->index, record->m_scale_shard[matrix], rep,
- record->m_scale_offset[matrix],
- (size_t)record->m_scale_bytes[matrix],
- slot->slab + scale_cursor) != 0)
+ if (v4_read_per_matrix_segment(
+ state, record->m_scale_shard[matrix], rep, slot,
+ scale_cursor, record->m_scale_offset[matrix],
+ record->m_scale_bytes[matrix], &used_direct,
+ &used_fallback) != 0)
return -1;
scale_cursor += record->m_scale_bytes[matrix];
}
uint64_t weight_cursor = scale_cursor;
for (int matrix = 0; matrix < V4_MATRIX_COUNT; matrix++) {
- if (coli_st_read_at_rep(state->index, record->m_weight_shard[matrix], rep,
- record->m_weight_offset[matrix],
- (size_t)record->m_weight_bytes[matrix],
- slot->slab + weight_cursor) != 0)
+ if (v4_read_per_matrix_segment(
+ state, record->m_weight_shard[matrix], rep, slot,
+ weight_cursor, record->m_weight_offset[matrix],
+ record->m_weight_bytes[matrix], &used_direct,
+ &used_fallback) != 0)
return -1;
weight_cursor += record->m_weight_bytes[matrix];
}
+ if (used_direct && !used_fallback) {
+ __atomic_fetch_add(&v4_direct_reads, UINT64_C(1), __ATOMIC_RELAXED);
+ __atomic_fetch_add(&v4_direct_payload_bytes, record->record_bytes,
+ __ATOMIC_RELAXED);
+ } else if (used_fallback) {
+ __atomic_fetch_add(&v4_direct_fallbacks, UINT64_C(1),
+ __ATOMIC_RELAXED);
+ }
return 0;
}
int direct_available = slot->aligned_slab &&
@@ -8764,6 +8819,37 @@ int coli_v4_test_expert_slot_index(ColiExpertStore *store, ColiExpertKey key) {
pthread_mutex_unlock(&state->mutex);
return result;
}
+
+void coli_v4_test_reset_direct_io_stats(void) {
+ __atomic_store_n(&v4_direct_reads, 0, __ATOMIC_RELAXED);
+ __atomic_store_n(&v4_direct_flock_reads, 0, __ATOMIC_RELAXED);
+ __atomic_store_n(&v4_direct_payload_bytes, 0, __ATOMIC_RELAXED);
+ __atomic_store_n(&v4_direct_fallbacks, 0, __ATOMIC_RELAXED);
+}
+
+uint64_t coli_v4_test_direct_reads(void) {
+ return __atomic_load_n(&v4_direct_reads, __ATOMIC_RELAXED);
+}
+
+uint64_t coli_v4_test_direct_fallbacks(void) {
+ return __atomic_load_n(&v4_direct_fallbacks, __ATOMIC_RELAXED);
+}
+
+int coli_v4_test_force_streaming_direct(ColiExpertStore *store) {
+ if (!store || !store->state) return -1;
+ V4ExpertStoreState *state = store->state;
+ if (!state->index) return -1;
+ int enabled = 0;
+ for (int i = 0; i < state->index->nfd; i++) {
+ if (state->index->dfds[i] < 0 && state->index->fds[i] >= 0) {
+ int twin = dup(state->index->fds[i]);
+ if (twin < 0) return -1;
+ state->index->dfds[i] = twin;
+ }
+ if (state->index->dfds[i] >= 0) enabled = 1;
+ }
+ return enabled ? 0 : -1;
+}
#endif
/* Let the layer currently sweeping a batched CPU prefill borrow the complete
diff --git a/c/deepseek_v4_internal.h b/c/deepseek_v4_internal.h
index 2bcdf9cd..bfba3108 100644
--- a/c/deepseek_v4_internal.h
+++ b/c/deepseek_v4_internal.h
@@ -1002,6 +1002,12 @@ extern void (*coli_v4_test_expert_wait_hook)(ColiExpertKey key);
extern uint64_t coli_v4_test_fp4_batch_calls;
extern uint64_t coli_v4_test_expert_victim_probes;
int coli_v4_test_expert_slot_index(ColiExpertStore *store, ColiExpertKey key);
+void coli_v4_test_reset_direct_io_stats(void);
+uint64_t coli_v4_test_direct_reads(void);
+uint64_t coli_v4_test_direct_fallbacks(void);
+/* Point missing O_DIRECT twins at a dup of the buffered fd so tests can
+ * exercise the direct-window path on filesystems that refuse O_DIRECT. */
+int coli_v4_test_force_streaming_direct(ColiExpertStore *store);
ColiV4Session *coli_v4_test_session_bare_create(ColiV4Engine *engine);
void coli_v4_test_session_bare_destroy(ColiV4Session *session);
diff --git a/c/tests/test_deepseek_v4.c b/c/tests/test_deepseek_v4.c
index 55224b82..7dd6b829 100644
--- a/c/tests/test_deepseek_v4.c
+++ b/c/tests/test_deepseek_v4.c
@@ -1063,6 +1063,163 @@ static int test_expert_store_miss_scaling(void) {
"(44/104/208 slots=1 probe each)");
return 0;
}
+
+/* REAP-style layout: per-matrix scale/weight pairs so contiguous_group()
+ * fails and build_record() sets per_matrix=1. Payload is padded past 4 KiB
+ * so an O_DIRECT window has a real aligned pread. */
+static int write_per_matrix_fixture(const char *path, int expert_count) {
+ static const char *matrix_names[3] = {"w1", "w2", "w3"};
+ if (expert_count < 1) return -1;
+ size_t header_capacity = 256 + (size_t)expert_count * 1024;
+ char *header = malloc(header_capacity);
+ size_t payload_size = 8192;
+ unsigned char *payload = malloc(payload_size);
+ if (!header || !payload) {
+ free(payload);
+ free(header);
+ return -1;
+ }
+ size_t used = (size_t)snprintf(header, header_capacity, "{");
+ size_t cursor = 0;
+ for (int expert = 0; expert < expert_count; expert++) {
+ for (int matrix = 0; matrix < 3; matrix++) {
+ size_t scale = cursor;
+ size_t weight = cursor + 1;
+ int count = snprintf(
+ header + used, header_capacity - used,
+ "%s\"layers.0.ffn.experts.%d.%s.scale\":{"
+ "\"dtype\":\"F8_E8M0\",\"shape\":[1,1],"
+ "\"data_offsets\":[%zu,%zu]}",
+ used > 1 ? "," : "", expert, matrix_names[matrix],
+ scale, scale + 1);
+ if (count < 0 || (size_t)count >= header_capacity - used) {
+ free(payload); free(header); return -1;
+ }
+ used += (size_t)count;
+ count = snprintf(
+ header + used, header_capacity - used,
+ ",\"layers.0.ffn.experts.%d.%s.weight\":{"
+ "\"dtype\":\"I8\",\"shape\":[1,16],"
+ "\"data_offsets\":[%zu,%zu]}",
+ expert, matrix_names[matrix], weight, weight + 16);
+ if (count < 0 || (size_t)count >= header_capacity - used) {
+ free(payload); free(header); return -1;
+ }
+ used += (size_t)count;
+ cursor += 17;
+ }
+ }
+ if (used + 1 >= header_capacity) {
+ free(payload); free(header); return -1;
+ }
+ header[used++] = '}';
+ header[used] = '\0';
+ for (size_t i = 0; i < payload_size; i++)
+ payload[i] = (unsigned char)i;
+ uint64_t header_length = used;
+ int fd = open(path, O_CREAT | O_TRUNC | O_WRONLY | COMPAT_O_BINARY, 0600);
+ if (fd < 0) {
+ free(payload); free(header); return -1;
+ }
+ int result = write_all(fd, &header_length, sizeof(header_length)) ||
+ write_all(fd, header, (size_t)header_length) ||
+ write_all(fd, payload, payload_size);
+ close(fd);
+ free(payload);
+ free(header);
+ return result ? -1 : 0;
+}
+
+static int expect_per_matrix_expert(const ColiExpertView *view, int expert) {
+ int base = expert * 51;
+ return view->gate.format == COLI_TENSOR_FP4_NATIVE_BLOCK &&
+ view->gate.rows == 1 && view->gate.columns == 32 &&
+ view->gate.data_bytes == 16 && view->gate.scale_bytes == 1 &&
+ view->down.data_bytes == 16 && view->up.data_bytes == 16 &&
+ ((const unsigned char *)view->gate.scales)[0] ==
+ (unsigned char)base &&
+ ((const unsigned char *)view->down.scales)[0] ==
+ (unsigned char)(base + 17) &&
+ ((const unsigned char *)view->up.scales)[0] ==
+ (unsigned char)(base + 34) &&
+ ((const unsigned char *)view->gate.data)[0] ==
+ (unsigned char)(base + 1) &&
+ ((const unsigned char *)view->down.data)[0] ==
+ (unsigned char)(base + 18) &&
+ ((const unsigned char *)view->up.data)[0] ==
+ (unsigned char)(base + 35);
+}
+
+static int test_expert_store_per_matrix_direct(void) {
+ /* Would fail this test: per_matrix always increments v4_direct_fallbacks
+ * and never calls v4_read_direct_window even when a direct twin exists. */
+ char directory[] = "colibri-v4-reap-XXXXXX";
+ char path[256], error[256];
+ setenv("COLI_V4_AUTOPIN", "0", 1);
+ setenv("COLI_V4_SAVE_USAGE", "0", 1);
+ setenv("COLI_V4_ROWS16", "0", 1);
+ setenv("COLI_V4_PREWARM", "0", 1);
+ unsetenv("COLI_V4_DIRECT");
+ if (!mkdtemp(directory)) { perror("mkdtemp per_matrix"); return 1; }
+ snprintf(path, sizeof(path), "%s/model.safetensors", directory);
+ if (write_per_matrix_fixture(path, 2) != 0) {
+ perror("write_per_matrix_fixture");
+ rmdir(directory);
+ return 1;
+ }
+
+ ColiDeepSeekV4ExpertStoreOptions options = {
+ directory, 1, 2, 102, -1, 0, 0
+ };
+ ColiExpertStore *store = NULL;
+ if (coli_deepseek_v4_expert_store_open(&options, &store,
+ error, sizeof(error)) != 0) {
+ fprintf(stderr, "%s\n", error);
+ unlink(path); rmdir(directory);
+ return 1;
+ }
+ if (coli_v4_test_force_streaming_direct(store) != 0) {
+ fprintf(stderr, "per_matrix direct: no streaming-direct fd\n");
+ store->ops->destroy(store);
+ unlink(path); rmdir(directory);
+ return 1;
+ }
+ coli_v4_test_reset_direct_io_stats();
+ ColiExpertView view;
+ if (coli_expert_lookup(store, (ColiExpertKey){0, 0}, &view) != 0) {
+ fprintf(stderr, "per_matrix lookup failed\n");
+ store->ops->destroy(store);
+ unlink(path); rmdir(directory);
+ return 1;
+ }
+ if (!expect_per_matrix_expert(&view, 0)) {
+ fprintf(stderr,
+ "per_matrix slab mismatch: scales=%u/%u/%u weights=%u/%u/%u\n",
+ ((const unsigned char *)view.gate.scales)[0],
+ ((const unsigned char *)view.down.scales)[0],
+ ((const unsigned char *)view.up.scales)[0],
+ ((const unsigned char *)view.gate.data)[0],
+ ((const unsigned char *)view.down.data)[0],
+ ((const unsigned char *)view.up.data)[0]);
+ coli_expert_release(store, &view);
+ store->ops->destroy(store);
+ unlink(path); rmdir(directory);
+ return 1;
+ }
+ coli_expert_release(store, &view);
+ uint64_t reads = coli_v4_test_direct_reads();
+ uint64_t fallbacks = coli_v4_test_direct_fallbacks();
+ store->ops->destroy(store);
+ if (scratch_remove(directory, path)) return 1;
+ if (reads < 1 || fallbacks != 0) {
+ fprintf(stderr,
+ "per_matrix direct path unused: reads=%llu fallbacks=%llu\n",
+ (unsigned long long)reads, (unsigned long long)fallbacks);
+ return 1;
+ }
+ puts("DeepSeek-V4 ExpertStore per_matrix direct: ok");
+ return 0;
+}
/* ==== end test_deepseek_v4_expert_store.c ==== */
/* ==== begin test_deepseek_v4_kv_cache.c ==== */
@@ -1531,6 +1688,10 @@ int main(int argc, char **argv) {
fprintf(stderr, "FAIL: test_expert_store_miss_scaling\n");
return 1;
}
+ if (test_expert_store_per_matrix_direct() != 0) {
+ fprintf(stderr, "FAIL: test_expert_store_per_matrix_direct\n");
+ return 1;
+ }
if (test_kv_cache() != 0) {
fprintf(stderr, "FAIL: test_kv_cache\n");
return 1;