Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(v4): use direct reads for per_matrix REAP expert records (#1615)
Frank_zhu62702f2c8ca0
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.cindex e2a8bf27..d57f6e5b 100644--- a/c/deepseek_v4.c+++ b/c/deepseek_v4.c@@ -8223,8 +8223,9 @@ static size_t hot_slot_index(const V4ExpertStoreState *state,** FLOCK-packed checkpoints store [scales][weights] contiguously and need one* request. Standard HF checkpoints keep the ranges apart: weights use direct- * I/O while the much smaller scales use buffered pread. Any direct-I/O error- * falls back to the exact buffered path. */+ * I/O while the much smaller scales use buffered pread. REAP-style+ * per_matrix records issue one window per scale/weight segment. Any+ * direct-I/O error falls back to the exact buffered path. */static uint64_t v4_direct_reads;static uint64_t v4_direct_flock_reads;static uint64_t v4_direct_payload_bytes;@@ -8280,33 +8281,87 @@ static int v4_read_direct_window(const V4ExpertStoreState *state, int shard,return 0;}+/* Direct window into an interior slab offset. v4_read_direct_window bounces+ * at slab[0], so a second per_matrix segment would clobber earlier bytes. */+static int v4_read_direct_copy(const V4ExpertStoreState *state, int shard,+ int rep, unsigned char *destination,+ uint64_t offset, size_t length) {+ if (!destination) return -1;+ if (!length) return 0;+ if (length > SIZE_MAX - 8192u) return -1;+ unsigned char *bounce = NULL;+ if (posix_memalign((void **)&bounce, 4096, length + 8192u)) return -1;+ int result = v4_read_direct_window(state, shard, rep, bounce, offset,+ length, 0);+ if (!result) memcpy(destination, bounce, length);+ compat_aligned_free(bounce);+ return result;+}++static int v4_try_direct_segment(V4ExpertStoreState *state, int shard, int rep,+ V4ExpertSlot *slot, uint64_t dest,+ uint64_t offset, uint64_t bytes) {+ if (!slot->aligned_slab ||+ !coli_st_streaming_direct_available_rep(state->index, shard, rep))+ return -1;+ size_t length = (size_t)bytes;+ if (dest == 0)+ return v4_read_direct_window(state, shard, rep, slot->slab, offset,+ length, 0);+ return v4_read_direct_copy(state, shard, rep, slot->slab + dest, offset,+ length);+}++static int v4_read_per_matrix_segment(V4ExpertStoreState *state, int shard,+ int rep, V4ExpertSlot *slot,+ uint64_t dest, uint64_t offset,+ uint64_t bytes, int *used_direct,+ int *used_fallback) {+ if (!v4_try_direct_segment(state, shard, rep, slot, dest, offset, bytes)) {+ *used_direct = 1;+ return 0;+ }+ if (slot->aligned_slab &&+ coli_st_streaming_direct_available_rep(state->index, shard, rep))+ *used_fallback = 1;+ return coli_st_read_at_rep(state->index, shard, rep, offset, (size_t)bytes,+ slot->slab + dest);+}+static int v4_read_expert_record(V4ExpertStoreState *state,const V4ExpertRecord *record,V4ExpertSlot *slot, int rep) {if (record->per_matrix) {- int direct_available = slot->aligned_slab &&- coli_st_streaming_direct_available_rep(state->index, record->m_scale_shard[0], rep);- if (direct_available)- __atomic_fetch_add(&v4_direct_fallbacks, UINT64_C(1),- __ATOMIC_RELAXED);+ int used_direct = 0;+ int used_fallback = 0;uint64_t scale_cursor = 0;for (int matrix = 0; matrix < V4_MATRIX_COUNT; matrix++) {- if (coli_st_read_at_rep(state->index, record->m_scale_shard[matrix], rep,- record->m_scale_offset[matrix],- (size_t)record->m_scale_bytes[matrix],- slot->slab + scale_cursor) != 0)+ if (v4_read_per_matrix_segment(+ state, record->m_scale_shard[matrix], rep, slot,+ scale_cursor, record->m_scale_offset[matrix],+ record->m_scale_bytes[matrix], &used_direct,+ &used_fallback) != 0)return -1;scale_cursor += record->m_scale_bytes[matrix];}uint64_t weight_cursor = scale_cursor;for (int matrix = 0; matrix < V4_MATRIX_COUNT; matrix++) {- if (coli_st_read_at_rep(state->index, record->m_weight_shard[matrix], rep,- record->m_weight_offset[matrix],- (size_t)record->m_weight_bytes[matrix],- slot->slab + weight_cursor) != 0)+ if (v4_read_per_matrix_segment(+ state, record->m_weight_shard[matrix], rep, slot,+ weight_cursor, record->m_weight_offset[matrix],+ record->m_weight_bytes[matrix], &used_direct,+ &used_fallback) != 0)return -1;weight_cursor += record->m_weight_bytes[matrix];}+ if (used_direct && !used_fallback) {+ __atomic_fetch_add(&v4_direct_reads, UINT64_C(1), __ATOMIC_RELAXED);+ __atomic_fetch_add(&v4_direct_payload_bytes, record->record_bytes,+ __ATOMIC_RELAXED);+ } else if (used_fallback) {+ __atomic_fetch_add(&v4_direct_fallbacks, UINT64_C(1),+ __ATOMIC_RELAXED);+ }return 0;}int direct_available = slot->aligned_slab &&@@ -8764,6 +8819,37 @@ int coli_v4_test_expert_slot_index(ColiExpertStore *store, ColiExpertKey key) {pthread_mutex_unlock(&state->mutex);return result;}++void coli_v4_test_reset_direct_io_stats(void) {+ __atomic_store_n(&v4_direct_reads, 0, __ATOMIC_RELAXED);+ __atomic_store_n(&v4_direct_flock_reads, 0, __ATOMIC_RELAXED);+ __atomic_store_n(&v4_direct_payload_bytes, 0, __ATOMIC_RELAXED);+ __atomic_store_n(&v4_direct_fallbacks, 0, __ATOMIC_RELAXED);+}++uint64_t coli_v4_test_direct_reads(void) {+ return __atomic_load_n(&v4_direct_reads, __ATOMIC_RELAXED);+}++uint64_t coli_v4_test_direct_fallbacks(void) {+ return __atomic_load_n(&v4_direct_fallbacks, __ATOMIC_RELAXED);+}++int coli_v4_test_force_streaming_direct(ColiExpertStore *store) {+ if (!store || !store->state) return -1;+ V4ExpertStoreState *state = store->state;+ if (!state->index) return -1;+ int enabled = 0;+ for (int i = 0; i < state->index->nfd; i++) {+ if (state->index->dfds[i] < 0 && state->index->fds[i] >= 0) {+ int twin = dup(state->index->fds[i]);+ if (twin < 0) return -1;+ state->index->dfds[i] = twin;+ }+ if (state->index->dfds[i] >= 0) enabled = 1;+ }+ return enabled ? 0 : -1;+}#endif/* Let the layer currently sweeping a batched CPU prefill borrow the completediff --git a/c/deepseek_v4_internal.h b/c/deepseek_v4_internal.hindex 2bcdf9cd..bfba3108 100644--- a/c/deepseek_v4_internal.h+++ b/c/deepseek_v4_internal.h@@ -1002,6 +1002,12 @@ extern void (*coli_v4_test_expert_wait_hook)(ColiExpertKey key);extern uint64_t coli_v4_test_fp4_batch_calls;extern uint64_t coli_v4_test_expert_victim_probes;int coli_v4_test_expert_slot_index(ColiExpertStore *store, ColiExpertKey key);+void coli_v4_test_reset_direct_io_stats(void);+uint64_t coli_v4_test_direct_reads(void);+uint64_t coli_v4_test_direct_fallbacks(void);+/* Point missing O_DIRECT twins at a dup of the buffered fd so tests can+ * exercise the direct-window path on filesystems that refuse O_DIRECT. */+int coli_v4_test_force_streaming_direct(ColiExpertStore *store);ColiV4Session *coli_v4_test_session_bare_create(ColiV4Engine *engine);void coli_v4_test_session_bare_destroy(ColiV4Session *session);diff --git a/c/tests/test_deepseek_v4.c b/c/tests/test_deepseek_v4.cindex 55224b82..7dd6b829 100644--- a/c/tests/test_deepseek_v4.c+++ b/c/tests/test_deepseek_v4.c@@ -1063,6 +1063,163 @@ static int test_expert_store_miss_scaling(void) {"(44/104/208 slots=1 probe each)");return 0;}++/* REAP-style layout: per-matrix scale/weight pairs so contiguous_group()+ * fails and build_record() sets per_matrix=1. Payload is padded past 4 KiB+ * so an O_DIRECT window has a real aligned pread. */+static int write_per_matrix_fixture(const char *path, int expert_count) {+ static const char *matrix_names[3] = {"w1", "w2", "w3"};+ if (expert_count < 1) return -1;+ size_t header_capacity = 256 + (size_t)expert_count * 1024;+ char *header = malloc(header_capacity);+ size_t payload_size = 8192;+ unsigned char *payload = malloc(payload_size);+ if (!header || !payload) {+ free(payload);+ free(header);+ return -1;+ }+ size_t used = (size_t)snprintf(header, header_capacity, "{");+ size_t cursor = 0;+ for (int expert = 0; expert < expert_count; expert++) {+ for (int matrix = 0; matrix < 3; matrix++) {+ size_t scale = cursor;+ size_t weight = cursor + 1;+ int count = snprintf(+ header + used, header_capacity - used,+ "%s\"layers.0.ffn.experts.%d.%s.scale\":{"+ "\"dtype\":\"F8_E8M0\",\"shape\":[1,1],"+ "\"data_offsets\":[%zu,%zu]}",+ used > 1 ? "," : "", expert, matrix_names[matrix],+ scale, scale + 1);+ if (count < 0 || (size_t)count >= header_capacity - used) {+ free(payload); free(header); return -1;+ }+ used += (size_t)count;+ count = snprintf(+ header + used, header_capacity - used,+ ",\"layers.0.ffn.experts.%d.%s.weight\":{"+ "\"dtype\":\"I8\",\"shape\":[1,16],"+ "\"data_offsets\":[%zu,%zu]}",+ expert, matrix_names[matrix], weight, weight + 16);+ if (count < 0 || (size_t)count >= header_capacity - used) {+ free(payload); free(header); return -1;+ }+ used += (size_t)count;+ cursor += 17;+ }+ }+ if (used + 1 >= header_capacity) {+ free(payload); free(header); return -1;+ }+ header[used++] = '}';+ header[used] = '\0';+ for (size_t i = 0; i < payload_size; i++)+ payload[i] = (unsigned char)i;+ uint64_t header_length = used;+ int fd = open(path, O_CREAT | O_TRUNC | O_WRONLY | COMPAT_O_BINARY, 0600);+ if (fd < 0) {+ free(payload); free(header); return -1;+ }+ int result = write_all(fd, &header_length, sizeof(header_length)) ||+ write_all(fd, header, (size_t)header_length) ||+ write_all(fd, payload, payload_size);+ close(fd);+ free(payload);+ free(header);+ return result ? -1 : 0;+}++static int expect_per_matrix_expert(const ColiExpertView *view, int expert) {+ int base = expert * 51;+ return view->gate.format == COLI_TENSOR_FP4_NATIVE_BLOCK &&+ view->gate.rows == 1 && view->gate.columns == 32 &&+ view->gate.data_bytes == 16 && view->gate.scale_bytes == 1 &&+ view->down.data_bytes == 16 && view->up.data_bytes == 16 &&+ ((const unsigned char *)view->gate.scales)[0] ==+ (unsigned char)base &&+ ((const unsigned char *)view->down.scales)[0] ==+ (unsigned char)(base + 17) &&+ ((const unsigned char *)view->up.scales)[0] ==+ (unsigned char)(base + 34) &&+ ((const unsigned char *)view->gate.data)[0] ==+ (unsigned char)(base + 1) &&+ ((const unsigned char *)view->down.data)[0] ==+ (unsigned char)(base + 18) &&+ ((const unsigned char *)view->up.data)[0] ==+ (unsigned char)(base + 35);+}++static int test_expert_store_per_matrix_direct(void) {+ /* Would fail this test: per_matrix always increments v4_direct_fallbacks+ * and never calls v4_read_direct_window even when a direct twin exists. */+ char directory[] = "colibri-v4-reap-XXXXXX";+ char path[256], error[256];+ setenv("COLI_V4_AUTOPIN", "0", 1);+ setenv("COLI_V4_SAVE_USAGE", "0", 1);+ setenv("COLI_V4_ROWS16", "0", 1);+ setenv("COLI_V4_PREWARM", "0", 1);+ unsetenv("COLI_V4_DIRECT");+ if (!mkdtemp(directory)) { perror("mkdtemp per_matrix"); return 1; }+ snprintf(path, sizeof(path), "%s/model.safetensors", directory);+ if (write_per_matrix_fixture(path, 2) != 0) {+ perror("write_per_matrix_fixture");+ rmdir(directory);+ return 1;+ }++ ColiDeepSeekV4ExpertStoreOptions options = {+ directory, 1, 2, 102, -1, 0, 0+ };+ ColiExpertStore *store = NULL;+ if (coli_deepseek_v4_expert_store_open(&options, &store,+ error, sizeof(error)) != 0) {+ fprintf(stderr, "%s\n", error);+ unlink(path); rmdir(directory);+ return 1;+ }+ if (coli_v4_test_force_streaming_direct(store) != 0) {+ fprintf(stderr, "per_matrix direct: no streaming-direct fd\n");+ store->ops->destroy(store);+ unlink(path); rmdir(directory);+ return 1;+ }+ coli_v4_test_reset_direct_io_stats();+ ColiExpertView view;+ if (coli_expert_lookup(store, (ColiExpertKey){0, 0}, &view) != 0) {+ fprintf(stderr, "per_matrix lookup failed\n");+ store->ops->destroy(store);+ unlink(path); rmdir(directory);+ return 1;+ }+ if (!expect_per_matrix_expert(&view, 0)) {+ fprintf(stderr,+ "per_matrix slab mismatch: scales=%u/%u/%u weights=%u/%u/%u\n",+ ((const unsigned char *)view.gate.scales)[0],+ ((const unsigned char *)view.down.scales)[0],+ ((const unsigned char *)view.up.scales)[0],+ ((const unsigned char *)view.gate.data)[0],+ ((const unsigned char *)view.down.data)[0],+ ((const unsigned char *)view.up.data)[0]);+ coli_expert_release(store, &view);+ store->ops->destroy(store);+ unlink(path); rmdir(directory);+ return 1;+ }+ coli_expert_release(store, &view);+ uint64_t reads = coli_v4_test_direct_reads();+ uint64_t fallbacks = coli_v4_test_direct_fallbacks();+ store->ops->destroy(store);+ if (scratch_remove(directory, path)) return 1;+ if (reads < 1 || fallbacks != 0) {+ fprintf(stderr,+ "per_matrix direct path unused: reads=%llu fallbacks=%llu\n",+ (unsigned long long)reads, (unsigned long long)fallbacks);+ return 1;+ }+ puts("DeepSeek-V4 ExpertStore per_matrix direct: ok");+ return 0;+}/* ==== end test_deepseek_v4_expert_store.c ==== *//* ==== begin test_deepseek_v4_kv_cache.c ==== */@@ -1531,6 +1688,10 @@ int main(int argc, char **argv) {fprintf(stderr, "FAIL: test_expert_store_miss_scaling\n");return 1;}+ if (test_expert_store_per_matrix_direct() != 0) {+ fprintf(stderr, "FAIL: test_expert_store_per_matrix_direct\n");+ return 1;+ }if (test_kv_cache() != 0) {fprintf(stderr, "FAIL: test_kv_cache\n");return 1;