Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Merge remote-tracking branch 'origin/dev' into pr/v4-ssd-mirror

dcutugno6aa3069e93ff
diff --cc c/deepseek_v4.c
index 2586cd7b,9d552d11..551363f2
--- a/c/deepseek_v4.c
+++ b/c/deepseek_v4.c
@@@ -5799,15 -5535,20 +5825,22 @@@ static int lookup(ColiExpertStore *stor
}
/* A short read must never expose a partially overwritten old slot. */
slot->expert = -1;
+ /* DUAL-SSD: this expert's replica (deterministic per layer,eid). */
+ int rep = coli_st_expert_route(key.layer, key.expert);
+ struct timespec disk_t0;
+ clock_gettime(CLOCK_MONOTONIC, &disk_t0);
- if (coli_st_read_at_streaming(
- state->index, record->shard, record->scale_offset,
+ if (coli_st_read_at_streaming_rep(
+ state->index, record->shard, rep, record->scale_offset,
(size_t)record->scale_bytes, slot->slab) != 0 ||
- coli_st_read_at_streaming(
- state->index, record->shard, record->weight_offset,
+ coli_st_read_at_streaming_rep(
+ state->index, record->shard, rep, record->weight_offset,
(size_t)record->weight_bytes,
slot->slab + record->scale_bytes) != 0) {
+ struct timespec disk_t1;
+ clock_gettime(CLOCK_MONOTONIC, &disk_t1);
+ state->disk_sec +=
+ (double)(disk_t1.tv_sec - disk_t0.tv_sec) +
+ (disk_t1.tv_nsec - disk_t0.tv_nsec) * 1e-9;
pthread_mutex_unlock(&state->mutex);
memset(view, 0, sizeof(*view));
return -1;
@@@ -6460,9 -6220,15 +6526,16 @@@ static int lookup_hot(ColiExpertStore *
state->active_leases++;
slot->used = ++state->clock;
pthread_mutex_unlock(&state->mutex);
+ int rep = coli_st_expert_route(key.layer, key.expert);
+ struct timespec disk_t0;
+ clock_gettime(CLOCK_MONOTONIC, &disk_t0);
- int read_result = v4_read_expert_record(state, record, slot);
+ int read_result = v4_read_expert_record(state, record, slot, rep);
+ struct timespec disk_t1;
+ clock_gettime(CLOCK_MONOTONIC, &disk_t1);
pthread_mutex_lock(&state->mutex);
+ state->disk_sec +=
+ (double)(disk_t1.tv_sec - disk_t0.tv_sec) +
+ (disk_t1.tv_nsec - disk_t0.tv_nsec) * 1e-9;
if (read_result) {
slot->references = 0; slot->expert = -1;
if (state->active_leases) state->active_leases--;
@@@ -9124,24 -9100,17 +9407,36 @@@ static void v4_serve_one(ColiV4Engine *
hit_rate, v4_serve_rss_gb(), stats.prompt_tokens, length_limited,
session->prefix_reused);
fflush(stdout);
+ double expert_disk_s = engine->experts
+ ? coli_v4_expert_store_disk_sec(engine->experts) - disk_before
+ : 0.0;
+ double expert_matmul_s = engine->experts
+ ? coli_v4_expert_store_matmul_sec(engine->experts) - matmul_before
+ : 0.0;
+ v4_prof_emit(elapsed, stats.prompt_tokens, completion,
+ expert_disk_s, expert_matmul_s);
+ coli_v4_expert_store_emit_hits(engine->experts);
+ coli_v4_expert_store_emit_emap(engine->experts);
+ coli_v4_expert_store_emit_tiers(engine->experts);
+ /* DUAL-SSD: cumulative per-drive read split, once per turn. The CLI path
+ * prints this at exit; serve is long-lived, so surface it on stderr where
+ * the gateway tees it — it is the only way to see whether expert I/O
+ * actually aggregates both drives' bandwidth in production. */
+ if (coli_st_mirror_active()) {
+ /* One write: the gateway tees this stderr alongside its own lines,
+ * and piecewise fprintf interleaves mid-line under load. */
+ char line[256];
+ int at = snprintf(line, sizeof(line), "v4_mirror");
+ for (int r = 0; r < coli_st_mirror_nrep() && at < (int)sizeof(line); r++)
+ at += snprintf(line + at, sizeof(line) - at, " %s=%.2fGiB/%llu",
+ r ? "mirror" : "primary",
+ __atomic_load_n(&g_v4_mir_bytes[r], __ATOMIC_RELAXED)
+ / 1073741824.0,
+ (unsigned long long)__atomic_load_n(
+ &g_v4_mir_nread[r], __ATOMIC_RELAXED));
+ fprintf(stderr, "%s\n", line);
+ }
++
}
static int v4_serve_main(void) {
@@@ -10034,15 -10001,20 +10338,22 @@@ static int lookup(ColiExpertStore *stor
}
/* A short read must never expose a partially overwritten old slot. */
slot->expert = -1;
+ /* DUAL-SSD: this expert's replica (deterministic per layer,eid). */
+ int rep = coli_st_expert_route(key.layer, key.expert);
+ struct timespec disk_t0;
+ clock_gettime(CLOCK_MONOTONIC, &disk_t0);
- if (coli_st_read_at_streaming(
- state->index, record->shard, record->scale_offset,
+ if (coli_st_read_at_streaming_rep(
+ state->index, record->shard, rep, record->scale_offset,
(size_t)record->scale_bytes, slot->slab) != 0 ||
- coli_st_read_at_streaming(
- state->index, record->shard, record->weight_offset,
+ coli_st_read_at_streaming_rep(
+ state->index, record->shard, rep, record->weight_offset,
(size_t)record->weight_bytes,
slot->slab + record->scale_bytes) != 0) {
+ struct timespec disk_t1;
+ clock_gettime(CLOCK_MONOTONIC, &disk_t1);
+ state->disk_sec +=
+ (double)(disk_t1.tv_sec - disk_t0.tv_sec) +
+ (disk_t1.tv_nsec - disk_t0.tv_nsec) * 1e-9;
pthread_mutex_unlock(&state->mutex);
memset(view, 0, sizeof(*view));
return -1;