Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
perf(deepseek-v41): a step's expert misses go to the device together
JustVugg871d793dd93d
diff --git a/c/deepseek_v41.c b/c/deepseek_v41.cindex 7f5c54fa..bce4be39 100644--- a/c/deepseek_v41.c+++ b/c/deepseek_v41.c@@ -786,28 +786,81 @@ static void rms_into(float *out, const float *x, const float *weight, int n, flo/* --------------------------------------------------------- expert cache ---- */-static void expert_read(Model *m, const char *kind, int layer, int eid, Slot *s) {+/* One tensor still to fetch: which one, and where it lands. An expert is six of+ * these -- three weight matrices and their three scale sidecars. */+typedef struct { char name[160]; uint8_t *dest; int64_t size; } ExpertRead;++#define V41_EXPERT_TENSORS 6++/* Fill in the six reads that make up one expert. Naming them into a list instead+ * of reading them on the spot is the whole point: the list is what lets a layer's+ * misses go to the device together rather than one behind another. */+static int expert_read_list(Model *m, const char *kind, int layer, int eid, Slot *s,+ ExpertRead *out) {Cfg *c = &m->c;- char name[256];int64_t packed_gate = (int64_t)c->moe_inter * c->dim / 2;int64_t packed_down = (int64_t)c->dim * c->moe_inter / 2;int64_t scales_gate = (int64_t)c->moe_inter * (c->dim / 32);int64_t scales_down = (int64_t)c->dim * (c->moe_inter / 32);- double started = now_s();struct { const char *suffix; uint8_t *bytes; int64_t size; uint8_t *scale; int64_t scale_size; } part[3] = {{ "w1", s->w1, packed_gate, s->s1, scales_gate },{ "w3", s->w3, packed_gate, s->s3, scales_gate },{ "w2", s->w2, packed_down, s->s2, scales_down },};+ int n = 0;for (int i = 0; i < 3; i++) {- snprintf(name, sizeof(name), "%s.%d.ffn.experts.%d.%s.weight", kind, layer, eid, part[i].suffix);- st_read_raw_cap(&m->S, name, part[i].bytes, part[i].size, 1);- snprintf(name, sizeof(name), "%s.%d.ffn.experts.%d.%s.scale", kind, layer, eid, part[i].suffix);- st_read_raw_cap(&m->S, name, part[i].scale, part[i].scale_size, 1);- m->expert_bytes += part[i].size + part[i].scale_size;+ snprintf(out[n].name, sizeof(out[n].name), "%s.%d.ffn.experts.%d.%s.weight",+ kind, layer, eid, part[i].suffix);+ out[n].dest = part[i].bytes; out[n].size = part[i].size; n++;+ snprintf(out[n].name, sizeof(out[n].name), "%s.%d.ffn.experts.%d.%s.scale",+ kind, layer, eid, part[i].suffix);+ out[n].dest = part[i].scale; out[n].size = part[i].scale_size; n++;}- s->eid = eid;+ return n;+}++/* How many expert tensors to keep in flight. One at a time is what the engine did+ * and it leaves most of the device on the floor: measured on the released+ * checkpoint, 88.5 GB of expert reads landed at 2.40 GB/s, while the same blocks+ * through the same page cache go at 6.2 GB/s with one reader and 12.6 GB/s with+ * four (c/iobench.c, 5.6 MB blocks, caches dropped). A striped NVMe pair answers+ * several requests at once; a single blocking pread never asks it to.+ * V41_READ_DEPTH=1 restores the serial path for an A/B. */+static int expert_read_depth(void) {+ static int depth = -1;+ if (depth < 0) {+ const char *env = getenv("V41_READ_DEPTH");+ depth = env ? atoi(env) : 8;+ if (depth < 1) depth = 1;+ }+ return depth;+}++/* Read `n` reserved experts at once. The slot ids are stamped only after every+ * read has landed, so a slot never advertises an expert it does not yet hold. */+static void expert_fetch(Model *m, const char *kind, int layer,+ Slot *const *slot, const int *eid, int n) {+ if (n <= 0) return;+ int total = 0;+ ExpertRead *list = xmalloc((size_t)n * V41_EXPERT_TENSORS * sizeof(ExpertRead),+ "expert read list");+ for (int i = 0; i < n; i++)+ total += expert_read_list(m, kind, layer, eid[i], slot[i], list + total);+ double started = now_s();+ int threads = expert_read_depth();+ if (threads > total) threads = total;+ uint64_t bytes = 0;+#if defined(_OPENMP)+ #pragma omp parallel for schedule(dynamic, 1) num_threads(threads) reduction(+:bytes)+#endif+ for (int i = 0; i < total; i++) {+ st_read_raw_cap(&m->S, list[i].name, list[i].dest, list[i].size, 1);+ bytes += (uint64_t)list[i].size;+ }+ m->expert_bytes += bytes;m->t_disk += now_s() - started;+ for (int i = 0; i < n; i++) slot[i]->eid = eid[i];+ free(list);}/* One byte per expert: routed in this turn or not, for the dashboard's HITS line. */@@ -859,13 +912,56 @@ static Slot *expert_slot_at(Model *m, LCache *cache, const char *kind, int layerif (cache->slot[i].used < cache->slot[oldest].used) oldest = i;victim = &cache->slot[oldest];}- expert_read(m, kind, layer, eid, victim);victim->used = ++m->clock;+ expert_fetch(m, kind, layer, &victim, &eid, 1);return victim;}-static Slot *expert_slot(Model *m, int layer, int eid) {- return expert_slot_at(m, &m->cache[layer], "layers", layer, eid);+/* Resolve a whole step's routed experts to slots WITHOUT reading any of them, then+ * fetch the misses in one batch. Reserving first is what makes the batch possible:+ * a reserved victim is stamped with the newest clock and its id cleared, so a later+ * reservation in the same step neither evicts it again nor mistakes it for a hit.+ *+ * The argument holds only while the cache has room for the whole step (cap >= topk):+ * at reservation k at most k slots have been touched, so with cap > k there is+ * always an untouched one left to evict, and the untouched ones are exactly the+ * ones with an older clock. A cache smaller than topk has no such slot -- the+ * caller keeps the one-at-a-time path for that case. */+static void expert_slots_at(Model *m, LCache *cache, const char *kind, int layer,+ const int *chosen, int topk, Slot **out) {+ int miss_eid[64];+ Slot *miss_slot[64];+ int misses = 0;+ for (int k = 0; k < topk; k++) {+ int eid = chosen[k];+ if (!strcmp(kind, "layers")) ehit_mark(m, layer, eid);+ Slot *found = NULL;+ for (int i = 0; i < cache->n; i++)+ if (cache->slot[i].eid == eid) { found = &cache->slot[i]; break; }+ if (found) {+ found->used = ++m->clock;+ m->hits++;+ out[k] = found;+ continue;+ }+ m->miss++;+ Slot *victim;+ if (cache->n < cache->cap) {+ victim = &cache->slot[cache->n++];+ } else {+ int oldest = 0;+ for (int i = 1; i < cache->n; i++)+ if (cache->slot[i].used < cache->slot[oldest].used) oldest = i;+ victim = &cache->slot[oldest];+ }+ victim->used = ++m->clock;+ victim->eid = -1; /* not this expert yet: the read is still pending */+ out[k] = victim;+ miss_slot[misses] = victim;+ miss_eid[misses] = eid;+ misses++;+ }+ expert_fetch(m, kind, layer, miss_slot, miss_eid, misses);}/* ------------------------------------------------------------- loading ----- */@@ -1650,9 +1746,22 @@ static void moe_run_at(Model *m, Layer *l, LCache *cache, const char *kind, int}for (int k = 0; k < topk; k++) weights[k] *= c->route_scale;memset(out, 0, (size_t)c->dim * sizeof(float));- for (int k = 0; k < topk; k++) {- Slot *s = expert_slot_at(m, cache, kind, layer, chosen[k]);- expert_ffn(m, s->w1, s->s1, s->w3, s->s3, s->w2, s->s2, x, out, weights[k]);+ if (cache->cap >= topk) {+ /* Reserve every slot first, read the misses together, then compute. The+ * accumulation into `out` keeps its rank order, so this is the same+ * arithmetic in the same sequence -- only the reads moved. */+ Slot *slot[64];+ expert_slots_at(m, cache, kind, layer, chosen, topk, slot);+ for (int k = 0; k < topk; k++)+ expert_ffn(m, slot[k]->w1, slot[k]->s1, slot[k]->w3, slot[k]->s3,+ slot[k]->w2, slot[k]->s2, x, out, weights[k]);+ } else {+ /* A cache too small to hold one step's experts evicts one of them to make+ * room for the next, so the slots cannot all be live at once. */+ for (int k = 0; k < topk; k++) {+ Slot *s = expert_slot_at(m, cache, kind, layer, chosen[k]);+ expert_ffn(m, s->w1, s->s1, s->w3, s->s3, s->w2, s->s2, x, out, weights[k]);+ }}shared_ffn(m, l, x, out);free(scores);@@ -2799,6 +2908,7 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *snap) {double turn_started = now_s();double disk0 = m->t_disk, expert0 = m->t_expert, attn0 = m->t_attn, engram0 = m->t_engram;uint64_t forwards0 = m->forwards, hits0 = m->hits, miss0 = m->miss;+ uint64_t ebytes0 = m->expert_bytes;model_reset(m);int n_prompt = tok_encode(tokenizer, (const char *)command.payload,(int)command.payload_bytes, ids, c->max_positions);@@ -2847,6 +2957,8 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *snap) {free(pending_image); pending_image = NULL;forward_with_image(m, ids, n_prompt, logits, aligned, image_at, image_h, image_w, image_mask);free(aligned); free(image_mask);+ uint64_t prefill_bytes = m->expert_bytes - ebytes0;+ double prefill_disk = m->t_disk - disk0;int emitted = 0, limited = 1, cancelled = 0, done_early = 0;char piece[512];@@ -2926,6 +3038,20 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *snap) {tab->misses * (double)(m->engram.head_dim + m->engram.head_dim / 32) / 1e6);}}+ {+ /* Expert streaming, per turn: how many bytes the LRU had to fetch and+ * what rate the reads actually achieved. The rate is the number that+ * matters -- the device is worth several GB/s, so a figure far under+ * that says the loads are serialised, not that the disk is slow. */+ uint64_t turn_bytes = m->expert_bytes - ebytes0;+ double seconds = m->t_disk - disk0;+ if (turn_bytes)+ fprintf(stderr, "[v41] expert I/O: %llu misses, %.1f MB in %.2fs (%.2f GB/s); "+ "prefill %.1f MB in %.2fs\n",+ (unsigned long long)(m->miss - miss0), turn_bytes / 1e6, seconds,+ seconds > 0 ? turn_bytes / 1e9 / seconds : 0.0,+ prefill_bytes / 1e6, prefill_disk);+ }if (m->spec.active && m->spec.proposed > proposed0)fprintf(stderr, "[v41] DSpark: %llu of %llu drafts accepted this turn\n",(unsigned long long)(m->spec.accepted - accepted0),diff --git a/docs/deepseek-v41.md b/docs/deepseek-v41.mdindex 8fd983a8..333d8cc6 100644--- a/docs/deepseek-v41.md+++ b/docs/deepseek-v41.md@@ -146,10 +146,49 @@ reading the architecture implies instead -- each layer against its own owner's kThat is a different model, not a bug fix, and the CI asserts the two disagree so thedefault cannot be "tidied" by accident.+## Streaming the experts++Six of 384 experts are routed per layer per position, and each is 18.8 MB of fp4+weights plus ue8m0 scales. A turn of 26 prompt tokens and 24 generated ones moves+88.5 GB through the expert cache on the released checkpoint, so how fast those bytes+arrive is not a detail of the engine, it is most of the engine.++The reads go out together. A step's routed experts are all resolved to cache slots+first -- hits stamped, misses given a reserved victim -- and only then are the missing+tensors read, `V41_READ_DEPTH` of them at a time. Reserving before reading is what+makes the batch possible: a reserved victim carries the newest clock and a cleared id,+so a later reservation in the same step neither evicts it again nor mistakes it for a+hit. The arithmetic is untouched -- the experts are still accumulated in rank order --+and a cache too small to hold one step's experts (`cap < topk`) keeps the old+one-at-a-time path, because there the slots genuinely cannot all be live at once.++Measured on the released checkpoint, cold (page cache dropped before every run), same+prompt, same seed, on a striped pair of Samsung PM9A1:++| read depth | expert I/O | rate | wall | tok/s |+|---|---|---|---|---|+| 1 | 35.8 s | 2.47 GB/s | 67.0 s | 0.358 |+| 4 | 15.0 s | 5.91 GB/s | 45.0 s | 0.533 |+| 8 (default) | 12.9 / 13.0 s | 6.84 / 6.81 GB/s | 43.1 / 42.4 s | 0.557 / 0.566 |+| 12 | 13.1 s | 6.77 GB/s | 43.4 s | 0.554 |+| 16 | 13.5 s | 6.57 GB/s | 47.5 s | 0.506 |++Every row reads the same 88.5 GB in the same 4709 misses and emits the same 24 tokens;+only the depth changes. Past eight the device stops answering faster and the reader+threads start taking cores away from the matmuls, which is why the wall turns back up+while the rate barely moves. The depth-8 row is two independent cold runs, to show how+much of the wall is run-to-run noise: the I/O figures repeat to within half a percent,+the compute figures do not.++Prefill is where most of it goes: 54.6 GB of the 88.5, because each of the 26 prompt+positions routes independently and a 49-slot cache cannot hold what 26 positions ask+of one layer.+## Environment| variable | default | what it does ||---|---|---|+| `V41_READ_DEPTH` | 8 | expert tensors read at once when a step misses the cache (see above). `1` restores the serial read the engine used to do, for an A/B. || `V41_ENGRAM_ROWS` | 65536 | rows of engram cache per table. The traffic is Zipfian: common 2-grams repeat constantly, so a small cache absorbs most of it. 65536 rows is 64 MB per table on the released head_dim. || `V41_INDEX_OWNER` | unset | each layer scores against its own owner's index keys (see above). Changes the model's behaviour. || `V41_MAX_IMAGE_TOKENS` | the checkpoint's `max_image_tokens` | a ceiling on what one image costs in prompt tokens. |