Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(v4): pack hot pinned experts outside state->mutex (issue #900)

Gourav Kargwale53018cd8147
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.c
index 9d552d11..53f7a5aa 100644
--- a/c/deepseek_v4.c
+++ b/c/deepseek_v4.c
@@ -6112,6 +6112,76 @@ static int hot_pack_slot_locked(V4HotPolicy *policy,
#endif
}
+/* Issue #900: the FP4 rows16 packing is CPU-heavy (~5-6 s across a batched
+ * prefill) and used to run while holding state->mutex, serialising every
+ * other expert fetch on every OMP thread. The slot already holds our
+ * reference here, so its slab is stable and the packing can run lock-free.
+ * We only need the mutex for the final copy into the slab plus the packed[]
+ * flag flip, which hot_pack_slot_commit performs. Returns an owned buffer
+ * (free()'d by hot_pack_slot_commit) or NULL when packing is skipped or
+ * fails. Call outside state->mutex. */
+static unsigned char *hot_pack_slot_prepare(const V4ExpertRecord *record,
+ V4ExpertSlot *slot) {
+#ifndef COLI_FP4_ROWS16_KERNEL
+ (void)record; (void)slot; return NULL;
+#else
+ ColiTensorView gate, down, up;
+ fill_tensor_view(&gate, record, slot, V4_W1);
+ fill_tensor_view(&down, record, slot, V4_W2);
+ fill_tensor_view(&up, record, slot, V4_W3);
+ const ColiTensorView *views[3] = {&gate, &down, &up};
+ size_t total = 0;
+ for (int i = 0; i < 3; i++)
+ total += views[i]->data_bytes + views[i]->scale_bytes;
+ unsigned char *buf = malloc(total);
+ if (!buf) return NULL;
+ size_t off = 0;
+ for (int i = 0; i < 3; i++) {
+ unsigned char *packed_data = buf + off;
+ unsigned char *packed_scales = buf + off + views[i]->data_bytes;
+ if (coli_fp4_pack_rows16_v10(packed_data, packed_scales, views[i])) {
+ free(buf);
+ return NULL;
+ }
+ off += views[i]->data_bytes + views[i]->scale_bytes;
+ }
+ return buf;
+#endif
+}
+
+/* state->mutex held. Copies the prepared packed buffers into the slot slab
+ * and marks the slot packed, so the (slab layout, packed flag) pair stays
+ * consistent for any reader. Frees buf. No-op when buf is NULL. */
+static void hot_pack_slot_commit(V4HotPolicy *policy,
+ V4ExpertStoreState *state,
+ const V4ExpertRecord *record,
+ V4ExpertSlot *slot,
+ unsigned char *buf) {
+#ifndef COLI_FP4_ROWS16_KERNEL
+ (void)policy; (void)state; (void)record; (void)slot; (void)buf;
+#else
+ if (!buf) return;
+ ColiTensorView gate, down, up;
+ fill_tensor_view(&gate, record, slot, V4_W1);
+ fill_tensor_view(&down, record, slot, V4_W2);
+ fill_tensor_view(&up, record, slot, V4_W3);
+ const ColiTensorView *views[3] = {&gate, &down, &up};
+ size_t off = 0;
+ for (int i = 0; i < 3; i++) {
+ memcpy((void *)views[i]->data, buf + off, views[i]->data_bytes);
+ memcpy((void *)views[i]->scales, buf + off + views[i]->data_bytes,
+ views[i]->scale_bytes);
+ off += views[i]->data_bytes + views[i]->scale_bytes;
+ }
+ size_t slot_index = hot_slot_index(state, slot);
+ if (!policy->packed[slot_index]) {
+ policy->packed[slot_index] = 1;
+ policy->packed_slots++;
+ }
+ free(buf);
+#endif
+}
+
static void hot_repin_locked(V4HotPolicy *policy, V4ExpertStoreState *state,
int layer) {
if (!policy || policy->pin_count < 1) return;
@@ -6225,6 +6295,12 @@ static int lookup_hot(ColiExpertStore *store, ColiExpertKey key,
int read_result = v4_read_expert_record(state, record, slot);
struct timespec disk_t1;
clock_gettime(CLOCK_MONOTONIC, &disk_t1);
+ /* Pack outside the mutex: the slot holds our reference so its slab is
+ * stable, and the heavy FP4 rows16 work must not serialise other
+ * fetches (issue #900). Committed under the lock below. */
+ unsigned char *v4_pack_buf = NULL;
+ if (hot_is_pinned(policy, key.layer, key.expert))
+ v4_pack_buf = hot_pack_slot_prepare(record, slot);
pthread_mutex_lock(&state->mutex);
state->disk_sec +=
(double)(disk_t1.tv_sec - disk_t0.tv_sec) +
@@ -6232,14 +6308,15 @@ static int lookup_hot(ColiExpertStore *store, ColiExpertKey key,
if (read_result) {
slot->references = 0; slot->expert = -1;
if (state->active_leases) state->active_leases--;
+ free(v4_pack_buf);
pthread_mutex_unlock(&state->mutex);
memset(view, 0, sizeof(*view));
return -1;
}
slot->expert = key.expert; slot->used = ++state->clock;
state->stats.misses++; state->stats.bytes_read += record->record_bytes;
- if (hot_is_pinned(policy, key.layer, key.expert))
- hot_pack_slot_locked(policy, state, record, slot);
+ hot_pack_slot_commit(policy, state, record, slot, v4_pack_buf);
+ v4_pack_buf = NULL;
memset(view, 0, sizeof(*view)); view->key = key;
hot_fill_view(&view->gate, record, slot, V4_W1, policy, state);
hot_fill_view(&view->down, record, slot, V4_W2, policy, state);