Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

perf(dsv4): batch shared experts during prefill

JustVugga2f9e3f18d44
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.c
index 03b7eddb..ec942d3f 100644
--- a/c/deepseek_v4.c
+++ b/c/deepseek_v4.c
@@ -5296,6 +5296,63 @@ static int v4_apply_expert_batch(
batch_outputs, count, dimension, swiglu_limit);
}
+static int v4_shared_batch_enabled(void) {
+ const char *setting = getenv("COLI_V4_SHARED_BATCH");
+ return !setting || atoi(setting) != 0;
+}
+
+/* Shared-expert prefill has the same weights for every position. Running a
+ * matvec per item reloads those dense FP8 matrices up to 128 times; the native
+ * batch kernel keeps each matrix tile hot while preserving each item's scalar
+ * column-accumulation order. */
+static int v4_shared_expert_forward_batch_ref(
+ float *outputs, const ColiTensorView *gate_weight,
+ const ColiTensorView *down_weight, const ColiTensorView *up_weight,
+ const float *inputs, int batch, float swiglu_limit) {
+ if (!outputs || !gate_weight || !down_weight || !up_weight || !inputs ||
+ batch < 1 || batch > V4_EXPERT_BATCH_MAX || swiglu_limit < 0.0f ||
+ gate_weight->format != COLI_TENSOR_FP8_E4M3_BLOCK ||
+ down_weight->format != COLI_TENSOR_FP8_E4M3_BLOCK ||
+ up_weight->format != COLI_TENSOR_FP8_E4M3_BLOCK ||
+ gate_weight->rows != up_weight->rows ||
+ gate_weight->columns != up_weight->columns ||
+ down_weight->columns != gate_weight->rows ||
+ down_weight->rows != gate_weight->columns)
+ return -1;
+ size_t intermediate = (size_t)gate_weight->rows;
+ if ((size_t)batch > SIZE_MAX / intermediate) return -1;
+ size_t cells = (size_t)batch * intermediate;
+ if (cells > SIZE_MAX / (3 * sizeof(float))) return -1;
+ float *workspace = malloc(3 * cells * sizeof(*workspace));
+ if (!workspace) return -1;
+ float *gate = workspace;
+ float *up = gate + cells;
+ float *activated = up + cells;
+ int result = coli_fp8_matmul_batch_ref(
+ gate, gate_weight, inputs, batch);
+ if (!result)
+ result = coli_fp8_matmul_batch_ref(
+ up, up_weight, inputs, batch);
+ for (int item = 0; !result && item < batch; item++) {
+ float *item_gate = gate + (size_t)item * intermediate;
+ float *item_up = up + (size_t)item * intermediate;
+ float *item_activated = activated + (size_t)item * intermediate;
+ coli_bf16_round_array(item_gate, intermediate);
+ coli_bf16_round_array(item_up, intermediate);
+ result = coli_v4_swiglu(item_activated, item_gate, item_up,
+ (int)intermediate, swiglu_limit);
+ if (!result) coli_bf16_round_array(item_activated, intermediate);
+ }
+ if (!result)
+ result = coli_fp8_matmul_batch_ref(
+ outputs, down_weight, activated, batch);
+ if (!result)
+ coli_bf16_round_array(
+ outputs, (size_t)batch * (size_t)down_weight->rows);
+ free(workspace);
+ return result ? -1 : 0;
+}
+
static int v4_moe_batch_union(
float *outputs, const ColiDeepSeekV4LayerWeights *weights,
const ColiDeepSeekV4Config *config, ColiExpertStore *store,
@@ -5377,10 +5434,14 @@ static int v4_moe_batch_union(
fp8_view(&w2, weights, "ffn.shared_experts.w2") ||
fp8_view(&w3, weights, "ffn.shared_experts.w3")))
result = -1;
- for (int item = 0; !result && item < batch; item++)
- result = coli_v4_shared_expert_forward_ref(
- shared + (size_t)item * d, &w1, &w2, &w3,
- inputs + (size_t)item * d, config->swiglu_limit);
+ if (!result && batch > 1 && v4_shared_batch_enabled())
+ result = v4_shared_expert_forward_batch_ref(
+ shared, &w1, &w2, &w3, inputs, batch, config->swiglu_limit);
+ else
+ for (int item = 0; !result && item < batch; item++)
+ result = coli_v4_shared_expert_forward_ref(
+ shared + (size_t)item * d, &w1, &w2, &w3,
+ inputs + (size_t)item * d, config->swiglu_limit);
if (!result)
memset(outputs, 0, (size_t)batch * d * sizeof(*outputs));