Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

deepseek_v4: PROF reports block time, head time and forwards

JustVuggf14ca10e0331
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.c
index 040c7261..977a96f8 100644
--- a/c/deepseek_v4.c
+++ b/c/deepseek_v4.c
@@ -11698,10 +11698,33 @@ static float head_bf16_dot(const uint16_t *weight, const float *hidden,
return sum;
}
+/* PROF phases beyond the expert store (#1491): a turn's /profile used to show
+ * expert disk and expert matmul and a literal zero for everything else, so a
+ * warm decode on a GPU box read as 98% "other". These accumulate the time
+ * spent in the layer blocks (attention, indexer, dense, mixers and the experts
+ * within), the time in the head, and the positions forwarded; v4_serve_one
+ * reports per-turn deltas. Timing only: no numeric path changes. */
+static double g_v4_prof_block_s = 0.0, g_v4_prof_head_s = 0.0;
+static long long g_v4_prof_forwards = 0;
+static double spec_now(void); /* defined with the speculative-decode helpers below */
+
+static int head_argmax_impl(ColiV4Engine *engine, const float *hidden,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config,
+ int *best_token, float *best_logit);
static int head_argmax(ColiV4Engine *engine, const float *hidden,
const ColiSafetensorsIndex *index,
const ColiDeepSeekV4Config *config,
int *best_token, float *best_logit) {
+ double t0 = spec_now();
+ int result = head_argmax_impl(engine, hidden, index, config, best_token, best_logit);
+ g_v4_prof_head_s += spec_now() - t0;
+ return result;
+}
+static int head_argmax_impl(ColiV4Engine *engine, const float *hidden,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config,
+ int *best_token, float *best_logit) {
const ColiSafetensorsTensor *head = coli_st_find(index, "head.weight");
int d = config->hidden_size, vocab = config->vocab_size;
if (!head || head->dtype != COLI_ST_BF16 || d < 1 || vocab < 1)
@@ -12108,6 +12131,14 @@ static int v4_prefill_pool_enabled(void) {
return enabled;
}
+static int target_batch_impl(ColiV4Engine *engine, float **state_ptr, float **next_ptr,
+ ColiDeepSeekV4WindowAttentionState **attention,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config,
+ ColiExpertStore *experts, const int *tokens,
+ int start, int batch, int use_prefill_pool,
+ ColiV4SessionAbortFn should_abort, void *abort_ctx,
+ char *error, size_t error_size);
static int target_batch(ColiV4Engine *engine, float **state_ptr, float **next_ptr,
ColiDeepSeekV4WindowAttentionState **attention,
const ColiSafetensorsIndex *index,
@@ -12116,6 +12147,22 @@ static int target_batch(ColiV4Engine *engine, float **state_ptr, float **next_pt
int start, int batch, int use_prefill_pool,
ColiV4SessionAbortFn should_abort, void *abort_ctx,
char *error, size_t error_size) {
+ double t0 = spec_now();
+ int result = target_batch_impl(engine, state_ptr, next_ptr, attention, index, config,
+ experts, tokens, start, batch, use_prefill_pool,
+ should_abort, abort_ctx, error, error_size);
+ g_v4_prof_block_s += spec_now() - t0;
+ if (batch > 0) g_v4_prof_forwards += batch;
+ return result;
+}
+static int target_batch_impl(ColiV4Engine *engine, float **state_ptr, float **next_ptr,
+ ColiDeepSeekV4WindowAttentionState **attention,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config,
+ ColiExpertStore *experts, const int *tokens,
+ int start, int batch, int use_prefill_pool,
+ ColiV4SessionAbortFn should_abort, void *abort_ctx,
+ char *error, size_t error_size) {
if (!state_ptr || !next_ptr || !*state_ptr || !*next_ptr || !attention ||
!index || !config || !experts || !tokens || start < 0 || batch < 1) {
if (error && error_size)
@@ -12198,12 +12245,31 @@ static int target_batch(ColiV4Engine *engine, float **state_ptr, float **next_pt
return 0;
}
+static int target_token_impl(ColiV4Engine *engine, float **state_ptr, float **next_ptr,
+ ColiDeepSeekV4WindowAttentionState **attention,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config,
+ ColiExpertStore *experts, int token, int position,
+ char *error, size_t error_size);
static int target_token(ColiV4Engine *engine, float **state_ptr, float **next_ptr,
ColiDeepSeekV4WindowAttentionState **attention,
const ColiSafetensorsIndex *index,
const ColiDeepSeekV4Config *config,
ColiExpertStore *experts, int token, int position,
char *error, size_t error_size) {
+ double t0 = spec_now();
+ int result = target_token_impl(engine, state_ptr, next_ptr, attention, index, config,
+ experts, token, position, error, error_size);
+ g_v4_prof_block_s += spec_now() - t0;
+ g_v4_prof_forwards += 1;
+ return result;
+}
+static int target_token_impl(ColiV4Engine *engine, float **state_ptr, float **next_ptr,
+ ColiDeepSeekV4WindowAttentionState **attention,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config,
+ ColiExpertStore *experts, int token, int position,
+ char *error, size_t error_size) {
float *state = *state_ptr, *next = *next_ptr;
if (load_embedding(state, index, config, token)) return -1;
for (int layer_id = 0; layer_id < config->num_hidden_layers; layer_id++) {
@@ -13858,15 +13924,23 @@ static void v4_hwinfo_emit(void) {
}
/* PROF wall_s prompt_tokens completion_tokens expert_disk_s expert_wait_s
- * expert_matmul_s attention_s lm_head_s forwards β€” disk (I/O) and matmul
- * (expert-forward compute) are the two phases the runtime tracks per turn
- * (#890); the frontend folds the remainder β€” attention, head, framing β€” into
- * "other". Before this the matmul field was hardcoded 0 and every turn read as
- * 100% other whenever the model sat warm in page cache. */
+ * expert_matmul_s attention_s lm_head_s forwards. disk (I/O) and matmul come
+ * from the expert store (#890). attention_s is the layer-block time not
+ * attributed to the expert compute: attention, DSA indexer, dense projections
+ * and hyper-connection mixers, plus any expert wait that blocked the block
+ * (disk seconds are summed across loader lanes and can exceed the wall on
+ * their own, so they are reported as they are and not subtracted; clamped at
+ * zero). lm_head_s is the head matmul; forwards the
+ * positions pushed through the blocks (prefill rows, decode tokens, draft
+ * verifies). Before #1491 the last three were literal zeros and a warm decode
+ * on a GPU box read as 98% "other". The frontend still folds what is left
+ * (sampling, framing) into "other". */
static void v4_prof_emit(double wall_s, int prompt_tokens, int completion,
- double expert_disk_s, double expert_matmul_s) {
- printf("PROF %.3f %d %d %.3f 0.000 %.3f 0.000 0.000 0\n",
- wall_s, prompt_tokens, completion, expert_disk_s, expert_matmul_s);
+ double expert_disk_s, double expert_matmul_s,
+ double attention_s, double head_s, long long forwards) {
+ printf("PROF %.3f %d %d %.3f 0.000 %.3f %.3f %.3f %lld\n",
+ wall_s, prompt_tokens, completion, expert_disk_s, expert_matmul_s,
+ attention_s, head_s, forwards);
fflush(stdout);
}
@@ -14053,6 +14127,8 @@ static int v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,
engine->experts ? coli_v4_expert_store_disk_sec(engine->experts) : 0.0;
double matmul_before =
engine->experts ? coli_v4_expert_store_matmul_sec(engine->experts) : 0.0;
+ double block_before = g_v4_prof_block_s, head_before = g_v4_prof_head_s;
+ long long forwards_before = g_v4_prof_forwards;
V4ServeStream stream = {session, request->id, 0, 0};
ColiV4SessionGenerateStats stats = {0};
char error[512] = {0};
@@ -14098,8 +14174,14 @@ static int v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,
double expert_matmul_s = engine->experts
? coli_v4_expert_store_matmul_sec(engine->experts) - matmul_before
: 0.0;
+ /* Block time minus the expert compute measured inside it. The store's disk
+ * seconds are NOT subtracted: summed across loader lanes, they exceeded the
+ * wall on a cold tiny run (0.073 s of disk in a 0.034 s turn). */
+ double attention_s = (g_v4_prof_block_s - block_before) - expert_matmul_s;
+ if (attention_s < 0.0) attention_s = 0.0;
v4_prof_emit(elapsed, stats.prompt_tokens, completion,
- expert_disk_s, expert_matmul_s);
+ expert_disk_s, expert_matmul_s, attention_s,
+ g_v4_prof_head_s - head_before, g_v4_prof_forwards - forwards_before);
#ifdef COLI_V4_GPU_TIER
if (coli_v4_hybrid_enabled() && (g_v4_hyb_gpu_n + g_v4_hyb_cpu_n +
g_v4_hyb_upload_n + g_v4_hyb_skip_n))
diff --git a/c/tests/test_deepseek_v4_tiny.py b/c/tests/test_deepseek_v4_tiny.py
index 37f769f4..3db8edaa 100644
--- a/c/tests/test_deepseek_v4_tiny.py
+++ b/c/tests/test_deepseek_v4_tiny.py
@@ -7,6 +7,7 @@ import argparse
import json
import os
import re
+import time
import subprocess
import sys
import tempfile
@@ -205,9 +206,33 @@ def check_serve(binary: Path, model: Path, case: dict[str, object]) -> None:
)
if stats["prompt_tokens"] != len(case["prompt_ids"]):
raise AssertionError(f"serve round {ordinal}: bad prompt stats {stats}")
+ # #1491: the PROF line carries the phases beyond the expert store.
+ # Before, attention_s / lm_head_s / forwards were literal zeros and a
+ # warm turn read as 98% "other" in /profile.
+ # PROF follows DONE on the engine's stdout; generate() returns at
+ # DONE, so give the reader thread a moment to parse the next line.
+ deadline = time.time() + 5.0
+ while len(engine.profile) <= ordinal and time.time() < deadline:
+ time.sleep(0.02)
+ if len(engine.profile) <= ordinal:
+ raise AssertionError(f"serve round {ordinal}: no PROF line")
+ prof = engine.profile[-1]
+ if prof["completion_tokens"] != stats["completion_tokens"]:
+ raise AssertionError(f"serve round {ordinal}: PROF/DONE disagree: {prof} {stats}")
+ if prof["forwards"] < prof["completion_tokens"]:
+ raise AssertionError(f"serve round {ordinal}: forwards {prof['forwards']} below "
+ f"the {prof['completion_tokens']} tokens generated")
+ # the tiny head is microseconds and prints as 0.000; the blocks are not
+ if prof["attention_s"] <= 0.0 or prof["lm_head_s"] < 0.0:
+ raise AssertionError(f"serve round {ordinal}: block/head phases not timed: {prof}")
+ # disk seconds are summed across loader lanes and may exceed the wall
+ # on their own; the compute phases must not
+ accounted = prof["expert_matmul_s"] + prof["attention_s"] + prof["lm_head_s"]
+ if accounted > prof["wall_s"] * 1.05 + 0.05:
+ raise AssertionError(f"serve round {ordinal}: phases exceed wall: {prof}")
finally:
engine.close()
- print("PASS target serve: persistent SUBMIT/DATA/DONE protocol is token-exact")
+ print("PASS target serve: persistent SUBMIT/DATA/DONE protocol is token-exact, PROF phases filled")
def check_cli_uses_engine_context(binary: Path, model: Path, temporary: Path) -> None: