Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
deepseek_v4: the numeric channel (SUBMIT logprobs=k, pin=1), so brio works on the ninth engine (#1648)
JustVugg6f152cde8efe
diff --git a/c/Makefile b/c/Makefileindex edc703f6..d0ea5caf 100644--- a/c/Makefile+++ b/c/Makefile@@ -709,6 +709,9 @@ deepseek-v4-tiny-check: deepseek-v4-tiny-generate$(PYTHON) tests/test_deepseek_v4_prefix.py \--binary $(CURDIR)/$(if $(IS_WIN),deepseek_v4.exe,deepseek_v4) \--fixture $(CURDIR)/deepseek_v4_tiny+ $(PYTHON) tests/test_deepseek_v4_brio.py \+ --binary $(CURDIR)/$(if $(IS_WIN),deepseek_v4.exe,deepseek_v4) \+ --fixture $(CURDIR)/deepseek_v4_tinydeepseek-v4-oracle: deepseek-v4@test -n "$(MODEL)" || { echo "usage: make deepseek-v4-oracle MODEL=/path/to/checkpoint" >&2; exit 2; }diff --git a/c/deepseek_v4.c b/c/deepseek_v4.cindex 66a83141..87fb870b 100644--- a/c/deepseek_v4.c+++ b/c/deepseek_v4.c@@ -11635,6 +11635,7 @@ int coli_v4_prompt_build(char **output, size_t *output_length,#include "json.h"#include "native_quant.h"#include "serve_codec.h"+#include "decode_batch.h" /* coli_logprob_tail: the numeric channel's tail, same bytes as the other engines */#include "tok.h"static int load_embedding(float *state, const ColiSafetensorsIndex *index,@@ -11772,55 +11773,39 @@ static int head_argmax(ColiV4Engine *engine, const float *hidden,g_v4_prof_head_s += spec_now() - t0;return result;}-static int head_argmax_impl(ColiV4Engine *engine, const float *hidden,+/* Every head score of one hidden row, in vocabulary order. head_argmax used+ * to run this matmul and keep only the maximum; the numeric channel (SUBMIT+ * logprobs=k, docs/brio.md) needs the whole row, so the row is computed here+ * once and the argmax is a scan over it. Same head_bf16_dot per row, same scan+ * order: the token picked and its logit do not change. */+static int head_scores_impl(ColiV4Engine *engine, const float *hidden,const ColiSafetensorsIndex *index,- const ColiDeepSeekV4Config *config,- int *best_token, float *best_logit) {+ const ColiDeepSeekV4Config *config, float *scores) {const ColiSafetensorsTensor *head = coli_st_find(index, "head.weight");int d = config->hidden_size, vocab = config->vocab_size;- if (!head || head->dtype != COLI_ST_BF16 || d < 1 || vocab < 1)+ if (!head || head->dtype != COLI_ST_BF16 || d < 1 || vocab < 1 || !scores)return -1;int shard = coli_st_tensor_shard(index, head);size_t resident_bytes = (size_t)vocab * (size_t)d * sizeof(uint16_t);const uint16_t *resident = coli_v4_head_cache_data(engine, shard, (uint64_t)head->off, resident_bytes);-/* The normal V4 memory plan keeps the BF16 head resident. Compute all* rows in one OpenMP team directly from that allocation: the old tiled* path copied the complete ~1 GiB head and created ~2,000 teams per token.* Each row retains the same scalar accumulation order and the final scan* retains vocabulary order, so logits/tie-breaking do not change. */if (resident) {- float *scores = malloc((size_t)vocab * sizeof(*scores));- if (!scores) return -1;#pragma omp parallel for schedule(static)for (int row = 0; row < vocab; row++) {const uint16_t *weight = resident + (size_t)row * d;scores[row] = head_bf16_dot(weight, hidden, d);}- int winner = -1;- float maximum = -FLT_MAX;- for (int row = 0; row < vocab; row++)- if (scores[row] > maximum) {- maximum = scores[row];- winner = row;- }- free(scores);- *best_token = winner;- *best_logit = maximum;- return winner < 0 ? -1 : 0;+ return 0;}-/* Low-memory fallback: stream small row tiles exactly as before. */enum { ROWS = 64 };uint16_t *raw = malloc((size_t)ROWS * d * sizeof(*raw));- float *scores = malloc((size_t)ROWS * sizeof(*scores));- if (!raw || !scores) {- free(scores); free(raw);- return -1;- }- int winner = -1;- float maximum = -FLT_MAX;+ if (!raw) return -1;for (int start = 0; start < vocab; start += ROWS) {int rows = vocab - start < ROWS ? vocab - start : ROWS;size_t bytes = (size_t)rows * d * sizeof(*raw);@@ -11828,26 +11813,54 @@ static int head_argmax_impl(ColiV4Engine *engine, const float *hidden,engine, index, shard,(uint64_t)head->off + (uint64_t)start * d * sizeof(*raw),bytes, raw)) {- free(scores); free(raw);+ free(raw);return -1;}#pragma omp parallel forfor (int row = 0; row < rows; row++) {const uint16_t *weight = raw + (size_t)row * d;- scores[row] = head_bf16_dot(weight, hidden, d);+ scores[start + row] = head_bf16_dot(weight, hidden, d);}- for (int row = 0; row < rows; row++)- if (scores[row] > maximum) {- maximum = scores[row];- winner = start + row;- }}- free(scores); free(raw);+ free(raw);+ return 0;+}+/* First maximum in vocabulary order: the tie-break head_argmax always had. */+static int head_scores_argmax(const float *scores, int vocab,+ int *best_token, float *best_logit) {+ int winner = -1;+ float maximum = -FLT_MAX;+ for (int row = 0; row < vocab; row++)+ if (scores[row] > maximum) {+ maximum = scores[row];+ winner = row;+ }*best_token = winner;*best_logit = maximum;return winner < 0 ? -1 : 0;}-+static int head_argmax_impl(ColiV4Engine *engine, const float *hidden,+ const ColiSafetensorsIndex *index,+ const ColiDeepSeekV4Config *config,+ int *best_token, float *best_logit) {+ int vocab = config->vocab_size;+ if (vocab < 1) return -1;+ float *scores = malloc((size_t)vocab * sizeof(*scores));+ if (!scores) return -1;+ int result = head_scores_impl(engine, hidden, index, config, scores);+ if (!result) result = head_scores_argmax(scores, vocab, best_token, best_logit);+ free(scores);+ return result;+}+/* The whole row, under the same head-time meter as head_argmax. */+static int head_scores(ColiV4Engine *engine, const float *hidden,+ const ColiSafetensorsIndex *index,+ const ColiDeepSeekV4Config *config, float *scores) {+ double t0 = spec_now();+ int result = head_scores_impl(engine, hidden, index, config, scores);+ g_v4_prof_head_s += spec_now() - t0;+ return result;+}static int head_argmax_batch(ColiV4Engine *engine, const float *hidden,const ColiSafetensorsIndex *index,const ColiDeepSeekV4Config *config, int batch,@@ -12651,6 +12664,8 @@ static void session_free_attention(ColiV4Session *session) {void coli_v4_session_destroy(ColiV4Session *session) {if (!session) return;kv_prefix_free(&session->fed);+ free(session->pin_ids); free(session->pin_scores);+ free(session->echo_hidden); free(session->echo_scores);session_free_attention(session);session_free_buffers(session);if (session->tokenizer_ready) {@@ -13135,7 +13150,8 @@ int coli_v4_session_generate(ColiV4Session *session,ColiV4SessionGenerateStats *stats_out,char *error, size_t error_size) {if (!session || !session->engine || !prompt || !options ||- options->max_new_tokens < 1) {+ (options->max_new_tokens < 1 &&+ !(options->max_new_tokens == 0 && options->logprobs > 0))) {if (error && error_size)snprintf(error, error_size, "invalid V4 session generate arguments");return -1;@@ -13243,6 +13259,34 @@ int coli_v4_session_generate(ColiV4Session *session,fprintf(stderr, "[PREFIX] hint boundary at %d tokens\n", ckpt_at);}session->prefix_reused = reuse;+ /* The numeric channel (docs/brio.md). Scratch sized to the head, kept on+ * the session so every early return below leaves nothing behind. */+ const int vocab = config->vocab_size;+ const int echo = options->logprobs > 0 && options->on_echo != NULL;+ const int want_scores = echo || options->pin || options->on_scores != NULL;+ if (want_scores && (!session->echo_hidden || !session->echo_scores)) {+ free(session->echo_hidden);+ free(session->echo_scores);+ session->echo_hidden = malloc((size_t)config->hidden_size * sizeof(float));+ session->echo_scores = malloc((size_t)vocab * sizeof(float));+ if (!session->echo_hidden || !session->echo_scores) {+ if (error && error_size)+ snprintf(error, error_size, "out of memory for the logprob channel");+ return -1;+ }+ }+ /* Position `reuse` is the first fresh token, and its predictor lives in+ * the state we continue from, which nothing below recomputes. When that+ * state is the pinned prompt end, its scores were kept for exactly this: a+ * closed-set caller pins the prompt, then asks about each option, and the+ * option's first token is usually its only one. Any other reuse has no+ * predictor to report; the caller sees the position missing, as with the+ * other engines. */+ if (echo && reuse > 0 && reuse < prompt_count && session->pin_scores &&+ session->pin_len == reuse &&+ !memcmp(session->pin_ids, session->prompt_ids, (size_t)reuse * sizeof(int)))+ options->on_echo(options->scores_user_data, reuse,+ session->prompt_ids[reuse], session->pin_scores, vocab);if (reuse && getenv("V4_PREFIX_LOG"))fprintf(stderr, "[PREFIX] reusing %d of %d prompt tokens\n",reuse, prompt_count);@@ -13313,6 +13357,25 @@ int coli_v4_session_generate(ColiV4Session *session,}kv_prefix_record(&session->fed, session->prompt_ids + done_upto,done_upto, seg);+ /* Read-out of the prefill: row `item` of this segment is position+ * done_upto+item and predicts the token at the next one. One head pass+ * per row, paid only by the requests that opened the channel. */+ if (echo) {+ for (int item = 0; item < seg; item++) {+ int at = done_upto + item + 1;+ if (at >= prompt_count) break;+ if (final_hidden(session->echo_hidden, state + (size_t)item * hd,+ index, config, error, error_size) ||+ head_scores(engine, session->echo_hidden, index, config,+ session->echo_scores)) {+ kv_prefix_taint(&session->fed);+ return -1;+ }+ options->on_echo(options->scores_user_data, at,+ session->prompt_ids[at], session->echo_scores,+ vocab);+ }+ }done_upto += seg;session->fed.len = done_upto;tail_rows = seg;@@ -13346,10 +13409,36 @@ int coli_v4_session_generate(ColiV4Session *session,int current = 0;float current_logit = 0.0f;if (final_hidden(hidden, last, index, config, error, error_size) ||- head_argmax(engine, hidden, index, config, ¤t, ¤t_logit)) {+ (want_scores+ ? (head_scores(engine, hidden, index, config, session->echo_scores) ||+ head_scores_argmax(session->echo_scores, vocab, ¤t,+ ¤t_logit))+ : head_argmax(engine, hidden, index, config, ¤t,+ ¤t_logit))) {kv_prefix_taint(&session->fed);return -1;}+ if (options->pin) {+ /* Keep what the snapshot cannot: the scores at the prompt end. The+ * attention state goes to a v4_ckpt slot regardless of the size gate+ * above: a pinned prompt is short by nature (a document and a+ * question) and is about to be extended by every option. */+ int *ids = realloc(session->pin_ids, (size_t)prompt_count * sizeof(int));+ float *keep = realloc(session->pin_scores, (size_t)vocab * sizeof(float));+ if (ids) session->pin_ids = ids;+ if (keep) session->pin_scores = keep;+ if (ids && keep) {+ memcpy(session->pin_ids, session->prompt_ids,+ (size_t)prompt_count * sizeof(int));+ memcpy(session->pin_scores, session->echo_scores,+ (size_t)vocab * sizeof(float));+ session->pin_len = prompt_count;+ } else {+ session->pin_len = 0; /* an optimisation, never an error */+ }+ if (v4_ckpt_min_tokens() && !v4_ckpt_have(session->prompt_ids, prompt_count))+ v4_ckpt_store(session, prompt_count, 1);+ }/* The prompt is in the attention state from here on; record it before the* decode loop so a failure mid-generation still leaves fed describing what* was actually fed. */@@ -13357,10 +13446,19 @@ int coli_v4_session_generate(ColiV4Session *session,session->fed.len = prompt_count;int generated_count = 0;int last_processed = prompt_count - 1;- generated[generated_count++] = current;- int done = session_emit_token(session, on_token, user_data, current,+ /* max_new == 0 is the read-only request of the numeric channel: the+ * prompt is in the state, its read-out went through on_echo, nothing is+ * generated and `done` skips the loop; the tail then reports zero. */+ int done = 1;+ if (max_new > 0) {+ generated[generated_count++] = current;+ if (options->on_scores)+ options->on_scores(options->scores_user_data, last_processed, current,+ session->echo_scores, vocab);+ done = session_emit_token(session, on_token, user_data, current,current_logit, last_processed,generated_count, options->stop_at_sentence);+ }double first_at = spec_now();int draft_limit = getenv("V4_DRAFT") ? atoi(getenv("V4_DRAFT")) : 0;@@ -13372,7 +13470,11 @@ int coli_v4_session_generate(ColiV4Session *session,while (!done && generated_count < max_new) {int remaining = max_new - generated_count;- if (!options->no_dspark && !session->spec_disabled && remaining >= 3) {+ /* A draft block accepts several tokens from one target pass and has+ * no per-token scores to report, so the numeric channel takes the+ * plain path: same greedy tokens, one head row each. */+ if (!options->no_dspark && options->logprobs <= 0 &&+ !session->spec_disabled && remaining >= 3) {int inputs[25] = {0}, drafts[24] = {0};int predictions[25] = {0};float logits[25] = {0};@@ -13590,12 +13692,20 @@ int coli_v4_session_generate(ColiV4Session *session,session->state = state;session->next = next;if (final_hidden(hidden, state, index, config, error, error_size) ||- head_argmax(engine, hidden, index, config, ¤t, ¤t_logit)) {+ (options->on_scores+ ? (head_scores(engine, hidden, index, config, session->echo_scores) ||+ head_scores_argmax(session->echo_scores, vocab, ¤t,+ ¤t_logit))+ : head_argmax(engine, hidden, index, config, ¤t,+ ¤t_logit))) {kv_prefix_taint(&session->fed);return -1;}last_processed = position;generated[generated_count++] = current;+ if (options->on_scores)+ options->on_scores(options->scores_user_data, last_processed, current,+ session->echo_scores, vocab);done = session_emit_token(session, on_token, user_data, current,current_logit, last_processed,generated_count,@@ -13810,6 +13920,8 @@ typedef struct {float top_p;int extension_bytes;int prefix_bytes;+ int logprobs; /* SUBMIT logprobs=k: 0 = channel closed (opt-in) */+ int pin; /* SUBMIT pin=1: keep the prompt end for the next prompts */} V4ServeRequest;typedef struct {@@ -13817,6 +13929,8 @@ typedef struct {const char *request_id;int cancelled;int fatal;+ int logprobs;+ char tail[1024]; /* the next DATA frame's logprob tail, from on_scores */} V4ServeStream;static const ColiServeWireProfile v4_wire = {@@ -14049,6 +14163,8 @@ static int v4_serve_read_request(FILE *input, FILE *output,request->top_p = command.top_p;request->extension_bytes = (int)command.extension_bytes;request->prefix_bytes = prefix_bytes;+ request->logprobs = command.logprobs;+ request->pin = command.pin;coli_serve_command_dispose(&command);return 2;}@@ -14092,8 +14208,15 @@ static int v4_serve_token(void *user_data, int token, float logit,char piece[1024];int bytes = tok_decode(&stream->session->tokenizer, &token, 1,piece, (int)sizeof(piece) - 1);- v4_serve_data(stdout, stream->request_id, piece, bytes);+ /* With the channel open the frame carries the tail on_scores left+ * here: "DATA <id> <n> <lp> <k> [tid tlp]*k", one frame per token. */+ if (stream->logprobs > 0 && bytes > 0)+ coli_serve_write_data_lp(stdout, stream->request_id, piece,+ (size_t)bytes, stream->tail);+ else+ v4_serve_data(stdout, stream->request_id, piece, bytes);}+ stream->tail[0] = 0;if (v4_serve_drain_commands(stream)) {stream->cancelled = 1;return 1;@@ -14129,6 +14252,34 @@ static void v4_serve_done(FILE *output, const char *id, int completion,coli_serve_write_done_i32_suffix(output, id, &done, &prefix_reused, 1);}+/* ECHO frame of the numeric channel, the same bytes the other engines'+ * serve_echo writes: "ECHO <id> <n> <pos> <lp> <k> [tid tlp]*k" and the+ * token's bytes DATA-framed after it. `scores` are raw head logits;+ * coli_logprob_tail does the normalisation and the top-k. */+static void v4_serve_echo(void *user_data, int position, int token,+ const float *scores, int vocab) {+ V4ServeStream *stream = user_data;+ char tail[1024], piece[1024];+ coli_logprob_tail(tail, sizeof tail, scores, vocab, token, stream->logprobs);+ int bytes = tok_decode(&stream->session->tokenizer, &token, 1, piece,+ (int)sizeof(piece) - 1);+ if (bytes < 0) bytes = 0;+ printf("ECHO %s %d %d%s\n", stream->request_id, bytes, position, tail);+ if (bytes > 0) fwrite(piece, 1, (size_t)bytes, stdout);+ fputc('\n', stdout);+ fflush(stdout);+}++/* The tail of the next DATA frame, computed while the scores exist and+ * written by v4_serve_token right after. */+static void v4_serve_scores(void *user_data, int position, int token,+ const float *scores, int vocab) {+ (void)position;+ V4ServeStream *stream = user_data;+ coli_logprob_tail(stream->tail, sizeof stream->tail, scores, vocab, token,+ stream->logprobs);+}+static int v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,V4ServeRequest *request) {if (request->extension_bytes) {@@ -14190,7 +14341,7 @@ static int v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,engine->experts ? coli_v4_expert_store_matmul_sec(engine->experts) : 0.0;double block_before = g_v4_prof_block_s, head_before = g_v4_prof_head_s;long long forwards_before = g_v4_prof_forwards;- V4ServeStream stream = {session, request->id, 0, 0};+ V4ServeStream stream = {session, request->id, 0, 0, request->logprobs, {0}};ColiV4SessionGenerateStats stats = {0};char error[512] = {0};double started = spec_now();@@ -14203,6 +14354,11 @@ static int v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,.should_abort = v4_serve_abort,.abort_user_data = &stream,.prefix_bytes = (size_t)request->prefix_bytes,+ .logprobs = request->logprobs,+ .pin = request->pin,+ .on_echo = request->logprobs > 0 ? v4_serve_echo : NULL,+ .on_scores = request->logprobs > 0 ? v4_serve_scores : NULL,+ .scores_user_data = &stream,},v4_serve_token, &stream, &stats, error, sizeof(error));double elapsed = spec_now() - started;@@ -14219,6 +14375,7 @@ static int v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,int completion = stats.generated_tokens - (stats.eos_stopped ? 1 : 0);if (completion < 0) completion = 0;int length_limited = !stream.cancelled && !stats.eos_stopped &&+ request->max_tokens > 0 && /* a read-only request is not cut short */stats.generated_tokens >= request->max_tokens;double decode = stats.decode_sec > 0.0 ? stats.decode_sec : elapsed;/* Trailing field: prompt tokens served from the previous turn's attentiondiff --git a/c/deepseek_v4.h b/c/deepseek_v4.hindex d57d1ab6..5b1ad375 100644--- a/c/deepseek_v4.h+++ b/c/deepseek_v4.h@@ -124,12 +124,31 @@ typedef struct {* uninterruptible as before. */typedef int (*ColiV4SessionAbortFn)(void *user_data);+/* The numeric channel (SUBMIT logprobs=k, docs/brio.md): raw head scores,+ * vocab_size floats, valid only for the duration of the callback. on_echo+ * fires once per prompt position whose predictor this call computed, with the+ * token that actually stands there; on_scores fires right before the on_token+ * of every generated token, with the scores it was picked from. */+typedef void (*ColiV4SessionScoresFn)(void *user_data, int position, int token,+ const float *scores, int vocab);+typedef struct {- int max_new_tokens; /* required; clamped by session cap */+ int max_new_tokens; /* required; clamped by session cap; 0 only with logprobs > 0 */int stop_at_sentence;int no_dspark; /* disable speculative draft/verification */ColiV4SessionAbortFn should_abort; /* optional prefill abort poll */void *abort_user_data;+ /* SUBMIT logprobs=k / pin=1. logprobs > 0 opens the channel above, turns+ * speculative decoding off for the request (a draft accepted in a block+ * has no scores of its own) and allows max_new_tokens == 0, "read the+ * prompt and stop". pin keeps the prompt-end scores and a snapshot of the+ * attention state, so the prompts that extend this one start from here+ * with their first fresh token's predictor intact. */+ int logprobs;+ int pin;+ ColiV4SessionScoresFn on_echo;+ ColiV4SessionScoresFn on_scores;+ void *scores_user_data;/* Optional: byte length of the prompt's stable leading prefix (the* rendered system turn). The session snapshots the attention state at* that token boundary during this prefill so later conversations thatdiff --git a/c/deepseek_v4_internal.h b/c/deepseek_v4_internal.hindex 2bcdf9cd..604e4916 100644--- a/c/deepseek_v4_internal.h+++ b/c/deepseek_v4_internal.h@@ -966,6 +966,20 @@ struct ColiV4Session {uint64_t spec_drafted;uint64_t spec_accepted;int spec_disabled;+ /* Prompt-end capture for SUBMIT pin=1: the ids fed and the head scores+ * that predict the token after them. A later prompt that starts with+ * exactly these ids gets its first fresh token's predictor from here; that+ * token is the one a closed-set caller asks about (docs/brio.md). The+ * attention state itself goes to a v4_ckpt slot; this is the part the+ * snapshot does not hold. */+ int *pin_ids;+ int pin_len;+ float *pin_scores;+ /* Scratch for the numeric channel, one hidden row and one row of head+ * scores, allocated on first use and freed with the session so the many+ * early returns of generate() leave nothing behind. */+ float *echo_hidden;+ float *echo_scores;};/* RAM-tiered expert open used by coli_v4_engine_open (replaces ld --wrap).diff --git a/c/tests/test_deepseek_v4_brio.py b/c/tests/test_deepseek_v4_brio.pynew file mode 100644index 00000000..3fb77104--- /dev/null+++ b/c/tests/test_deepseek_v4_brio.py@@ -0,0 +1,289 @@+#!/usr/bin/env python3+"""The numeric channel on the DeepSeek V4 serve path (SUBMIT logprobs=k / pin=1).++This is what `POST /v1/brio` speaks (docs/brio.md), and #1648 found that this+engine accepted the header and then refused the request. The properties under+test are the exact ones a closed-set caller relies on:++- a read-only request (max_tokens=0, logprobs>0) answers with one ECHO frame+ per prompt position after the first, and generates nothing;+- the ECHO at position p is the head over prompt[:p]: its best token is the+ token a greedy generation from that same prefix produces;+- a pinned prompt gives every prompt that extends it the predictor of its+ first fresh token, from the kept scores, and those scores are the ones a+ cold prefill computes;+- with the channel open, generation is unchanged and every DATA frame carries+ the tail of the token it holds.++Speaks SUBMIT/ECHO/DATA/DONE directly, as tests/test_deepseek_v4_prefix.py does.+"""+from __future__ import annotations++import argparse+import json+import math+import os+import re+import subprocess+import sys+from pathlib import Path+++def token_prompt(ids: list[int]) -> str:+ return "".join(f"<t{token:03d}>" for token in ids)+++def parse_tokens(text: str) -> list[int]:+ return [int(match) for match in re.findall(r"<t(\d+)>", text)]+++class Reply:+ def __init__(self) -> None:+ self.accepted = -1+ self.echoes: dict[int, dict] = {} # position -> {"token", "lp", "top": {tid: tlp}}+ self.data: list[tuple[bytes, dict | None]] = []+ self.reuse = -1+ self.completion = -1++ @property+ def text(self) -> str:+ return b"".join(piece for piece, _ in self.data).decode("utf-8", "replace")+++def parse_tail(fields: list[str]) -> dict | None:+ """`<lp> <k> [tid tlp]*k` -> {"lp": float, "top": {tid: tlp}}; None for no tail."""+ if not fields:+ return None+ lp = None if fields[0] == "nan" else float(fields[0])+ k = int(fields[1])+ pairs = fields[2:2 + 2 * k]+ top = {int(pairs[i]): float(pairs[i + 1]) for i in range(0, len(pairs), 2)}+ return {"lp": lp, "top": top}+++class Serve:+ """One persistent `SERVE=1` engine process."""++ def __init__(self, binary: Path, model: Path, ctx: str = "128") -> None:+ env = dict(os.environ, SERVE="1", SNAP=str(model), CTX=ctx)+ self.process = subprocess.Popen(+ [str(binary)], stdin=subprocess.PIPE, stdout=subprocess.PIPE,+ stderr=subprocess.PIPE, env=env,+ )+ self.counter = 0++ def submit(self, prompt: str, max_tokens: int, logprobs: int = 0,+ pin: bool = False) -> Reply:+ self.counter += 1+ request_id = f"r{self.counter}"+ payload = prompt.encode("utf-8")+ header = f"SUBMIT {request_id} 0 {len(payload)} {max_tokens} 0.0 1.0 0"+ if logprobs:+ header += f" logprobs={logprobs}"+ if pin:+ header += " pin=1"+ assert self.process.stdin is not None+ self.process.stdin.write(header.encode("ascii") + b"\n" + payload + b"\n")+ self.process.stdin.flush()++ reply = Reply()+ stream = self.process.stdout+ assert stream is not None+ while True:+ line = stream.readline()+ if not line:+ raise AssertionError(+ f"engine closed stdout during {request_id}; "+ f"stderr:\n{self._drain_stderr()}")+ fields = line.decode("utf-8", "replace").split()+ if not fields:+ continue+ kind = fields[0]+ if kind == "ERROR":+ raise AssertionError(f"engine ERROR: {line!r}")+ if kind == "ACCEPT":+ reply.accepted = int(fields[2])+ elif kind == "DATA":+ size = int(fields[2])+ piece = stream.read(size)+ stream.read(1)+ reply.data.append((piece, parse_tail(fields[3:])))+ elif kind == "ECHO":+ # ECHO <id> <n> <pos> <lp> <k> [tid tlp]*k, then n bytes + LF+ size = int(fields[2])+ piece = stream.read(size)+ stream.read(1)+ tail = parse_tail(fields[4:])+ assert tail is not None+ reply.echoes[int(fields[3])] = {"token": piece.decode("utf-8", "replace"), **tail}+ elif kind == "DONE":+ # DONE <id> STAT <completion> <tok/s> <hit%> <rss> <prompt> <cap> [<reuse>]+ reply.completion = int(fields[3])+ if len(fields) >= 10:+ reply.reuse = int(fields[9])+ break+ return reply++ def _drain_stderr(self) -> str:+ assert self.process.stderr is not None+ try:+ return self.process.stderr.read(8192).decode("utf-8", "replace")+ except Exception: # pragma: no cover+ return "<unavailable>"++ def close(self) -> None:+ try:+ if self.process.stdin:+ self.process.stdin.close()+ self.process.wait(timeout=30)+ except Exception: # pragma: no cover+ self.process.kill()+++def best_of(echo: dict) -> int:+ return max(echo["top"], key=lambda tid: echo["top"][tid])+++def check_read_only(binary: Path, model: Path, ids: list[int]) -> None:+ """max_tokens=0 with logprobs: one ECHO per position after the first, no DATA."""+ serve = Serve(binary, model)+ try:+ reply = serve.submit(token_prompt(ids), 0, logprobs=3)+ finally:+ serve.close()+ n = len(ids)+ if reply.accepted != n:+ raise AssertionError(f"ACCEPT said {reply.accepted} tokens for a {n}-token prompt")+ if reply.data:+ raise AssertionError(f"a read-only request generated {len(reply.data)} DATA frames")+ if reply.completion != 0:+ raise AssertionError(f"DONE reported {reply.completion} completion tokens")+ if sorted(reply.echoes) != list(range(1, n)):+ raise AssertionError(f"ECHO positions {sorted(reply.echoes)}, expected 1..{n - 1}")+ for position, echo in reply.echoes.items():+ if parse_tokens(echo["token"]) != [ids[position]]:+ raise AssertionError(f"ECHO at {position} names {echo['token']!r}, prompt has {ids[position]}")+ if echo["lp"] is None or echo["lp"] > 1e-6:+ raise AssertionError(f"ECHO at {position}: log-probability {echo['lp']} is not <= 0")+ if len(echo["top"]) != 3:+ raise AssertionError(f"ECHO at {position}: asked top-3, got {len(echo['top'])}")+ mass = sum(math.exp(v) for v in echo["top"].values())+ if mass > 1.0 + 1e-4:+ raise AssertionError(f"ECHO at {position}: top-3 mass {mass} exceeds 1")+ if ids[position] in echo["top"] and abs(echo["top"][ids[position]] - echo["lp"]) > 1e-5:+ raise AssertionError(f"ECHO at {position}: the token's own lp differs from its top-k entry")+ print(f"PASS read-only: {n - 1} ECHO frames, zero generated tokens")+++def check_echo_is_the_head(binary: Path, model: Path, ids: list[int]) -> None:+ """The best token of the ECHO at p is what greedy decoding emits after prompt[:p]."""+ serve = Serve(binary, model)+ try:+ full = serve.submit(token_prompt(ids), 0, logprobs=4)+ for position in sorted({1, len(ids) // 2, len(ids) - 1}):+ cold = Serve(binary, model)+ try:+ gen = cold.submit(token_prompt(ids[:position]), 1)+ finally:+ cold.close()+ produced = parse_tokens(gen.text)+ expected = best_of(full.echoes[position])+ if produced != [expected]:+ raise AssertionError(+ f"position {position}: ECHO's best token is {expected}, "+ f"greedy generation from the same prefix gave {produced}")+ finally:+ serve.close()+ print("PASS echo is the head: best ECHO token == greedy token from the same prefix")+++def check_pin(binary: Path, model: Path, ids: list[int], a: int, b: int) -> None:+ """Pin the prompt, ask two options: both start from the pin with the first token's predictor."""+ n = len(ids)+ serve = Serve(binary, model)+ try:+ pinned = serve.submit(token_prompt(ids), 0, logprobs=2, pin=True)+ if pinned.completion != 0 or pinned.data:+ raise AssertionError("the pinned read generated tokens")+ first = serve.submit(token_prompt(ids + [a]), 0, logprobs=2)+ second = serve.submit(token_prompt(ids + [b]), 0, logprobs=2)+ finally:+ serve.close()+ for name, reply in (("first option", first), ("second option", second)):+ if reply.reuse != n:+ raise AssertionError(f"{name}: reused {reply.reuse} tokens, the pin holds {n}")+ if sorted(reply.echoes) != [n]:+ raise AssertionError(f"{name}: ECHO positions {sorted(reply.echoes)}, expected [{n}]")+ if first.echoes[n]["top"] != second.echoes[n]["top"]:+ raise AssertionError("the two options saw different predictors for the same pinned prompt")+ if parse_tokens(first.echoes[n]["token"]) != [a] or parse_tokens(second.echoes[n]["token"]) != [b]:+ raise AssertionError("the ECHO at the option position does not name the option token")+ # the kept scores are what a cold prefill computes at that position+ cold = Serve(binary, model)+ try:+ fresh = cold.submit(token_prompt(ids + [a]), 0, logprobs=2)+ finally:+ cold.close()+ if fresh.reuse != 0:+ raise AssertionError(f"cold engine reported reuse={fresh.reuse}")+ warm_top, cold_top = first.echoes[n]["top"], fresh.echoes[n]["top"]+ if warm_top.keys() != cold_top.keys() or any(abs(warm_top[t] - cold_top[t]) > 1e-4 for t in warm_top):+ raise AssertionError(f"pinned predictor {warm_top} differs from a cold prefill's {cold_top}")+ if abs(first.echoes[n]["lp"] - fresh.echoes[n]["lp"]) > 1e-4:+ raise AssertionError("the option's log-probability differs between the pin and a cold prefill")+ print(f"PASS pin: both options reused {n} tokens and read the pinned predictor, equal to a cold prefill")+++def check_generation_with_tail(binary: Path, model: Path, ids: list[int], max_new: int) -> None:+ """The channel does not change greedy output, and every DATA carries its token's tail."""+ plain = Serve(binary, model)+ try:+ without = plain.submit(token_prompt(ids), max_new)+ finally:+ plain.close()+ channel = Serve(binary, model)+ try:+ with_lp = channel.submit(token_prompt(ids), max_new, logprobs=2)+ finally:+ channel.close()+ if without.text != with_lp.text:+ raise AssertionError(f"logprobs changed the output:\n plain: {without.text!r}\n channel: {with_lp.text!r}")+ if not with_lp.data:+ raise AssertionError("no DATA frames with the channel open")+ for piece, tail in with_lp.data:+ if tail is None:+ raise AssertionError(f"DATA {piece!r} carries no logprob tail")+ token = parse_tokens(piece.decode("utf-8", "replace"))+ if len(token) != 1:+ raise AssertionError(f"DATA frame is not one token: {piece!r}")+ if len(tail["top"]) != 2:+ raise AssertionError(f"asked top-2, got {len(tail['top'])}")+ # greedy: the emitted token IS the best one, so its lp is the top entry+ if best_of(tail) != token[0] or abs(tail["top"][token[0]] - tail["lp"]) > 1e-5:+ raise AssertionError(f"DATA {token}: tail {tail} does not name it as the best token")+ print(f"PASS generation: {len(with_lp.data)} tokens unchanged, each DATA frame with its tail")+++def main() -> int:+ parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])+ parser.add_argument("--binary", type=Path, required=True)+ parser.add_argument("--fixture", type=Path, required=True)+ arguments = parser.parse_args()+ binary = arguments.binary.resolve()+ reference = json.loads((arguments.fixture / "ref.json").read_text("utf-8"))+ case = reference["cases"]["short"]+ ids = list(case["prompt_ids"])+ greedy = list(case["greedy_new_ids"])+ # two options: the token greedy decoding would pick, and one it would not+ a = greedy[0]+ b = next(t for t in ids if t != a)+ check_read_only(binary, arguments.fixture, ids)+ check_echo_is_the_head(binary, arguments.fixture, ids)+ check_pin(binary, arguments.fixture, ids, a, b)+ check_generation_with_tail(binary, arguments.fixture, ids, 4)+ print("PASS DeepSeek V4 numeric channel: all checks completed")+ return 0+++if __name__ == "__main__":+ sys.exit(main())