Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

deepseek_v4: the numeric channel (SUBMIT logprobs=k, pin=1), so brio works on the ninth engine (#1648)

JustVugg6f152cde8efe
diff --git a/c/Makefile b/c/Makefile
index edc703f6..d0ea5caf 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -709,6 +709,9 @@ deepseek-v4-tiny-check: deepseek-v4-tiny-generate
$(PYTHON) tests/test_deepseek_v4_prefix.py \
--binary $(CURDIR)/$(if $(IS_WIN),deepseek_v4.exe,deepseek_v4) \
--fixture $(CURDIR)/deepseek_v4_tiny
+ $(PYTHON) tests/test_deepseek_v4_brio.py \
+ --binary $(CURDIR)/$(if $(IS_WIN),deepseek_v4.exe,deepseek_v4) \
+ --fixture $(CURDIR)/deepseek_v4_tiny
deepseek-v4-oracle: deepseek-v4
@test -n "$(MODEL)" || { echo "usage: make deepseek-v4-oracle MODEL=/path/to/checkpoint" >&2; exit 2; }
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.c
index 66a83141..87fb870b 100644
--- a/c/deepseek_v4.c
+++ b/c/deepseek_v4.c
@@ -11635,6 +11635,7 @@ int coli_v4_prompt_build(char **output, size_t *output_length,
#include "json.h"
#include "native_quant.h"
#include "serve_codec.h"
+#include "decode_batch.h" /* coli_logprob_tail: the numeric channel's tail, same bytes as the other engines */
#include "tok.h"
static int load_embedding(float *state, const ColiSafetensorsIndex *index,
@@ -11772,55 +11773,39 @@ static int head_argmax(ColiV4Engine *engine, const float *hidden,
g_v4_prof_head_s += spec_now() - t0;
return result;
}
-static int head_argmax_impl(ColiV4Engine *engine, const float *hidden,
+/* Every head score of one hidden row, in vocabulary order. head_argmax used
+ * to run this matmul and keep only the maximum; the numeric channel (SUBMIT
+ * logprobs=k, docs/brio.md) needs the whole row, so the row is computed here
+ * once and the argmax is a scan over it. Same head_bf16_dot per row, same scan
+ * order: the token picked and its logit do not change. */
+static int head_scores_impl(ColiV4Engine *engine, const float *hidden,
const ColiSafetensorsIndex *index,
- const ColiDeepSeekV4Config *config,
- int *best_token, float *best_logit) {
+ const ColiDeepSeekV4Config *config, float *scores) {
const ColiSafetensorsTensor *head = coli_st_find(index, "head.weight");
int d = config->hidden_size, vocab = config->vocab_size;
- if (!head || head->dtype != COLI_ST_BF16 || d < 1 || vocab < 1)
+ if (!head || head->dtype != COLI_ST_BF16 || d < 1 || vocab < 1 || !scores)
return -1;
int shard = coli_st_tensor_shard(index, head);
size_t resident_bytes = (size_t)vocab * (size_t)d * sizeof(uint16_t);
const uint16_t *resident = coli_v4_head_cache_data(
engine, shard, (uint64_t)head->off, resident_bytes);
-
/* The normal V4 memory plan keeps the BF16 head resident. Compute all
* rows in one OpenMP team directly from that allocation: the old tiled
* path copied the complete ~1 GiB head and created ~2,000 teams per token.
* Each row retains the same scalar accumulation order and the final scan
* retains vocabulary order, so logits/tie-breaking do not change. */
if (resident) {
- float *scores = malloc((size_t)vocab * sizeof(*scores));
- if (!scores) return -1;
#pragma omp parallel for schedule(static)
for (int row = 0; row < vocab; row++) {
const uint16_t *weight = resident + (size_t)row * d;
scores[row] = head_bf16_dot(weight, hidden, d);
}
- int winner = -1;
- float maximum = -FLT_MAX;
- for (int row = 0; row < vocab; row++)
- if (scores[row] > maximum) {
- maximum = scores[row];
- winner = row;
- }
- free(scores);
- *best_token = winner;
- *best_logit = maximum;
- return winner < 0 ? -1 : 0;
+ return 0;
}
-
/* Low-memory fallback: stream small row tiles exactly as before. */
enum { ROWS = 64 };
uint16_t *raw = malloc((size_t)ROWS * d * sizeof(*raw));
- float *scores = malloc((size_t)ROWS * sizeof(*scores));
- if (!raw || !scores) {
- free(scores); free(raw);
- return -1;
- }
- int winner = -1;
- float maximum = -FLT_MAX;
+ if (!raw) return -1;
for (int start = 0; start < vocab; start += ROWS) {
int rows = vocab - start < ROWS ? vocab - start : ROWS;
size_t bytes = (size_t)rows * d * sizeof(*raw);
@@ -11828,26 +11813,54 @@ static int head_argmax_impl(ColiV4Engine *engine, const float *hidden,
engine, index, shard,
(uint64_t)head->off + (uint64_t)start * d * sizeof(*raw),
bytes, raw)) {
- free(scores); free(raw);
+ free(raw);
return -1;
}
#pragma omp parallel for
for (int row = 0; row < rows; row++) {
const uint16_t *weight = raw + (size_t)row * d;
- scores[row] = head_bf16_dot(weight, hidden, d);
+ scores[start + row] = head_bf16_dot(weight, hidden, d);
}
- for (int row = 0; row < rows; row++)
- if (scores[row] > maximum) {
- maximum = scores[row];
- winner = start + row;
- }
}
- free(scores); free(raw);
+ free(raw);
+ return 0;
+}
+/* First maximum in vocabulary order: the tie-break head_argmax always had. */
+static int head_scores_argmax(const float *scores, int vocab,
+ int *best_token, float *best_logit) {
+ int winner = -1;
+ float maximum = -FLT_MAX;
+ for (int row = 0; row < vocab; row++)
+ if (scores[row] > maximum) {
+ maximum = scores[row];
+ winner = row;
+ }
*best_token = winner;
*best_logit = maximum;
return winner < 0 ? -1 : 0;
}
-
+static int head_argmax_impl(ColiV4Engine *engine, const float *hidden,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config,
+ int *best_token, float *best_logit) {
+ int vocab = config->vocab_size;
+ if (vocab < 1) return -1;
+ float *scores = malloc((size_t)vocab * sizeof(*scores));
+ if (!scores) return -1;
+ int result = head_scores_impl(engine, hidden, index, config, scores);
+ if (!result) result = head_scores_argmax(scores, vocab, best_token, best_logit);
+ free(scores);
+ return result;
+}
+/* The whole row, under the same head-time meter as head_argmax. */
+static int head_scores(ColiV4Engine *engine, const float *hidden,
+ const ColiSafetensorsIndex *index,
+ const ColiDeepSeekV4Config *config, float *scores) {
+ double t0 = spec_now();
+ int result = head_scores_impl(engine, hidden, index, config, scores);
+ g_v4_prof_head_s += spec_now() - t0;
+ return result;
+}
static int head_argmax_batch(ColiV4Engine *engine, const float *hidden,
const ColiSafetensorsIndex *index,
const ColiDeepSeekV4Config *config, int batch,
@@ -12651,6 +12664,8 @@ static void session_free_attention(ColiV4Session *session) {
void coli_v4_session_destroy(ColiV4Session *session) {
if (!session) return;
kv_prefix_free(&session->fed);
+ free(session->pin_ids); free(session->pin_scores);
+ free(session->echo_hidden); free(session->echo_scores);
session_free_attention(session);
session_free_buffers(session);
if (session->tokenizer_ready) {
@@ -13135,7 +13150,8 @@ int coli_v4_session_generate(ColiV4Session *session,
ColiV4SessionGenerateStats *stats_out,
char *error, size_t error_size) {
if (!session || !session->engine || !prompt || !options ||
- options->max_new_tokens < 1) {
+ (options->max_new_tokens < 1 &&
+ !(options->max_new_tokens == 0 && options->logprobs > 0))) {
if (error && error_size)
snprintf(error, error_size, "invalid V4 session generate arguments");
return -1;
@@ -13243,6 +13259,34 @@ int coli_v4_session_generate(ColiV4Session *session,
fprintf(stderr, "[PREFIX] hint boundary at %d tokens\n", ckpt_at);
}
session->prefix_reused = reuse;
+ /* The numeric channel (docs/brio.md). Scratch sized to the head, kept on
+ * the session so every early return below leaves nothing behind. */
+ const int vocab = config->vocab_size;
+ const int echo = options->logprobs > 0 && options->on_echo != NULL;
+ const int want_scores = echo || options->pin || options->on_scores != NULL;
+ if (want_scores && (!session->echo_hidden || !session->echo_scores)) {
+ free(session->echo_hidden);
+ free(session->echo_scores);
+ session->echo_hidden = malloc((size_t)config->hidden_size * sizeof(float));
+ session->echo_scores = malloc((size_t)vocab * sizeof(float));
+ if (!session->echo_hidden || !session->echo_scores) {
+ if (error && error_size)
+ snprintf(error, error_size, "out of memory for the logprob channel");
+ return -1;
+ }
+ }
+ /* Position `reuse` is the first fresh token, and its predictor lives in
+ * the state we continue from, which nothing below recomputes. When that
+ * state is the pinned prompt end, its scores were kept for exactly this: a
+ * closed-set caller pins the prompt, then asks about each option, and the
+ * option's first token is usually its only one. Any other reuse has no
+ * predictor to report; the caller sees the position missing, as with the
+ * other engines. */
+ if (echo && reuse > 0 && reuse < prompt_count && session->pin_scores &&
+ session->pin_len == reuse &&
+ !memcmp(session->pin_ids, session->prompt_ids, (size_t)reuse * sizeof(int)))
+ options->on_echo(options->scores_user_data, reuse,
+ session->prompt_ids[reuse], session->pin_scores, vocab);
if (reuse && getenv("V4_PREFIX_LOG"))
fprintf(stderr, "[PREFIX] reusing %d of %d prompt tokens\n",
reuse, prompt_count);
@@ -13313,6 +13357,25 @@ int coli_v4_session_generate(ColiV4Session *session,
}
kv_prefix_record(&session->fed, session->prompt_ids + done_upto,
done_upto, seg);
+ /* Read-out of the prefill: row `item` of this segment is position
+ * done_upto+item and predicts the token at the next one. One head pass
+ * per row, paid only by the requests that opened the channel. */
+ if (echo) {
+ for (int item = 0; item < seg; item++) {
+ int at = done_upto + item + 1;
+ if (at >= prompt_count) break;
+ if (final_hidden(session->echo_hidden, state + (size_t)item * hd,
+ index, config, error, error_size) ||
+ head_scores(engine, session->echo_hidden, index, config,
+ session->echo_scores)) {
+ kv_prefix_taint(&session->fed);
+ return -1;
+ }
+ options->on_echo(options->scores_user_data, at,
+ session->prompt_ids[at], session->echo_scores,
+ vocab);
+ }
+ }
done_upto += seg;
session->fed.len = done_upto;
tail_rows = seg;
@@ -13346,10 +13409,36 @@ int coli_v4_session_generate(ColiV4Session *session,
int current = 0;
float current_logit = 0.0f;
if (final_hidden(hidden, last, index, config, error, error_size) ||
- head_argmax(engine, hidden, index, config, &current, &current_logit)) {
+ (want_scores
+ ? (head_scores(engine, hidden, index, config, session->echo_scores) ||
+ head_scores_argmax(session->echo_scores, vocab, &current,
+ &current_logit))
+ : head_argmax(engine, hidden, index, config, &current,
+ &current_logit))) {
kv_prefix_taint(&session->fed);
return -1;
}
+ if (options->pin) {
+ /* Keep what the snapshot cannot: the scores at the prompt end. The
+ * attention state goes to a v4_ckpt slot regardless of the size gate
+ * above: a pinned prompt is short by nature (a document and a
+ * question) and is about to be extended by every option. */
+ int *ids = realloc(session->pin_ids, (size_t)prompt_count * sizeof(int));
+ float *keep = realloc(session->pin_scores, (size_t)vocab * sizeof(float));
+ if (ids) session->pin_ids = ids;
+ if (keep) session->pin_scores = keep;
+ if (ids && keep) {
+ memcpy(session->pin_ids, session->prompt_ids,
+ (size_t)prompt_count * sizeof(int));
+ memcpy(session->pin_scores, session->echo_scores,
+ (size_t)vocab * sizeof(float));
+ session->pin_len = prompt_count;
+ } else {
+ session->pin_len = 0; /* an optimisation, never an error */
+ }
+ if (v4_ckpt_min_tokens() && !v4_ckpt_have(session->prompt_ids, prompt_count))
+ v4_ckpt_store(session, prompt_count, 1);
+ }
/* The prompt is in the attention state from here on; record it before the
* decode loop so a failure mid-generation still leaves fed describing what
* was actually fed. */
@@ -13357,10 +13446,19 @@ int coli_v4_session_generate(ColiV4Session *session,
session->fed.len = prompt_count;
int generated_count = 0;
int last_processed = prompt_count - 1;
- generated[generated_count++] = current;
- int done = session_emit_token(session, on_token, user_data, current,
+ /* max_new == 0 is the read-only request of the numeric channel: the
+ * prompt is in the state, its read-out went through on_echo, nothing is
+ * generated and `done` skips the loop; the tail then reports zero. */
+ int done = 1;
+ if (max_new > 0) {
+ generated[generated_count++] = current;
+ if (options->on_scores)
+ options->on_scores(options->scores_user_data, last_processed, current,
+ session->echo_scores, vocab);
+ done = session_emit_token(session, on_token, user_data, current,
current_logit, last_processed,
generated_count, options->stop_at_sentence);
+ }
double first_at = spec_now();
int draft_limit = getenv("V4_DRAFT") ? atoi(getenv("V4_DRAFT")) : 0;
@@ -13372,7 +13470,11 @@ int coli_v4_session_generate(ColiV4Session *session,
while (!done && generated_count < max_new) {
int remaining = max_new - generated_count;
- if (!options->no_dspark && !session->spec_disabled && remaining >= 3) {
+ /* A draft block accepts several tokens from one target pass and has
+ * no per-token scores to report, so the numeric channel takes the
+ * plain path: same greedy tokens, one head row each. */
+ if (!options->no_dspark && options->logprobs <= 0 &&
+ !session->spec_disabled && remaining >= 3) {
int inputs[25] = {0}, drafts[24] = {0};
int predictions[25] = {0};
float logits[25] = {0};
@@ -13590,12 +13692,20 @@ int coli_v4_session_generate(ColiV4Session *session,
session->state = state;
session->next = next;
if (final_hidden(hidden, state, index, config, error, error_size) ||
- head_argmax(engine, hidden, index, config, &current, &current_logit)) {
+ (options->on_scores
+ ? (head_scores(engine, hidden, index, config, session->echo_scores) ||
+ head_scores_argmax(session->echo_scores, vocab, &current,
+ &current_logit))
+ : head_argmax(engine, hidden, index, config, &current,
+ &current_logit))) {
kv_prefix_taint(&session->fed);
return -1;
}
last_processed = position;
generated[generated_count++] = current;
+ if (options->on_scores)
+ options->on_scores(options->scores_user_data, last_processed, current,
+ session->echo_scores, vocab);
done = session_emit_token(session, on_token, user_data, current,
current_logit, last_processed,
generated_count,
@@ -13810,6 +13920,8 @@ typedef struct {
float top_p;
int extension_bytes;
int prefix_bytes;
+ int logprobs; /* SUBMIT logprobs=k: 0 = channel closed (opt-in) */
+ int pin; /* SUBMIT pin=1: keep the prompt end for the next prompts */
} V4ServeRequest;
typedef struct {
@@ -13817,6 +13929,8 @@ typedef struct {
const char *request_id;
int cancelled;
int fatal;
+ int logprobs;
+ char tail[1024]; /* the next DATA frame's logprob tail, from on_scores */
} V4ServeStream;
static const ColiServeWireProfile v4_wire = {
@@ -14049,6 +14163,8 @@ static int v4_serve_read_request(FILE *input, FILE *output,
request->top_p = command.top_p;
request->extension_bytes = (int)command.extension_bytes;
request->prefix_bytes = prefix_bytes;
+ request->logprobs = command.logprobs;
+ request->pin = command.pin;
coli_serve_command_dispose(&command);
return 2;
}
@@ -14092,8 +14208,15 @@ static int v4_serve_token(void *user_data, int token, float logit,
char piece[1024];
int bytes = tok_decode(&stream->session->tokenizer, &token, 1,
piece, (int)sizeof(piece) - 1);
- v4_serve_data(stdout, stream->request_id, piece, bytes);
+ /* With the channel open the frame carries the tail on_scores left
+ * here: "DATA <id> <n> <lp> <k> [tid tlp]*k", one frame per token. */
+ if (stream->logprobs > 0 && bytes > 0)
+ coli_serve_write_data_lp(stdout, stream->request_id, piece,
+ (size_t)bytes, stream->tail);
+ else
+ v4_serve_data(stdout, stream->request_id, piece, bytes);
}
+ stream->tail[0] = 0;
if (v4_serve_drain_commands(stream)) {
stream->cancelled = 1;
return 1;
@@ -14129,6 +14252,34 @@ static void v4_serve_done(FILE *output, const char *id, int completion,
coli_serve_write_done_i32_suffix(output, id, &done, &prefix_reused, 1);
}
+/* ECHO frame of the numeric channel, the same bytes the other engines'
+ * serve_echo writes: "ECHO <id> <n> <pos> <lp> <k> [tid tlp]*k" and the
+ * token's bytes DATA-framed after it. `scores` are raw head logits;
+ * coli_logprob_tail does the normalisation and the top-k. */
+static void v4_serve_echo(void *user_data, int position, int token,
+ const float *scores, int vocab) {
+ V4ServeStream *stream = user_data;
+ char tail[1024], piece[1024];
+ coli_logprob_tail(tail, sizeof tail, scores, vocab, token, stream->logprobs);
+ int bytes = tok_decode(&stream->session->tokenizer, &token, 1, piece,
+ (int)sizeof(piece) - 1);
+ if (bytes < 0) bytes = 0;
+ printf("ECHO %s %d %d%s\n", stream->request_id, bytes, position, tail);
+ if (bytes > 0) fwrite(piece, 1, (size_t)bytes, stdout);
+ fputc('\n', stdout);
+ fflush(stdout);
+}
+
+/* The tail of the next DATA frame, computed while the scores exist and
+ * written by v4_serve_token right after. */
+static void v4_serve_scores(void *user_data, int position, int token,
+ const float *scores, int vocab) {
+ (void)position;
+ V4ServeStream *stream = user_data;
+ coli_logprob_tail(stream->tail, sizeof stream->tail, scores, vocab, token,
+ stream->logprobs);
+}
+
static int v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,
V4ServeRequest *request) {
if (request->extension_bytes) {
@@ -14190,7 +14341,7 @@ static int v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,
engine->experts ? coli_v4_expert_store_matmul_sec(engine->experts) : 0.0;
double block_before = g_v4_prof_block_s, head_before = g_v4_prof_head_s;
long long forwards_before = g_v4_prof_forwards;
- V4ServeStream stream = {session, request->id, 0, 0};
+ V4ServeStream stream = {session, request->id, 0, 0, request->logprobs, {0}};
ColiV4SessionGenerateStats stats = {0};
char error[512] = {0};
double started = spec_now();
@@ -14203,6 +14354,11 @@ static int v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,
.should_abort = v4_serve_abort,
.abort_user_data = &stream,
.prefix_bytes = (size_t)request->prefix_bytes,
+ .logprobs = request->logprobs,
+ .pin = request->pin,
+ .on_echo = request->logprobs > 0 ? v4_serve_echo : NULL,
+ .on_scores = request->logprobs > 0 ? v4_serve_scores : NULL,
+ .scores_user_data = &stream,
},
v4_serve_token, &stream, &stats, error, sizeof(error));
double elapsed = spec_now() - started;
@@ -14219,6 +14375,7 @@ static int v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,
int completion = stats.generated_tokens - (stats.eos_stopped ? 1 : 0);
if (completion < 0) completion = 0;
int length_limited = !stream.cancelled && !stats.eos_stopped &&
+ request->max_tokens > 0 && /* a read-only request is not cut short */
stats.generated_tokens >= request->max_tokens;
double decode = stats.decode_sec > 0.0 ? stats.decode_sec : elapsed;
/* Trailing field: prompt tokens served from the previous turn's attention
diff --git a/c/deepseek_v4.h b/c/deepseek_v4.h
index d57d1ab6..5b1ad375 100644
--- a/c/deepseek_v4.h
+++ b/c/deepseek_v4.h
@@ -124,12 +124,31 @@ typedef struct {
* uninterruptible as before. */
typedef int (*ColiV4SessionAbortFn)(void *user_data);
+/* The numeric channel (SUBMIT logprobs=k, docs/brio.md): raw head scores,
+ * vocab_size floats, valid only for the duration of the callback. on_echo
+ * fires once per prompt position whose predictor this call computed, with the
+ * token that actually stands there; on_scores fires right before the on_token
+ * of every generated token, with the scores it was picked from. */
+typedef void (*ColiV4SessionScoresFn)(void *user_data, int position, int token,
+ const float *scores, int vocab);
+
typedef struct {
- int max_new_tokens; /* required; clamped by session cap */
+ int max_new_tokens; /* required; clamped by session cap; 0 only with logprobs > 0 */
int stop_at_sentence;
int no_dspark; /* disable speculative draft/verification */
ColiV4SessionAbortFn should_abort; /* optional prefill abort poll */
void *abort_user_data;
+ /* SUBMIT logprobs=k / pin=1. logprobs > 0 opens the channel above, turns
+ * speculative decoding off for the request (a draft accepted in a block
+ * has no scores of its own) and allows max_new_tokens == 0, "read the
+ * prompt and stop". pin keeps the prompt-end scores and a snapshot of the
+ * attention state, so the prompts that extend this one start from here
+ * with their first fresh token's predictor intact. */
+ int logprobs;
+ int pin;
+ ColiV4SessionScoresFn on_echo;
+ ColiV4SessionScoresFn on_scores;
+ void *scores_user_data;
/* Optional: byte length of the prompt's stable leading prefix (the
* rendered system turn). The session snapshots the attention state at
* that token boundary during this prefill so later conversations that
diff --git a/c/deepseek_v4_internal.h b/c/deepseek_v4_internal.h
index 2bcdf9cd..604e4916 100644
--- a/c/deepseek_v4_internal.h
+++ b/c/deepseek_v4_internal.h
@@ -966,6 +966,20 @@ struct ColiV4Session {
uint64_t spec_drafted;
uint64_t spec_accepted;
int spec_disabled;
+ /* Prompt-end capture for SUBMIT pin=1: the ids fed and the head scores
+ * that predict the token after them. A later prompt that starts with
+ * exactly these ids gets its first fresh token's predictor from here; that
+ * token is the one a closed-set caller asks about (docs/brio.md). The
+ * attention state itself goes to a v4_ckpt slot; this is the part the
+ * snapshot does not hold. */
+ int *pin_ids;
+ int pin_len;
+ float *pin_scores;
+ /* Scratch for the numeric channel, one hidden row and one row of head
+ * scores, allocated on first use and freed with the session so the many
+ * early returns of generate() leave nothing behind. */
+ float *echo_hidden;
+ float *echo_scores;
};
/* RAM-tiered expert open used by coli_v4_engine_open (replaces ld --wrap).
diff --git a/c/tests/test_deepseek_v4_brio.py b/c/tests/test_deepseek_v4_brio.py
new file mode 100644
index 00000000..3fb77104
--- /dev/null
+++ b/c/tests/test_deepseek_v4_brio.py
@@ -0,0 +1,289 @@
+#!/usr/bin/env python3
+"""The numeric channel on the DeepSeek V4 serve path (SUBMIT logprobs=k / pin=1).
+
+This is what `POST /v1/brio` speaks (docs/brio.md), and #1648 found that this
+engine accepted the header and then refused the request. The properties under
+test are the exact ones a closed-set caller relies on:
+
+- a read-only request (max_tokens=0, logprobs>0) answers with one ECHO frame
+ per prompt position after the first, and generates nothing;
+- the ECHO at position p is the head over prompt[:p]: its best token is the
+ token a greedy generation from that same prefix produces;
+- a pinned prompt gives every prompt that extends it the predictor of its
+ first fresh token, from the kept scores, and those scores are the ones a
+ cold prefill computes;
+- with the channel open, generation is unchanged and every DATA frame carries
+ the tail of the token it holds.
+
+Speaks SUBMIT/ECHO/DATA/DONE directly, as tests/test_deepseek_v4_prefix.py does.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+
+def token_prompt(ids: list[int]) -> str:
+ return "".join(f"<t{token:03d}>" for token in ids)
+
+
+def parse_tokens(text: str) -> list[int]:
+ return [int(match) for match in re.findall(r"<t(\d+)>", text)]
+
+
+class Reply:
+ def __init__(self) -> None:
+ self.accepted = -1
+ self.echoes: dict[int, dict] = {} # position -> {"token", "lp", "top": {tid: tlp}}
+ self.data: list[tuple[bytes, dict | None]] = []
+ self.reuse = -1
+ self.completion = -1
+
+ @property
+ def text(self) -> str:
+ return b"".join(piece for piece, _ in self.data).decode("utf-8", "replace")
+
+
+def parse_tail(fields: list[str]) -> dict | None:
+ """`<lp> <k> [tid tlp]*k` -> {"lp": float, "top": {tid: tlp}}; None for no tail."""
+ if not fields:
+ return None
+ lp = None if fields[0] == "nan" else float(fields[0])
+ k = int(fields[1])
+ pairs = fields[2:2 + 2 * k]
+ top = {int(pairs[i]): float(pairs[i + 1]) for i in range(0, len(pairs), 2)}
+ return {"lp": lp, "top": top}
+
+
+class Serve:
+ """One persistent `SERVE=1` engine process."""
+
+ def __init__(self, binary: Path, model: Path, ctx: str = "128") -> None:
+ env = dict(os.environ, SERVE="1", SNAP=str(model), CTX=ctx)
+ self.process = subprocess.Popen(
+ [str(binary)], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, env=env,
+ )
+ self.counter = 0
+
+ def submit(self, prompt: str, max_tokens: int, logprobs: int = 0,
+ pin: bool = False) -> Reply:
+ self.counter += 1
+ request_id = f"r{self.counter}"
+ payload = prompt.encode("utf-8")
+ header = f"SUBMIT {request_id} 0 {len(payload)} {max_tokens} 0.0 1.0 0"
+ if logprobs:
+ header += f" logprobs={logprobs}"
+ if pin:
+ header += " pin=1"
+ assert self.process.stdin is not None
+ self.process.stdin.write(header.encode("ascii") + b"\n" + payload + b"\n")
+ self.process.stdin.flush()
+
+ reply = Reply()
+ stream = self.process.stdout
+ assert stream is not None
+ while True:
+ line = stream.readline()
+ if not line:
+ raise AssertionError(
+ f"engine closed stdout during {request_id}; "
+ f"stderr:\n{self._drain_stderr()}")
+ fields = line.decode("utf-8", "replace").split()
+ if not fields:
+ continue
+ kind = fields[0]
+ if kind == "ERROR":
+ raise AssertionError(f"engine ERROR: {line!r}")
+ if kind == "ACCEPT":
+ reply.accepted = int(fields[2])
+ elif kind == "DATA":
+ size = int(fields[2])
+ piece = stream.read(size)
+ stream.read(1)
+ reply.data.append((piece, parse_tail(fields[3:])))
+ elif kind == "ECHO":
+ # ECHO <id> <n> <pos> <lp> <k> [tid tlp]*k, then n bytes + LF
+ size = int(fields[2])
+ piece = stream.read(size)
+ stream.read(1)
+ tail = parse_tail(fields[4:])
+ assert tail is not None
+ reply.echoes[int(fields[3])] = {"token": piece.decode("utf-8", "replace"), **tail}
+ elif kind == "DONE":
+ # DONE <id> STAT <completion> <tok/s> <hit%> <rss> <prompt> <cap> [<reuse>]
+ reply.completion = int(fields[3])
+ if len(fields) >= 10:
+ reply.reuse = int(fields[9])
+ break
+ return reply
+
+ def _drain_stderr(self) -> str:
+ assert self.process.stderr is not None
+ try:
+ return self.process.stderr.read(8192).decode("utf-8", "replace")
+ except Exception: # pragma: no cover
+ return "<unavailable>"
+
+ def close(self) -> None:
+ try:
+ if self.process.stdin:
+ self.process.stdin.close()
+ self.process.wait(timeout=30)
+ except Exception: # pragma: no cover
+ self.process.kill()
+
+
+def best_of(echo: dict) -> int:
+ return max(echo["top"], key=lambda tid: echo["top"][tid])
+
+
+def check_read_only(binary: Path, model: Path, ids: list[int]) -> None:
+ """max_tokens=0 with logprobs: one ECHO per position after the first, no DATA."""
+ serve = Serve(binary, model)
+ try:
+ reply = serve.submit(token_prompt(ids), 0, logprobs=3)
+ finally:
+ serve.close()
+ n = len(ids)
+ if reply.accepted != n:
+ raise AssertionError(f"ACCEPT said {reply.accepted} tokens for a {n}-token prompt")
+ if reply.data:
+ raise AssertionError(f"a read-only request generated {len(reply.data)} DATA frames")
+ if reply.completion != 0:
+ raise AssertionError(f"DONE reported {reply.completion} completion tokens")
+ if sorted(reply.echoes) != list(range(1, n)):
+ raise AssertionError(f"ECHO positions {sorted(reply.echoes)}, expected 1..{n - 1}")
+ for position, echo in reply.echoes.items():
+ if parse_tokens(echo["token"]) != [ids[position]]:
+ raise AssertionError(f"ECHO at {position} names {echo['token']!r}, prompt has {ids[position]}")
+ if echo["lp"] is None or echo["lp"] > 1e-6:
+ raise AssertionError(f"ECHO at {position}: log-probability {echo['lp']} is not <= 0")
+ if len(echo["top"]) != 3:
+ raise AssertionError(f"ECHO at {position}: asked top-3, got {len(echo['top'])}")
+ mass = sum(math.exp(v) for v in echo["top"].values())
+ if mass > 1.0 + 1e-4:
+ raise AssertionError(f"ECHO at {position}: top-3 mass {mass} exceeds 1")
+ if ids[position] in echo["top"] and abs(echo["top"][ids[position]] - echo["lp"]) > 1e-5:
+ raise AssertionError(f"ECHO at {position}: the token's own lp differs from its top-k entry")
+ print(f"PASS read-only: {n - 1} ECHO frames, zero generated tokens")
+
+
+def check_echo_is_the_head(binary: Path, model: Path, ids: list[int]) -> None:
+ """The best token of the ECHO at p is what greedy decoding emits after prompt[:p]."""
+ serve = Serve(binary, model)
+ try:
+ full = serve.submit(token_prompt(ids), 0, logprobs=4)
+ for position in sorted({1, len(ids) // 2, len(ids) - 1}):
+ cold = Serve(binary, model)
+ try:
+ gen = cold.submit(token_prompt(ids[:position]), 1)
+ finally:
+ cold.close()
+ produced = parse_tokens(gen.text)
+ expected = best_of(full.echoes[position])
+ if produced != [expected]:
+ raise AssertionError(
+ f"position {position}: ECHO's best token is {expected}, "
+ f"greedy generation from the same prefix gave {produced}")
+ finally:
+ serve.close()
+ print("PASS echo is the head: best ECHO token == greedy token from the same prefix")
+
+
+def check_pin(binary: Path, model: Path, ids: list[int], a: int, b: int) -> None:
+ """Pin the prompt, ask two options: both start from the pin with the first token's predictor."""
+ n = len(ids)
+ serve = Serve(binary, model)
+ try:
+ pinned = serve.submit(token_prompt(ids), 0, logprobs=2, pin=True)
+ if pinned.completion != 0 or pinned.data:
+ raise AssertionError("the pinned read generated tokens")
+ first = serve.submit(token_prompt(ids + [a]), 0, logprobs=2)
+ second = serve.submit(token_prompt(ids + [b]), 0, logprobs=2)
+ finally:
+ serve.close()
+ for name, reply in (("first option", first), ("second option", second)):
+ if reply.reuse != n:
+ raise AssertionError(f"{name}: reused {reply.reuse} tokens, the pin holds {n}")
+ if sorted(reply.echoes) != [n]:
+ raise AssertionError(f"{name}: ECHO positions {sorted(reply.echoes)}, expected [{n}]")
+ if first.echoes[n]["top"] != second.echoes[n]["top"]:
+ raise AssertionError("the two options saw different predictors for the same pinned prompt")
+ if parse_tokens(first.echoes[n]["token"]) != [a] or parse_tokens(second.echoes[n]["token"]) != [b]:
+ raise AssertionError("the ECHO at the option position does not name the option token")
+ # the kept scores are what a cold prefill computes at that position
+ cold = Serve(binary, model)
+ try:
+ fresh = cold.submit(token_prompt(ids + [a]), 0, logprobs=2)
+ finally:
+ cold.close()
+ if fresh.reuse != 0:
+ raise AssertionError(f"cold engine reported reuse={fresh.reuse}")
+ warm_top, cold_top = first.echoes[n]["top"], fresh.echoes[n]["top"]
+ if warm_top.keys() != cold_top.keys() or any(abs(warm_top[t] - cold_top[t]) > 1e-4 for t in warm_top):
+ raise AssertionError(f"pinned predictor {warm_top} differs from a cold prefill's {cold_top}")
+ if abs(first.echoes[n]["lp"] - fresh.echoes[n]["lp"]) > 1e-4:
+ raise AssertionError("the option's log-probability differs between the pin and a cold prefill")
+ print(f"PASS pin: both options reused {n} tokens and read the pinned predictor, equal to a cold prefill")
+
+
+def check_generation_with_tail(binary: Path, model: Path, ids: list[int], max_new: int) -> None:
+ """The channel does not change greedy output, and every DATA carries its token's tail."""
+ plain = Serve(binary, model)
+ try:
+ without = plain.submit(token_prompt(ids), max_new)
+ finally:
+ plain.close()
+ channel = Serve(binary, model)
+ try:
+ with_lp = channel.submit(token_prompt(ids), max_new, logprobs=2)
+ finally:
+ channel.close()
+ if without.text != with_lp.text:
+ raise AssertionError(f"logprobs changed the output:\n plain: {without.text!r}\n channel: {with_lp.text!r}")
+ if not with_lp.data:
+ raise AssertionError("no DATA frames with the channel open")
+ for piece, tail in with_lp.data:
+ if tail is None:
+ raise AssertionError(f"DATA {piece!r} carries no logprob tail")
+ token = parse_tokens(piece.decode("utf-8", "replace"))
+ if len(token) != 1:
+ raise AssertionError(f"DATA frame is not one token: {piece!r}")
+ if len(tail["top"]) != 2:
+ raise AssertionError(f"asked top-2, got {len(tail['top'])}")
+ # greedy: the emitted token IS the best one, so its lp is the top entry
+ if best_of(tail) != token[0] or abs(tail["top"][token[0]] - tail["lp"]) > 1e-5:
+ raise AssertionError(f"DATA {token}: tail {tail} does not name it as the best token")
+ print(f"PASS generation: {len(with_lp.data)} tokens unchanged, each DATA frame with its tail")
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
+ parser.add_argument("--binary", type=Path, required=True)
+ parser.add_argument("--fixture", type=Path, required=True)
+ arguments = parser.parse_args()
+ binary = arguments.binary.resolve()
+ reference = json.loads((arguments.fixture / "ref.json").read_text("utf-8"))
+ case = reference["cases"]["short"]
+ ids = list(case["prompt_ids"])
+ greedy = list(case["greedy_new_ids"])
+ # two options: the token greedy decoding would pick, and one it would not
+ a = greedy[0]
+ b = next(t for t in ids if t != a)
+ check_read_only(binary, arguments.fixture, ids)
+ check_echo_is_the_head(binary, arguments.fixture, ids)
+ check_pin(binary, arguments.fixture, ids, a, b)
+ check_generation_with_tail(binary, arguments.fixture, ids, 4)
+ print("PASS DeepSeek V4 numeric channel: all checks completed")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())