Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(v41): clamp output budgets and reject oversized prompts

ZacharyZcR71622ebea0a1
diff --git a/c/Makefile b/c/Makefile
index 55a8bfeb..273ff24a 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1295,6 +1295,9 @@ tests/test_qwen38_tokenizer$(EXE): tests/test_qwen38_tokenizer.c qwen38.c qwen38
tests/test_qwen38_config$(EXE): tests/test_qwen38_config.c qwen38.c qwen38_core.h qwen38_nfc.h qwen38_nfc_tables.h st.h json.h compat.h quant.h idot.h route_trace.h tok_unicode.h tok_unicode_o200k.h
$(CC) $(NOCUDA_CFLAGS) $< -o $@ $(NOCUDA_LDFLAGS)
+tests/test_dsv41_serve_budget$(EXE): tests/test_dsv41_serve_budget.c deepseek_v41.c cli_args.h st.h json.h tok.h tok_unicode.h compat.h omp_tune.h kv_prefix.h pin_pool.h quant.h idot.h sparse_attn.h hyper_connections.h serve_codec.h serve_poll.h
+ $(CC) $(NOCUDA_CFLAGS) $< -o $@ $(NOCUDA_LDFLAGS)
+
tests/test_qwen38_serve_framing$(EXE): tests/test_qwen38_serve_framing.c qwen38.c qwen38_core.h qwen38_nfc.h qwen38_nfc_tables.h st.h json.h compat.h quant.h idot.h route_trace.h serve_codec.h tok_unicode.h tok_unicode_o200k.h
$(CC) $(NOCUDA_CFLAGS) $< -o $@ $(NOCUDA_LDFLAGS)
diff --git a/c/deepseek_v41.c b/c/deepseek_v41.c
index e4f31b28..93860212 100644
--- a/c/deepseek_v41.c
+++ b/c/deepseek_v41.c
@@ -3522,6 +3522,16 @@ static int serve_eos(Model *m, const char *snap, int *ids, int cap) {
return n;
}
+/* max_tokens is a ceiling, as on V4 and Qwen (#1641). A score-only
+ * request may fill the context; generation needs at least one free position. */
+static int serve_budget(int prompt, int requested, int context, int logprobs) {
+ if (prompt < 1 || prompt > context) return -1;
+ int budget = requested > 0 ? requested : (logprobs > 0 ? 0 : 256);
+ int room = context - prompt;
+ if (budget > 0 && room == 0) return -1;
+ return budget < room ? budget : room;
+}
+
static void serve_loop(Model *m, Tok *tokenizer, const char *snap) {
Cfg *c = &m->c;
coli_serve_stdio_init();
@@ -3530,7 +3540,9 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *snap) {
coli_serve_write_ready(stdout, rss_gb());
serve_emap(m);
float *logits = xmalloc((size_t)c->vocab * sizeof(float), "logits");
- int *ids = xmalloc((size_t)c->max_positions * sizeof(int), "prompt ids");
+ /* tok_encode stops at its output capacity: one extra id distinguishes
+ * a full, valid read-only prompt from a silently truncated one. */
+ int *ids = xmalloc(((size_t)c->max_positions + 1) * sizeof(int), "prompt ids");
float *pending_image = NULL;
int pending_h = 0, pending_w = 0;
@@ -3586,7 +3598,22 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *snap) {
mir_reads0[r] = g_mir_nread[r];
}
int n_prompt = tok_encode(tokenizer, (const char *)command.payload,
- (int)command.payload_bytes, ids, c->max_positions);
+ (int)command.payload_bytes, ids, c->max_positions + 1);
+ int budget = serve_budget(n_prompt, command.max_tokens, c->max_positions,
+ command.logprobs);
+ if (budget < 0) {
+ char message[128];
+ snprintf(message, sizeof(message),
+ "CONTEXT_EXCEEDED prompt_tokens=%d requested=%d capacity=%d",
+ n_prompt, command.max_tokens, c->max_positions);
+ coli_serve_write_error(stdout, command.id,
+ n_prompt < 1 ? "EMPTY_PROMPT" : message);
+ coli_serve_command_dispose(&command); continue;
+ }
+ if (command.max_tokens > budget)
+ fprintf(stderr, "[serve] max_tokens %d clamped to %d (context %d - prompt %d); "
+ "raise CTX for longer answers\n",
+ command.max_tokens, budget, c->max_positions, n_prompt);
/* Decided BEFORE the reset, because the reset is what it decides about.
* A chat client resends the whole transcript every turn; if this prompt
* begins with the ids the state was built from, that state already IS
@@ -3634,23 +3661,6 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *snap) {
fflush(stderr);
}
if (!reuse) model_reset(m);
- if (n_prompt < 1) {
- coli_serve_write_error(stdout, command.id, "EMPTY_PROMPT");
- coli_serve_command_dispose(&command); continue;
- }
- /* max_tokens=0 con logprobs>0 vuol dire "leggi e fermati": non e un
- * valore mancante da rimpiazzare con un default, ed e' proprio il caso
- * in cui un menu chiuso non vuole pagare un passo di decodifica per
- * opzione. Senza logprobs 0 resta "non specificato" -> 256, come prima. */
- int budget = command.max_tokens > 0 ? command.max_tokens
- : (command.logprobs > 0 ? 0 : 256);
- if (n_prompt + budget > c->max_positions) {
- char message[128];
- snprintf(message, sizeof(message), "CONTEXT_EXCEEDED %d %d",
- n_prompt + budget, c->max_positions);
- coli_serve_write_error(stdout, command.id, message);
- coli_serve_command_dispose(&command); continue;
- }
coli_serve_write_accept(stdout, command.id, n_prompt);
float *aligned = NULL;
uint8_t *image_mask = NULL;
diff --git a/c/tests/test_dsv41_prefix_serve.py b/c/tests/test_dsv41_prefix_serve.py
index 949b0f5e..d7413e6e 100644
--- a/c/tests/test_dsv41_prefix_serve.py
+++ b/c/tests/test_dsv41_prefix_serve.py
@@ -30,6 +30,55 @@ ENGINE = HERE / ("deepseek_v41.exe" if sys.platform == "win32" else "deepseek_v4
FIXTURE = Path(os.environ.get("DSV41_TINY", HERE / "dsv41_tiny"))
[email protected](ENGINE.exists() and (FIXTURE / "model.safetensors").exists(),
+ "built V4.1 engine and tiny fixture required")
+class Dsv41ContextTest(unittest.TestCase):
+ def setUp(self):
+ self.engine = ServeEngine(ENGINE, ["8"],
+ {"SNAP": str(FIXTURE), "SERVE": "1", "CTX": "32",
+ "OMP_NUM_THREADS": "2", "V41_DSPARK": "0"}, reuse=False)
+ self.addCleanup(self.engine.p.stdout.close)
+ self.addCleanup(self.engine.p.stderr.close)
+ self.addCleanup(self.engine.close)
+
+ def request(self, size, budget, logprobs=0):
+ prompt = b"a" * size # the fixture's tokenizer has one id per byte
+ header = f"SUBMIT ctx 0 {size} {budget} 0 1 logprobs={logprobs}\n".encode()
+ self.engine.p.stdin.write(header + prompt + b"\n")
+ self.engine.p.stdin.flush()
+ frames = []
+ while True:
+ line = self.engine.p.stdout.readline()
+ self.assertTrue(line, "engine closed before completing the request")
+ fields = line.decode().split()
+ if fields[0] in ("DATA", "ECHO"):
+ self.engine.p.stdout.read(int(fields[2]))
+ self.engine.p.stdout.readline()
+ frames.append(fields)
+ if fields[0] in ("DONE", "ERROR"):
+ return frames
+
+ def test_output_ceiling_leaves_one_token(self):
+ frames = self.request(31, 5000)
+ self.assertEqual(frames[-1][:4], ["DONE", "ctx", "STAT", "1"])
+ self.assertIn(["ACCEPT", "ctx", "31"], frames)
+
+ def test_full_context_is_valid_only_for_scoring(self):
+ frames = self.request(32, 0, logprobs=1)
+ self.assertEqual(frames[-1][:4], ["DONE", "ctx", "STAT", "0"])
+ self.assertEqual(sum(f[0] == "ECHO" for f in frames), 31)
+ self.assertFalse(any(f[0] == "DATA" for f in frames))
+ frames = self.request(32, 1)
+ self.assertEqual(frames[-1][:3], ["ERROR", "ctx", "CONTEXT_EXCEEDED"])
+
+ def test_oversized_scoring_prompt_is_not_truncated(self):
+ frames = self.request(33, 0, logprobs=1)
+ self.assertEqual(frames[-1][:3], ["ERROR", "ctx", "CONTEXT_EXCEEDED"])
+ self.assertFalse(any(f[0] in ("ACCEPT", "ECHO", "DATA") for f in frames))
+ # Refusal drains the request and leaves the next one usable.
+ self.assertEqual(self.request(4, 1)[-1][:4], ["DONE", "ctx", "STAT", "1"])
+
+
@unittest.skipUnless(ENGINE.exists(), "deepseek_v41 is not built")
@unittest.skipUnless((FIXTURE / "model.safetensors").exists(),
"tiny dsv41 fixture is absent (tools/make_dsv41_tiny.py)")
diff --git a/c/tests/test_dsv41_serve_budget.c b/c/tests/test_dsv41_serve_budget.c
new file mode 100644
index 00000000..6477527f
--- /dev/null
+++ b/c/tests/test_dsv41_serve_budget.c
@@ -0,0 +1,28 @@
+/* The serve contract accepts a fitting prompt even if the requested output
+ * budget is too large, and never turns a read-only request into generation. */
+#define main dsv41_main_unused
+#include "../deepseek_v41.c"
+#undef main
+#include <limits.h>
+
+#define CHECK(c) do { if (!(c)) { \
+ fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #c); return 1; \
+} } while (0)
+
+int main(void) {
+ CHECK(serve_budget(20, 5000, 4096, 0) == 4076);
+ CHECK(serve_budget(20, 10, 4096, 0) == 10);
+ CHECK(serve_budget(4095, 5000, 4096, 0) == 1);
+ CHECK(serve_budget(4096, 1, 4096, 0) == -1);
+ CHECK(serve_budget(4097, 1, 4096, 0) == -1);
+ CHECK(serve_budget(0, 1, 4096, 0) == -1);
+ CHECK(serve_budget(20, 0, 4096, 0) == 256);
+ CHECK(serve_budget(4095, 0, 4096, 0) == 1);
+ CHECK(serve_budget(4096, 0, 4096, 1) == 0);
+ CHECK(serve_budget(4097, 0, 4096, 1) == -1);
+ CHECK(serve_budget(20, 0, 4096, 1) == 0);
+ CHECK(serve_budget(4095, 100, 4096, 1) == 1);
+ CHECK(serve_budget(20, INT_MAX, 4096, 0) == 4076);
+ puts("dsv41 serve budget: ok");
+ return 0;
+}