Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(v41): clamp output budgets and reject oversized prompts
ZacharyZcR71622ebea0a1
diff --git a/c/Makefile b/c/Makefileindex 55a8bfeb..273ff24a 100644--- a/c/Makefile+++ b/c/Makefile@@ -1295,6 +1295,9 @@ tests/test_qwen38_tokenizer$(EXE): tests/test_qwen38_tokenizer.c qwen38.c qwen38tests/test_qwen38_config$(EXE): tests/test_qwen38_config.c qwen38.c qwen38_core.h qwen38_nfc.h qwen38_nfc_tables.h st.h json.h compat.h quant.h idot.h route_trace.h tok_unicode.h tok_unicode_o200k.h$(CC) $(NOCUDA_CFLAGS) $< -o $@ $(NOCUDA_LDFLAGS)+tests/test_dsv41_serve_budget$(EXE): tests/test_dsv41_serve_budget.c deepseek_v41.c cli_args.h st.h json.h tok.h tok_unicode.h compat.h omp_tune.h kv_prefix.h pin_pool.h quant.h idot.h sparse_attn.h hyper_connections.h serve_codec.h serve_poll.h+ $(CC) $(NOCUDA_CFLAGS) $< -o $@ $(NOCUDA_LDFLAGS)+tests/test_qwen38_serve_framing$(EXE): tests/test_qwen38_serve_framing.c qwen38.c qwen38_core.h qwen38_nfc.h qwen38_nfc_tables.h st.h json.h compat.h quant.h idot.h route_trace.h serve_codec.h tok_unicode.h tok_unicode_o200k.h$(CC) $(NOCUDA_CFLAGS) $< -o $@ $(NOCUDA_LDFLAGS)diff --git a/c/deepseek_v41.c b/c/deepseek_v41.cindex e4f31b28..93860212 100644--- a/c/deepseek_v41.c+++ b/c/deepseek_v41.c@@ -3522,6 +3522,16 @@ static int serve_eos(Model *m, const char *snap, int *ids, int cap) {return n;}+/* max_tokens is a ceiling, as on V4 and Qwen (#1641). A score-only+ * request may fill the context; generation needs at least one free position. */+static int serve_budget(int prompt, int requested, int context, int logprobs) {+ if (prompt < 1 || prompt > context) return -1;+ int budget = requested > 0 ? requested : (logprobs > 0 ? 0 : 256);+ int room = context - prompt;+ if (budget > 0 && room == 0) return -1;+ return budget < room ? budget : room;+}+static void serve_loop(Model *m, Tok *tokenizer, const char *snap) {Cfg *c = &m->c;coli_serve_stdio_init();@@ -3530,7 +3540,9 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *snap) {coli_serve_write_ready(stdout, rss_gb());serve_emap(m);float *logits = xmalloc((size_t)c->vocab * sizeof(float), "logits");- int *ids = xmalloc((size_t)c->max_positions * sizeof(int), "prompt ids");+ /* tok_encode stops at its output capacity: one extra id distinguishes+ * a full, valid read-only prompt from a silently truncated one. */+ int *ids = xmalloc(((size_t)c->max_positions + 1) * sizeof(int), "prompt ids");float *pending_image = NULL;int pending_h = 0, pending_w = 0;@@ -3586,7 +3598,22 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *snap) {mir_reads0[r] = g_mir_nread[r];}int n_prompt = tok_encode(tokenizer, (const char *)command.payload,- (int)command.payload_bytes, ids, c->max_positions);+ (int)command.payload_bytes, ids, c->max_positions + 1);+ int budget = serve_budget(n_prompt, command.max_tokens, c->max_positions,+ command.logprobs);+ if (budget < 0) {+ char message[128];+ snprintf(message, sizeof(message),+ "CONTEXT_EXCEEDED prompt_tokens=%d requested=%d capacity=%d",+ n_prompt, command.max_tokens, c->max_positions);+ coli_serve_write_error(stdout, command.id,+ n_prompt < 1 ? "EMPTY_PROMPT" : message);+ coli_serve_command_dispose(&command); continue;+ }+ if (command.max_tokens > budget)+ fprintf(stderr, "[serve] max_tokens %d clamped to %d (context %d - prompt %d); "+ "raise CTX for longer answers\n",+ command.max_tokens, budget, c->max_positions, n_prompt);/* Decided BEFORE the reset, because the reset is what it decides about.* A chat client resends the whole transcript every turn; if this prompt* begins with the ids the state was built from, that state already IS@@ -3634,23 +3661,6 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *snap) {fflush(stderr);}if (!reuse) model_reset(m);- if (n_prompt < 1) {- coli_serve_write_error(stdout, command.id, "EMPTY_PROMPT");- coli_serve_command_dispose(&command); continue;- }- /* max_tokens=0 con logprobs>0 vuol dire "leggi e fermati": non e un- * valore mancante da rimpiazzare con un default, ed e' proprio il caso- * in cui un menu chiuso non vuole pagare un passo di decodifica per- * opzione. Senza logprobs 0 resta "non specificato" -> 256, come prima. */- int budget = command.max_tokens > 0 ? command.max_tokens- : (command.logprobs > 0 ? 0 : 256);- if (n_prompt + budget > c->max_positions) {- char message[128];- snprintf(message, sizeof(message), "CONTEXT_EXCEEDED %d %d",- n_prompt + budget, c->max_positions);- coli_serve_write_error(stdout, command.id, message);- coli_serve_command_dispose(&command); continue;- }coli_serve_write_accept(stdout, command.id, n_prompt);float *aligned = NULL;uint8_t *image_mask = NULL;diff --git a/c/tests/test_dsv41_prefix_serve.py b/c/tests/test_dsv41_prefix_serve.pyindex 949b0f5e..d7413e6e 100644--- a/c/tests/test_dsv41_prefix_serve.py+++ b/c/tests/test_dsv41_prefix_serve.py@@ -30,6 +30,55 @@ ENGINE = HERE / ("deepseek_v41.exe" if sys.platform == "win32" else "deepseek_v4FIXTURE = Path(os.environ.get("DSV41_TINY", HERE / "dsv41_tiny"))[email protected](ENGINE.exists() and (FIXTURE / "model.safetensors").exists(),+ "built V4.1 engine and tiny fixture required")+class Dsv41ContextTest(unittest.TestCase):+ def setUp(self):+ self.engine = ServeEngine(ENGINE, ["8"],+ {"SNAP": str(FIXTURE), "SERVE": "1", "CTX": "32",+ "OMP_NUM_THREADS": "2", "V41_DSPARK": "0"}, reuse=False)+ self.addCleanup(self.engine.p.stdout.close)+ self.addCleanup(self.engine.p.stderr.close)+ self.addCleanup(self.engine.close)++ def request(self, size, budget, logprobs=0):+ prompt = b"a" * size # the fixture's tokenizer has one id per byte+ header = f"SUBMIT ctx 0 {size} {budget} 0 1 logprobs={logprobs}\n".encode()+ self.engine.p.stdin.write(header + prompt + b"\n")+ self.engine.p.stdin.flush()+ frames = []+ while True:+ line = self.engine.p.stdout.readline()+ self.assertTrue(line, "engine closed before completing the request")+ fields = line.decode().split()+ if fields[0] in ("DATA", "ECHO"):+ self.engine.p.stdout.read(int(fields[2]))+ self.engine.p.stdout.readline()+ frames.append(fields)+ if fields[0] in ("DONE", "ERROR"):+ return frames++ def test_output_ceiling_leaves_one_token(self):+ frames = self.request(31, 5000)+ self.assertEqual(frames[-1][:4], ["DONE", "ctx", "STAT", "1"])+ self.assertIn(["ACCEPT", "ctx", "31"], frames)++ def test_full_context_is_valid_only_for_scoring(self):+ frames = self.request(32, 0, logprobs=1)+ self.assertEqual(frames[-1][:4], ["DONE", "ctx", "STAT", "0"])+ self.assertEqual(sum(f[0] == "ECHO" for f in frames), 31)+ self.assertFalse(any(f[0] == "DATA" for f in frames))+ frames = self.request(32, 1)+ self.assertEqual(frames[-1][:3], ["ERROR", "ctx", "CONTEXT_EXCEEDED"])++ def test_oversized_scoring_prompt_is_not_truncated(self):+ frames = self.request(33, 0, logprobs=1)+ self.assertEqual(frames[-1][:3], ["ERROR", "ctx", "CONTEXT_EXCEEDED"])+ self.assertFalse(any(f[0] in ("ACCEPT", "ECHO", "DATA") for f in frames))+ # Refusal drains the request and leaves the next one usable.+ self.assertEqual(self.request(4, 1)[-1][:4], ["DONE", "ctx", "STAT", "1"])++@unittest.skipUnless(ENGINE.exists(), "deepseek_v41 is not built")@unittest.skipUnless((FIXTURE / "model.safetensors").exists(),"tiny dsv41 fixture is absent (tools/make_dsv41_tiny.py)")diff --git a/c/tests/test_dsv41_serve_budget.c b/c/tests/test_dsv41_serve_budget.cnew file mode 100644index 00000000..6477527f--- /dev/null+++ b/c/tests/test_dsv41_serve_budget.c@@ -0,0 +1,28 @@+/* The serve contract accepts a fitting prompt even if the requested output+ * budget is too large, and never turns a read-only request into generation. */+#define main dsv41_main_unused+#include "../deepseek_v41.c"+#undef main+#include <limits.h>++#define CHECK(c) do { if (!(c)) { \+ fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #c); return 1; \+} } while (0)++int main(void) {+ CHECK(serve_budget(20, 5000, 4096, 0) == 4076);+ CHECK(serve_budget(20, 10, 4096, 0) == 10);+ CHECK(serve_budget(4095, 5000, 4096, 0) == 1);+ CHECK(serve_budget(4096, 1, 4096, 0) == -1);+ CHECK(serve_budget(4097, 1, 4096, 0) == -1);+ CHECK(serve_budget(0, 1, 4096, 0) == -1);+ CHECK(serve_budget(20, 0, 4096, 0) == 256);+ CHECK(serve_budget(4095, 0, 4096, 0) == 1);+ CHECK(serve_budget(4096, 0, 4096, 1) == 0);+ CHECK(serve_budget(4097, 0, 4096, 1) == -1);+ CHECK(serve_budget(20, 0, 4096, 1) == 0);+ CHECK(serve_budget(4095, 100, 4096, 1) == 1);+ CHECK(serve_budget(20, INT_MAX, 4096, 0) == 4076);+ puts("dsv41 serve budget: ok");+ return 0;+}