Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(serve): olmoe and inkling refuse an over-long prompt with CONTEXT_EXCEEDED
kevin932783d47b1319bb
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 782af024..25d86973 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -590,6 +590,10 @@ jobs:run: |cd cOLMOE_TINY=olmoe_tiny_c python3 -m unittest -v tests.test_olmoe_dashboard_hits+ - name: "Over-long prompt: CONTEXT_EXCEEDED, so the gateway answers 400"+ run: |+ cd c+ COLI_OLMOE_FIXTURE=olmoe_tiny_c python3 -m unittest -v tests.test_olmoe_context_exceededdsv41-tiny-check:name: DeepSeek V4.1 tiny oracle (engram + DSA + fp4 experts + vision)diff --git a/c/inkling.c b/c/inkling.cindex 1496c362..7d74b97e 100644--- a/c/inkling.c+++ b/c/inkling.c@@ -2102,12 +2102,19 @@ static void apply_rep_penalty(float *logit, int n, const int *hist, int nhist, f}}-/* reject a prompt that would overrun the served KV bound (CTX_MAX, default 8192) */+/* reject a prompt that would overrun the served KV bound (CTX_MAX, default 8192).+ * The refusal is the frame the gateway turns into a 400 context_length_exceeded+ * (#506, #1381); free text here reached the client as a 500. One request is+ * served at a time, so the returned buffer is only read before the next call. */static const char *prompt_reject(int np, int want) {+ static char message[96];const char *cm = getenv("CTX_MAX");int ctx_max = cm ? atoi(cm) : 8192;- if (np + want > ctx_max) return "context exceeds CTX_MAX";- return NULL;+ if (np + want <= ctx_max) return NULL;+ snprintf(message, sizeof(message),+ "CONTEXT_EXCEEDED prompt_tokens=%d requested=%d capacity=%d",+ np, want, ctx_max);+ return message;}typedef struct { char id[64]; int max_tok; float temp, top_p; char *payload; int plen;diff --git a/c/olmoe.c b/c/olmoe.cindex 15083f78..745eeb3d 100644--- a/c/olmoe.c+++ b/c/olmoe.c@@ -1443,7 +1443,10 @@ static int serve_one(Model *m, Tok *T, SReq *q, int ctx_cap) {if (np <= 0) { coli_serve_write_error(stdout, q->id, "empty prompt"); free(ids); return 0; }if (np + q->max_tok > ctx_cap) {char message[128];- snprintf(message, sizeof(message), "context exceeds CTX (%d + %d > %d)",+ /* The frame the gateway turns into a 400 context_length_exceeded+ * (#506, #1381). Free text here reached the client as a 500. */+ snprintf(message, sizeof(message),+ "CONTEXT_EXCEEDED prompt_tokens=%d requested=%d capacity=%d",np, q->max_tok, ctx_cap);coli_serve_write_error(stdout, q->id, message); free(ids); return 0;}diff --git a/c/tests/test_inkling_serve_framing.c b/c/tests/test_inkling_serve_framing.cindex b78ebf59..9bd1e7cc 100644--- a/c/tests/test_inkling_serve_framing.c+++ b/c/tests/test_inkling_serve_framing.c@@ -100,11 +100,23 @@ static void test_malformed_submit_cannot_become_control(void)fclose(input); fclose(output);}+/* The gateway answers `ERROR <id> CONTEXT_EXCEEDED ...` with a 400+ * context_length_exceeded; any other refusal text reaches the client as a 500. */+static void test_over_long_prompt_is_refused_with_the_context_frame(void)+{+ assert(setenv("CTX_MAX","16",1)==0);+ assert(prompt_reject(12,4)==NULL);+ const char *refusal=prompt_reject(30,4);+ assert(refusal);+ assert(strcmp(refusal,"CONTEXT_EXCEEDED prompt_tokens=30 requested=4 capacity=16")==0);+}+int main(void){test_text_and_audio_submits_are_exact();test_controls_and_queue_full_preserve_behavior();test_malformed_submit_cannot_become_control();+ test_over_long_prompt_is_refused_with_the_context_frame();puts("inkling serve framing baseline: ok");return 0;}diff --git a/c/tests/test_olmoe_context_exceeded.py b/c/tests/test_olmoe_context_exceeded.pynew file mode 100644index 00000000..bf19f100--- /dev/null+++ b/c/tests/test_olmoe_context_exceeded.py@@ -0,0 +1,91 @@+"""An OLMoE prompt longer than CTX is a 400 context_length_exceeded, not a 500.++The gateway maps one engine frame to the OpenAI error clients act on:+`ERROR <id> CONTEXT_EXCEEDED ...` (#506, both spellings since #1381). OLMoE+refused an over-long prompt with free text instead,+`ERROR <id> context exceeds CTX (30 + 4 > 16)`, which the gateway can only+treat as an engine fault: HTTP 500 "The colibri engine failed to process the+request.", streaming or not. A client that compacts its conversation on+context_length_exceeded never gets the chance to.++What is asserted is the frame the real engine writes on a real container, and+what the gateway's own mapping makes of that frame. Needs a converted tiny+OLMoE container and a built engine, the ones the OLMoE tiny oracle job builds.+"""+import os+import subprocess+import unittest+from pathlib import Path++from openai_server import APIError, _engine_error++HERE = Path(__file__).resolve().parent.parent+ENGINE = HERE / ("olmoe.exe" if os.name == "nt" else "olmoe")+FIXTURE = Path(os.environ.get("COLI_OLMOE_FIXTURE", ""))+CTX = 16+MAX_TOKENS = 4+++def serve_turn(prompt):+ """Submit one prompt in serve mode; return the line that ends the turn."""+ environment = {**os.environ, "SNAP": str(FIXTURE), "SERVE": "1", "CTX": str(CTX)}+ process = subprocess.Popen([str(ENGINE), "4", "8"], env=environment,+ stdin=subprocess.PIPE, stdout=subprocess.PIPE,+ stderr=subprocess.DEVNULL, bufsize=0)+ try:+ while b"READY" not in (line := process.stdout.readline()):+ if not line:+ raise AssertionError("the engine exited before READY")+ process.stdin.write(f"SUBMIT 7 0 {len(prompt)} {MAX_TOKENS} 0 1\n".encode()+ + prompt + b"\n")+ process.stdin.flush()+ for _ in range(200):+ line = process.stdout.readline()+ if not line:+ raise AssertionError("the engine closed the stream mid-turn")+ text = line.decode("latin-1").rstrip("\n")+ if text.startswith("DATA "):+ process.stdout.read(int(text.split()[2]))+ process.stdout.readline()+ elif text.startswith(("ERROR ", "DONE ")):+ return text+ raise AssertionError("no ERROR or DONE within 200 lines")+ finally:+ process.stdin.close()+ try:+ process.wait(30)+ except subprocess.TimeoutExpired:+ process.kill()+ process.wait()+ process.stdout.close()++[email protected](ENGINE.exists(), "olmoe engine is not built")[email protected](FIXTURE.name and (FIXTURE / "config.json").is_file(),+ "COLI_OLMOE_FIXTURE not set to a converted OLMoE container")+class OlmoeContextExceededTest(unittest.TestCase):+ OVER_LONG = b"abc" * 10 # 30 byte-level tokens, 30 + 4 > CTX++ def test_an_over_long_prompt_is_refused_with_the_context_frame(self):+ fields = serve_turn(self.OVER_LONG).split()+ self.assertEqual(fields[:3], ["ERROR", "7", "CONTEXT_EXCEEDED"], fields)+ numbers = dict(field.split("=", 1) for field in fields[3:])+ self.assertEqual(numbers, {"prompt_tokens": "30",+ "requested": str(MAX_TOKENS),+ "capacity": str(CTX)})++ def test_the_gateway_answers_it_with_a_400(self):+ fields = serve_turn(self.OVER_LONG).split()+ error = _engine_error(fields[2:], " ".join(fields[2:]))+ self.assertIsInstance(error, APIError,+ "the gateway reads this frame as an engine fault (HTTP 500)")+ self.assertEqual((error.status, error.code), (400, "context_length_exceeded"))+ self.assertIn(f"maximum context length is {CTX} tokens", error.message)+ self.assertIn("at least 30 tokens", error.message)++ def test_a_prompt_that_fits_is_still_served(self):+ self.assertTrue(serve_turn(b"abc").startswith("DONE 7 "))+++if __name__ == "__main__":+ unittest.main()