Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(serve): olmoe and inkling refuse an over-long prompt with CONTEXT_EXCEEDED

kevin932783d47b1319bb
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 782af024..25d86973 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -590,6 +590,10 @@ jobs:
run: |
cd c
OLMOE_TINY=olmoe_tiny_c python3 -m unittest -v tests.test_olmoe_dashboard_hits
+ - name: "Over-long prompt: CONTEXT_EXCEEDED, so the gateway answers 400"
+ run: |
+ cd c
+ COLI_OLMOE_FIXTURE=olmoe_tiny_c python3 -m unittest -v tests.test_olmoe_context_exceeded
dsv41-tiny-check:
name: DeepSeek V4.1 tiny oracle (engram + DSA + fp4 experts + vision)
diff --git a/c/inkling.c b/c/inkling.c
index 1496c362..7d74b97e 100644
--- a/c/inkling.c
+++ b/c/inkling.c
@@ -2102,12 +2102,19 @@ static void apply_rep_penalty(float *logit, int n, const int *hist, int nhist, f
}
}
-/* reject a prompt that would overrun the served KV bound (CTX_MAX, default 8192) */
+/* reject a prompt that would overrun the served KV bound (CTX_MAX, default 8192).
+ * The refusal is the frame the gateway turns into a 400 context_length_exceeded
+ * (#506, #1381); free text here reached the client as a 500. One request is
+ * served at a time, so the returned buffer is only read before the next call. */
static const char *prompt_reject(int np, int want) {
+ static char message[96];
const char *cm = getenv("CTX_MAX");
int ctx_max = cm ? atoi(cm) : 8192;
- if (np + want > ctx_max) return "context exceeds CTX_MAX";
- return NULL;
+ if (np + want <= ctx_max) return NULL;
+ snprintf(message, sizeof(message),
+ "CONTEXT_EXCEEDED prompt_tokens=%d requested=%d capacity=%d",
+ np, want, ctx_max);
+ return message;
}
typedef struct { char id[64]; int max_tok; float temp, top_p; char *payload; int plen;
diff --git a/c/olmoe.c b/c/olmoe.c
index 15083f78..745eeb3d 100644
--- a/c/olmoe.c
+++ b/c/olmoe.c
@@ -1443,7 +1443,10 @@ static int serve_one(Model *m, Tok *T, SReq *q, int ctx_cap) {
if (np <= 0) { coli_serve_write_error(stdout, q->id, "empty prompt"); free(ids); return 0; }
if (np + q->max_tok > ctx_cap) {
char message[128];
- snprintf(message, sizeof(message), "context exceeds CTX (%d + %d > %d)",
+ /* The frame the gateway turns into a 400 context_length_exceeded
+ * (#506, #1381). Free text here reached the client as a 500. */
+ snprintf(message, sizeof(message),
+ "CONTEXT_EXCEEDED prompt_tokens=%d requested=%d capacity=%d",
np, q->max_tok, ctx_cap);
coli_serve_write_error(stdout, q->id, message); free(ids); return 0;
}
diff --git a/c/tests/test_inkling_serve_framing.c b/c/tests/test_inkling_serve_framing.c
index b78ebf59..9bd1e7cc 100644
--- a/c/tests/test_inkling_serve_framing.c
+++ b/c/tests/test_inkling_serve_framing.c
@@ -100,11 +100,23 @@ static void test_malformed_submit_cannot_become_control(void)
fclose(input); fclose(output);
}
+/* The gateway answers `ERROR <id> CONTEXT_EXCEEDED ...` with a 400
+ * context_length_exceeded; any other refusal text reaches the client as a 500. */
+static void test_over_long_prompt_is_refused_with_the_context_frame(void)
+{
+ assert(setenv("CTX_MAX","16",1)==0);
+ assert(prompt_reject(12,4)==NULL);
+ const char *refusal=prompt_reject(30,4);
+ assert(refusal);
+ assert(strcmp(refusal,"CONTEXT_EXCEEDED prompt_tokens=30 requested=4 capacity=16")==0);
+}
+
int main(void)
{
test_text_and_audio_submits_are_exact();
test_controls_and_queue_full_preserve_behavior();
test_malformed_submit_cannot_become_control();
+ test_over_long_prompt_is_refused_with_the_context_frame();
puts("inkling serve framing baseline: ok");
return 0;
}
diff --git a/c/tests/test_olmoe_context_exceeded.py b/c/tests/test_olmoe_context_exceeded.py
new file mode 100644
index 00000000..bf19f100
--- /dev/null
+++ b/c/tests/test_olmoe_context_exceeded.py
@@ -0,0 +1,91 @@
+"""An OLMoE prompt longer than CTX is a 400 context_length_exceeded, not a 500.
+
+The gateway maps one engine frame to the OpenAI error clients act on:
+`ERROR <id> CONTEXT_EXCEEDED ...` (#506, both spellings since #1381). OLMoE
+refused an over-long prompt with free text instead,
+`ERROR <id> context exceeds CTX (30 + 4 > 16)`, which the gateway can only
+treat as an engine fault: HTTP 500 "The colibri engine failed to process the
+request.", streaming or not. A client that compacts its conversation on
+context_length_exceeded never gets the chance to.
+
+What is asserted is the frame the real engine writes on a real container, and
+what the gateway's own mapping makes of that frame. Needs a converted tiny
+OLMoE container and a built engine, the ones the OLMoE tiny oracle job builds.
+"""
+import os
+import subprocess
+import unittest
+from pathlib import Path
+
+from openai_server import APIError, _engine_error
+
+HERE = Path(__file__).resolve().parent.parent
+ENGINE = HERE / ("olmoe.exe" if os.name == "nt" else "olmoe")
+FIXTURE = Path(os.environ.get("COLI_OLMOE_FIXTURE", ""))
+CTX = 16
+MAX_TOKENS = 4
+
+
+def serve_turn(prompt):
+ """Submit one prompt in serve mode; return the line that ends the turn."""
+ environment = {**os.environ, "SNAP": str(FIXTURE), "SERVE": "1", "CTX": str(CTX)}
+ process = subprocess.Popen([str(ENGINE), "4", "8"], env=environment,
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+ stderr=subprocess.DEVNULL, bufsize=0)
+ try:
+ while b"READY" not in (line := process.stdout.readline()):
+ if not line:
+ raise AssertionError("the engine exited before READY")
+ process.stdin.write(f"SUBMIT 7 0 {len(prompt)} {MAX_TOKENS} 0 1\n".encode()
+ + prompt + b"\n")
+ process.stdin.flush()
+ for _ in range(200):
+ line = process.stdout.readline()
+ if not line:
+ raise AssertionError("the engine closed the stream mid-turn")
+ text = line.decode("latin-1").rstrip("\n")
+ if text.startswith("DATA "):
+ process.stdout.read(int(text.split()[2]))
+ process.stdout.readline()
+ elif text.startswith(("ERROR ", "DONE ")):
+ return text
+ raise AssertionError("no ERROR or DONE within 200 lines")
+ finally:
+ process.stdin.close()
+ try:
+ process.wait(30)
+ except subprocess.TimeoutExpired:
+ process.kill()
+ process.wait()
+ process.stdout.close()
+
+
[email protected](ENGINE.exists(), "olmoe engine is not built")
[email protected](FIXTURE.name and (FIXTURE / "config.json").is_file(),
+ "COLI_OLMOE_FIXTURE not set to a converted OLMoE container")
+class OlmoeContextExceededTest(unittest.TestCase):
+ OVER_LONG = b"abc" * 10 # 30 byte-level tokens, 30 + 4 > CTX
+
+ def test_an_over_long_prompt_is_refused_with_the_context_frame(self):
+ fields = serve_turn(self.OVER_LONG).split()
+ self.assertEqual(fields[:3], ["ERROR", "7", "CONTEXT_EXCEEDED"], fields)
+ numbers = dict(field.split("=", 1) for field in fields[3:])
+ self.assertEqual(numbers, {"prompt_tokens": "30",
+ "requested": str(MAX_TOKENS),
+ "capacity": str(CTX)})
+
+ def test_the_gateway_answers_it_with_a_400(self):
+ fields = serve_turn(self.OVER_LONG).split()
+ error = _engine_error(fields[2:], " ".join(fields[2:]))
+ self.assertIsInstance(error, APIError,
+ "the gateway reads this frame as an engine fault (HTTP 500)")
+ self.assertEqual((error.status, error.code), (400, "context_length_exceeded"))
+ self.assertIn(f"maximum context length is {CTX} tokens", error.message)
+ self.assertIn("at least 30 tokens", error.message)
+
+ def test_a_prompt_that_fits_is_still_served(self):
+ self.assertTrue(serve_turn(b"abc").startswith("DONE 7 "))
+
+
+if __name__ == "__main__":
+ unittest.main()