Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(ctx): the context ceiling is announced, refused when impossible, and reported with the right number (#1376)
JustVuggaf3e7cf05ea2
diff --git a/c/coli b/c/coliindex 5fe9bc6d..692dcdeb 100755--- a/c/coli+++ b/c/coli@@ -492,7 +492,14 @@ def env_for_engine(a, arch, plan=None):except (TypeError, ValueError):passif a.ctx:- env[family_by_id(arch).limits.context_env] = str(a.ctx)+ limits = family_by_id(arch).limits+ # #1376: the engine clamps silently to its hard ceiling. A flag that is+ # accepted and then ignored is worse than one refused with the number.+ if limits.max_context and a.ctx > limits.max_context:+ sys.exit(f"{C.yel}--ctx {a.ctx} exceeds what {family_by_id(arch).display_name} "+ f"supports ({limits.max_context} tokens).{C.r}\n"+ f" Pass --ctx {limits.max_context} or less.")+ env[limits.context_env] = str(a.ctx)if arch == "deepseek_v4":# Speculation stays opt-in. Real multi-turn chat measured only 1/15# accepted prompt-lookup candidates; recurrent-state replay dominateddiff --git a/c/openai_server.py b/c/openai_server.pyindex bfb9436c..41748758 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -84,8 +84,20 @@ def _engine_error(fields, message):know how to compact a conversation actually get the chance to (previously the enginesilently truncated the prompt instead, which is #401)."""if fields and fields[0] == "CONTEXT_EXCEEDED":- limit = fields[2] if len(fields) > 2 else "the context"- used = fields[1] if len(fields) > 1 else "?"+ # Two spellings of the same frame. colibri and deepseek_v4 write the+ # original `CONTEXT_EXCEEDED <used> <limit>`; qwen36 and qwen38 write+ # `prompt_tokens=N requested=M capacity=C`. Reading the second by+ # position took "requested=M" (the completion budget) as the limit and+ # printed it raw: "maximum context length is requested=4 tokens", with+ # the real ceiling nowhere (#1376). Unifying the engines' spelling is+ # a separate change; the server must read both meanwhile.+ kv = dict(f.split("=", 1) for f in fields[1:] if "=" in f)+ if kv:+ limit = kv.get("capacity") or "the context"+ used = kv.get("prompt_tokens") or "?"+ else:+ limit = fields[2] if len(fields) > 2 else "the context"+ used = fields[1] if len(fields) > 1 else "?"return APIError(400,f"This model's maximum context length is {limit} tokens, however your "f"messages resulted in at least {used} tokens. Please shorten the "diff --git a/c/qwen36.c b/c/qwen36.cindex 26abfa35..b0be3ef7 100644--- a/c/qwen36.c+++ b/c/qwen36.c@@ -2751,8 +2751,11 @@ int main(int argc, char **argv) {float smooth = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f;float conf = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f;- fprintf(stderr, "== qwen36 Phase-2 engine | cache=%d/layer bits=%d pilot=%d wide=%d hot=%d smooth=%.2f conf=%.2f ==\n",- cap, bits, g_pilot, g_wide, hot_n, smooth, conf);+ /* #1376: every capacity knob announced itself here except the one that+ * refuses requests. The context ceiling surfaced only in the+ * CONTEXT_EXCEEDED line, i.e. after a request had already failed. */+ fprintf(stderr, "== qwen36 Phase-2 engine | cache=%d/layer bits=%d ctx=%d pilot=%d wide=%d hot=%d smooth=%.2f conf=%.2f ==\n",+ cap, bits, qwen36_max_ctx(), g_pilot, g_wide, hot_n, smooth, conf);int is_ref = 0;diff --git a/c/tests/test_cli_output.py b/c/tests/test_cli_output.pyindex 603e979c..61cd3c45 100644--- a/c/tests/test_cli_output.py+++ b/c/tests/test_cli_output.py@@ -377,5 +377,39 @@ class OmpThreadsForEveryEngineTest(unittest.TestCase):self.coli.env_for_engine(vram_args, "qwen38")+class ContextFlagHonestyTest(unittest.TestCase):+ """#1376: `--ctx` above what a family supports was accepted, then clamped+ by the engine in silence. A flag that appears to work and does not is+ worse than one refused with the number."""++ @classmethod+ def setUpClass(cls):+ loader = SourceFileLoader("coli_ctx_under_test", str(CLI))+ spec = importlib.util.spec_from_loader(loader.name, loader)+ cls.coli = importlib.util.module_from_spec(spec)+ loader.exec_module(cls.coli)++ class Args(types.SimpleNamespace):+ """env_for_engine reads whatever flags the family cares about; every+ one not set here reads as "not given", which is what argparse yields."""+ def __getattr__(self, name):+ return None++ def args(self, ctx):+ return self.Args(ctx=ctx, ram=0)++ def test_ctx_above_the_family_maximum_is_refused_with_the_number(self):+ family = self.coli.family_by_id("qwen36")+ too_big = family.limits.max_context + 1+ with self.assertRaises(SystemExit) as stop:+ self.coli.env_for_engine(self.args(too_big), "qwen36")+ self.assertIn(str(family.limits.max_context), str(stop.exception))++ def test_ctx_within_the_maximum_reaches_the_engine_variable(self):+ family = self.coli.family_by_id("qwen36")+ env = self.coli.env_for_engine(self.args(65536), "qwen36")+ self.assertEqual(env.get(family.limits.context_env), "65536")++if __name__ == "__main__":unittest.main()diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex 08379ba9..b9de137e 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -2611,5 +2611,35 @@ class ImageUrlPathGuard(unittest.TestCase):self.assertNotIn("secret-name", str(caught.exception))+class ContextExceededMessageTest(unittest.TestCase):+ """#1376: the engine writes `CONTEXT_EXCEEDED prompt_tokens=N requested=M+ capacity=C`. The message took fields[2] ("requested=M", the completion+ budget) as the limit and printed the raw key=value token, so a user with a+ 9000-token prompt on an 8192 ceiling read "maximum context length is+ requested=4 tokens". The number that mattered, capacity, appeared nowhere."""++ def test_limit_is_capacity_and_used_is_prompt_tokens(self):+ from openai_server import _engine_error+ err = _engine_error(["CONTEXT_EXCEEDED", "prompt_tokens=9000", "requested=4",+ "capacity=8192"], "ignored")+ text = str(err)+ self.assertIn("8192", text)+ self.assertIn("9000", text)+ self.assertNotIn("requested=", text)+ self.assertNotIn("prompt_tokens=", text)+ self.assertNotIn("capacity=", text)++ def test_the_positional_spelling_of_colibri_and_deepseek_still_reads(self):+ from openai_server import _engine_error+ text = str(_engine_error(["CONTEXT_EXCEEDED", "8321", "4094"], "ignored"))+ self.assertIn("4094", text)+ self.assertIn("8321", text)++ def test_missing_fields_do_not_crash_the_message(self):+ from openai_server import _engine_error+ text = str(_engine_error(["CONTEXT_EXCEEDED"], "ignored"))+ self.assertIn("the context", text)++if __name__ == "__main__":unittest.main()