Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(serve): check tools and messages before rendering the chat prompt
kevin9327ea8238ff0645
diff --git a/c/openai_server.py b/c/openai_server.pyindex fdab0091..f9e25c89 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -3567,26 +3567,11 @@ class ToolSideband:def reply(self):return "".join(self.parts) if self.seen else None-def generation_options(body, limit):- if body.get("n", 1) != 1:- raise APIError(400, "Colibri currently supports `n=1` only.", "n", "unsupported_value")- best_of = body.get("best_of", 1)- if best_of not in (None, 1):- raise APIError(400, "Colibri currently supports `best_of` equal to 1 only.",- "best_of", "unsupported_value")- logit_bias = body.get("logit_bias")- if logit_bias not in (None, {}):- raise APIError(400, "Colibri does not support a non-empty `logit_bias` yet.",- "logit_bias", "unsupported_value")- if body.get("suffix") is not None:- raise APIError(400, "Colibri does not support `suffix` infill yet.",- "suffix", "unsupported_parameter")- modalities = body.get("modalities")- if isinstance(modalities, list) and "audio" in modalities:- raise APIError(400, "Colibri does not support audio output via `modalities`.",- "modalities", "unsupported_value")- # `tools`/`functions` are handled by render_chat (declaration) + parse_tool_calls (output).- # Validate tools/functions structure early so malformed input fails with a clear error.+def validate_tools(body):+ """Refuse a malformed `tools`/`functions` with a 400 naming the field. The chat renderers+ iterate the list and parse_tool_calls reads each schema's `properties` and `required`, so+ this has to run before either: chat_completion() calls it ahead of rendering, and+ generation_options() calls it again for every other path."""tools_raw = body.get("tools") or body.get("functions")if tools_raw is not None:if not isinstance(tools_raw, list):@@ -3607,6 +3592,40 @@ def generation_options(body, limit):if not isinstance(fn["name"], str):raise APIError(400, f"Tool `name` must be a string at index {idx}.",f"tools.{idx}.function.name", "invalid_value")+ params = fn.get("parameters")+ if params is None:+ continue+ if not isinstance(params, dict):+ raise APIError(400, f"Tool `parameters` must be an object at index {idx}.",+ f"tools.{idx}.function.parameters", "invalid_value")+ if params.get("properties") is not None and not isinstance(params["properties"], dict):+ raise APIError(400, f"Tool `parameters.properties` must be an object at index {idx}.",+ f"tools.{idx}.function.parameters.properties", "invalid_value")+ if params.get("required") is not None and not isinstance(params["required"], list):+ raise APIError(400, f"Tool `parameters.required` must be an array at index {idx}.",+ f"tools.{idx}.function.parameters.required", "invalid_value")+++def generation_options(body, limit):+ if body.get("n", 1) != 1:+ raise APIError(400, "Colibri currently supports `n=1` only.", "n", "unsupported_value")+ best_of = body.get("best_of", 1)+ if best_of not in (None, 1):+ raise APIError(400, "Colibri currently supports `best_of` equal to 1 only.",+ "best_of", "unsupported_value")+ logit_bias = body.get("logit_bias")+ if logit_bias not in (None, {}):+ raise APIError(400, "Colibri does not support a non-empty `logit_bias` yet.",+ "logit_bias", "unsupported_value")+ if body.get("suffix") is not None:+ raise APIError(400, "Colibri does not support `suffix` infill yet.",+ "suffix", "unsupported_parameter")+ modalities = body.get("modalities")+ if isinstance(modalities, list) and "audio" in modalities:+ raise APIError(400, "Colibri does not support audio output via `modalities`.",+ "modalities", "unsupported_value")+ # `tools`/`functions` are handled by render_chat (declaration) + parse_tool_calls (output).+ validate_tools(body)choice = body.get("tool_choice")if choice is not None:if isinstance(choice, str):@@ -6689,10 +6708,17 @@ class APIHandler(BaseHTTPRequestHandler):preserve_thinking = body.get("preserve_thinking", not enable_thinking)if not isinstance(preserve_thinking, bool):raise APIError(400, "`preserve_thinking` must be a boolean.", "preserve_thinking")+ # The request's shape is checked before the image expanders and the renderer read it.+ # generation() validates `tools` too, but only after rendering, and a renderer handed a+ # `tools` of 5 or a `messages` of null raises TypeError, which do_POST answers with 500.+ validate_tools(body)tools = body.get("tools") or body.get("functions") or Nonetool_choice = body.get("tool_choice")audio_clips = [] if ARCH == "inkling" else None- messages, image = self.expand_images(body.get("messages"))+ messages = body.get("messages")+ if not isinstance(messages, list) or not messages:+ raise APIError(400, "`messages` must be a non-empty array.", "messages")+ messages, image = self.expand_images(messages)add_generation_prompt = resolve_generation_prompt(messages, body)prompt = render_chat_for_arch(messages, enable_thinking, reasoning_effort,tools, tool_choice, audio_out=audio_clips,diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex 72eaf450..8ef6c051 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -2747,6 +2747,43 @@ class HTTPTest(unittest.TestCase):self.addCleanup(caught.exception.close)self.assertEqual(caught.exception.code, 401)+ def test_a_malformed_tools_or_messages_is_a_client_error_on_every_family(self):+ """`tools: 5` or `messages: null` answered HTTP 500 on most families.++ generation_options() validates `tools`, but chat_completion() renders the prompt (and+ runs the image expanders) before it calls generation(), and those iterate `tools` and+ `messages` without checking them, so the TypeError reached do_POST's catch-all. A+ `parameters` that is not an object got past every check and failed in+ parse_tool_calls() after the whole generation had run.+ """+ import family_registry++ cases = [+ ({"tools": 5}, "tools"),+ ({"tools": True}, "tools"),+ ({"tools": [{"type": "function", "function": {"name": "f", "parameters": "x"}}]},+ "tools.0.function.parameters"),+ ({"tools": [{"type": "function", "function": {+ "name": "f", "parameters": {"type": "object", "properties": "x"}}}]},+ "tools.0.function.parameters.properties"),+ ({"tools": [{"type": "function", "function": {+ "name": "f", "parameters": {"type": "object", "required": "a"}}}]},+ "tools.0.function.parameters.required"),+ ({"messages": None}, "messages"),+ ({"messages": 5}, "messages"),+ ]+ for arch in family_registry.family_ids():+ for extra, param in cases:+ with self.subTest(arch=arch, param=param, value=next(iter(extra.values()))):+ body = {"model": "test-model",+ "messages": [{"role": "user", "content": "hi"}], **extra}+ with patch("openai_server.ARCH", arch):+ with self.assertRaises(HTTPError) as caught:+ self.request("/v1/chat/completions", body)+ self.addCleanup(caught.exception.close)+ self.assertEqual(caught.exception.code, 400)+ self.assertEqual(json.loads(caught.exception.read())["error"]["param"], param)+def test_metrics_counts_http_engine_failure_without_success(self):before = self.server.scheduler.snapshot()with patch.object(self.engine, "generate", side_effect=RuntimeError("injected failure")):