Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(serve): check tools and messages before rendering the chat prompt

kevin9327ea8238ff0645
diff --git a/c/openai_server.py b/c/openai_server.py
index fdab0091..f9e25c89 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -3567,26 +3567,11 @@ class ToolSideband:
def reply(self):
return "".join(self.parts) if self.seen else None
-def generation_options(body, limit):
- if body.get("n", 1) != 1:
- raise APIError(400, "Colibri currently supports `n=1` only.", "n", "unsupported_value")
- best_of = body.get("best_of", 1)
- if best_of not in (None, 1):
- raise APIError(400, "Colibri currently supports `best_of` equal to 1 only.",
- "best_of", "unsupported_value")
- logit_bias = body.get("logit_bias")
- if logit_bias not in (None, {}):
- raise APIError(400, "Colibri does not support a non-empty `logit_bias` yet.",
- "logit_bias", "unsupported_value")
- if body.get("suffix") is not None:
- raise APIError(400, "Colibri does not support `suffix` infill yet.",
- "suffix", "unsupported_parameter")
- modalities = body.get("modalities")
- if isinstance(modalities, list) and "audio" in modalities:
- raise APIError(400, "Colibri does not support audio output via `modalities`.",
- "modalities", "unsupported_value")
- # `tools`/`functions` are handled by render_chat (declaration) + parse_tool_calls (output).
- # Validate tools/functions structure early so malformed input fails with a clear error.
+def validate_tools(body):
+ """Refuse a malformed `tools`/`functions` with a 400 naming the field. The chat renderers
+ iterate the list and parse_tool_calls reads each schema's `properties` and `required`, so
+ this has to run before either: chat_completion() calls it ahead of rendering, and
+ generation_options() calls it again for every other path."""
tools_raw = body.get("tools") or body.get("functions")
if tools_raw is not None:
if not isinstance(tools_raw, list):
@@ -3607,6 +3592,40 @@ def generation_options(body, limit):
if not isinstance(fn["name"], str):
raise APIError(400, f"Tool `name` must be a string at index {idx}.",
f"tools.{idx}.function.name", "invalid_value")
+ params = fn.get("parameters")
+ if params is None:
+ continue
+ if not isinstance(params, dict):
+ raise APIError(400, f"Tool `parameters` must be an object at index {idx}.",
+ f"tools.{idx}.function.parameters", "invalid_value")
+ if params.get("properties") is not None and not isinstance(params["properties"], dict):
+ raise APIError(400, f"Tool `parameters.properties` must be an object at index {idx}.",
+ f"tools.{idx}.function.parameters.properties", "invalid_value")
+ if params.get("required") is not None and not isinstance(params["required"], list):
+ raise APIError(400, f"Tool `parameters.required` must be an array at index {idx}.",
+ f"tools.{idx}.function.parameters.required", "invalid_value")
+
+
+def generation_options(body, limit):
+ if body.get("n", 1) != 1:
+ raise APIError(400, "Colibri currently supports `n=1` only.", "n", "unsupported_value")
+ best_of = body.get("best_of", 1)
+ if best_of not in (None, 1):
+ raise APIError(400, "Colibri currently supports `best_of` equal to 1 only.",
+ "best_of", "unsupported_value")
+ logit_bias = body.get("logit_bias")
+ if logit_bias not in (None, {}):
+ raise APIError(400, "Colibri does not support a non-empty `logit_bias` yet.",
+ "logit_bias", "unsupported_value")
+ if body.get("suffix") is not None:
+ raise APIError(400, "Colibri does not support `suffix` infill yet.",
+ "suffix", "unsupported_parameter")
+ modalities = body.get("modalities")
+ if isinstance(modalities, list) and "audio" in modalities:
+ raise APIError(400, "Colibri does not support audio output via `modalities`.",
+ "modalities", "unsupported_value")
+ # `tools`/`functions` are handled by render_chat (declaration) + parse_tool_calls (output).
+ validate_tools(body)
choice = body.get("tool_choice")
if choice is not None:
if isinstance(choice, str):
@@ -6689,10 +6708,17 @@ class APIHandler(BaseHTTPRequestHandler):
preserve_thinking = body.get("preserve_thinking", not enable_thinking)
if not isinstance(preserve_thinking, bool):
raise APIError(400, "`preserve_thinking` must be a boolean.", "preserve_thinking")
+ # The request's shape is checked before the image expanders and the renderer read it.
+ # generation() validates `tools` too, but only after rendering, and a renderer handed a
+ # `tools` of 5 or a `messages` of null raises TypeError, which do_POST answers with 500.
+ validate_tools(body)
tools = body.get("tools") or body.get("functions") or None
tool_choice = body.get("tool_choice")
audio_clips = [] if ARCH == "inkling" else None
- messages, image = self.expand_images(body.get("messages"))
+ messages = body.get("messages")
+ if not isinstance(messages, list) or not messages:
+ raise APIError(400, "`messages` must be a non-empty array.", "messages")
+ messages, image = self.expand_images(messages)
add_generation_prompt = resolve_generation_prompt(messages, body)
prompt = render_chat_for_arch(messages, enable_thinking, reasoning_effort,
tools, tool_choice, audio_out=audio_clips,
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index 72eaf450..8ef6c051 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -2747,6 +2747,43 @@ class HTTPTest(unittest.TestCase):
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 401)
+ def test_a_malformed_tools_or_messages_is_a_client_error_on_every_family(self):
+ """`tools: 5` or `messages: null` answered HTTP 500 on most families.
+
+ generation_options() validates `tools`, but chat_completion() renders the prompt (and
+ runs the image expanders) before it calls generation(), and those iterate `tools` and
+ `messages` without checking them, so the TypeError reached do_POST's catch-all. A
+ `parameters` that is not an object got past every check and failed in
+ parse_tool_calls() after the whole generation had run.
+ """
+ import family_registry
+
+ cases = [
+ ({"tools": 5}, "tools"),
+ ({"tools": True}, "tools"),
+ ({"tools": [{"type": "function", "function": {"name": "f", "parameters": "x"}}]},
+ "tools.0.function.parameters"),
+ ({"tools": [{"type": "function", "function": {
+ "name": "f", "parameters": {"type": "object", "properties": "x"}}}]},
+ "tools.0.function.parameters.properties"),
+ ({"tools": [{"type": "function", "function": {
+ "name": "f", "parameters": {"type": "object", "required": "a"}}}]},
+ "tools.0.function.parameters.required"),
+ ({"messages": None}, "messages"),
+ ({"messages": 5}, "messages"),
+ ]
+ for arch in family_registry.family_ids():
+ for extra, param in cases:
+ with self.subTest(arch=arch, param=param, value=next(iter(extra.values()))):
+ body = {"model": "test-model",
+ "messages": [{"role": "user", "content": "hi"}], **extra}
+ with patch("openai_server.ARCH", arch):
+ with self.assertRaises(HTTPError) as caught:
+ self.request("/v1/chat/completions", body)
+ self.addCleanup(caught.exception.close)
+ self.assertEqual(caught.exception.code, 400)
+ self.assertEqual(json.loads(caught.exception.read())["error"]["param"], param)
+
def test_metrics_counts_http_engine_failure_without_success(self):
before = self.server.scheduler.snapshot()
with patch.object(self.engine, "generate", side_effect=RuntimeError("injected failure")):