Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
anthropic: image blocks are served, and a tool_result's picture is not dropped
Léon Avic Simmons6fcb03427baa
diff --git a/c/openai_server.py b/c/openai_server.pyindex 9d595ac9..b9affdd2 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -2785,19 +2785,57 @@ def split_thinking_reply(text, enable_thinking=True, add_generation_prompt=True)return "".join(thinking), "".join(answer)-def _anthropic_block_text(blocks, param):- """Text out of an Anthropic content array (tool_result content is the same shape)."""+def _anthropic_image_part(block, where):+ """An Anthropic `image` block as the OpenAI-shaped part expand_*_images() reads.++ A base64 source becomes the same data: URI a client of /v1/chat/completions would+ send, so both endpoints hand the expander one picture in one shape. A `url` source+ is passed through as-is: the expander already refuses to fetch remote URLs, and+ the Anthropic client must meet that refusal, not a second one."""+ source = block.get("source")+ if not isinstance(source, dict):+ raise APIError(400, "`image` blocks require a `source` object.", f"{where}.source")+ kind = source.get("type")+ if kind == "base64":+ media_type, data = source.get("media_type"), source.get("data")+ if not isinstance(media_type, str) or not media_type:+ raise APIError(400, "`image.source.media_type` is required for base64 sources.",+ f"{where}.source.media_type")+ if not isinstance(data, str) or not data:+ raise APIError(400, "`image.source.data` is required for base64 sources.",+ f"{where}.source.data")+ return {"type": "image_url", "image_url": {"url": f"data:{media_type};base64,{data}"}}+ if kind == "url":+ url = source.get("url")+ if not isinstance(url, str) or not url:+ raise APIError(400, "`image.source.url` is required for url sources.",+ f"{where}.source.url")+ return {"type": "image_url", "image_url": {"url": url}}+ raise APIError(400, "`image.source.type` must be base64 or url.", f"{where}.source.type",+ "unsupported_content_type")+++def _anthropic_block_text(blocks, param, images=None):+ """Text out of an Anthropic content array (tool_result content is the same shape).++ With `images`, a list, `image` blocks are allowed and their OpenAI-shaped parts are+ appended to it instead of raising: a tool_result can carry a picture (Claude Code's+ Read on an image file does), and the caller decides which turn it rides on."""if isinstance(blocks, str):return blocksif not isinstance(blocks, list):raise APIError(400, "Content must be a string or an array of blocks.", param)parts = []for index, block in enumerate(blocks):+ where = f"{param}.{index}"+ if isinstance(block, dict) and block.get("type") == "image" and images is not None:+ images.append(_anthropic_image_part(block, where))+ continueif not isinstance(block, dict) or block.get("type") != "text":raise APIError(400, "Colibri currently supports text blocks only here.",- f"{param}.{index}", "unsupported_content_type")+ where, "unsupported_content_type")if not isinstance(block.get("text"), str):- raise APIError(400, "Text blocks require a string `text` field.", f"{param}.{index}.text")+ raise APIError(400, "Text blocks require a string `text` field.", f"{where}.text")parts.append(block["text"])return "".join(parts)@@ -2834,7 +2872,10 @@ def anthropic_to_openai(body):if not isinstance(content, list):raise APIError(400, "Message content must be a string or an array of blocks.",f"messages.{index}.content")- texts, reasoning, calls, results = [], [], [], []+ # `parts` keeps the user's text and image blocks in the order the client put+ # them; `carried` collects pictures found inside tool_result blocks, which+ # ride the user turn that follows the tool turns (the tool role is text).+ texts, parts, carried, reasoning, calls, results = [], [], [], [], [], []for j, block in enumerate(content):where = f"messages.{index}.content.{j}"if not isinstance(block, dict):@@ -2844,6 +2885,12 @@ def anthropic_to_openai(body):if not isinstance(block.get("text"), str):raise APIError(400, "Text blocks require a string `text` field.", f"{where}.text")texts.append(block["text"])+ parts.append({"type": "text", "text": block["text"]})+ elif kind == "image":+ if role != "user":+ raise APIError(400, "`image` blocks are valid only in user messages.",+ where, "unsupported_content_type")+ parts.append(_anthropic_image_part(block, where))elif kind == "thinking":if role != "assistant":raise APIError(400, "`thinking` blocks are valid only in assistant messages.",@@ -2869,13 +2916,19 @@ def anthropic_to_openai(body):"function": {"name": name,"arguments": json.dumps(arguments, ensure_ascii=False)}})elif kind == "tool_result":+ pictures = []results.append({"role": "tool","tool_call_id": block.get("tool_use_id") or "","content": _anthropic_block_text(block.get("content", ""),- f"{where}.content")})+ f"{where}.content", pictures)})+ if pictures and role != "user":+ raise APIError(400, "`image` blocks are valid only in user messages.",+ where, "unsupported_content_type")+ carried.extend(pictures)else:- raise APIError(400, "Colibri supports `text`, `tool_use` and `tool_result` "- "content blocks only.", f"{where}.type", "unsupported_content_type")+ raise APIError(400, "Colibri supports `text`, `image`, `tool_use` and "+ "`tool_result` content blocks only.", f"{where}.type",+ "unsupported_content_type")# tool results precede the user's own text: they answer the previous assistant turnmessages.extend(results)text = "".join(texts)@@ -2887,6 +2940,11 @@ def anthropic_to_openai(body):if calls:entry["tool_calls"] = callsmessages.append(entry)+ elif carried or any(part["type"] != "text" for part in parts):+ # A picture is in this turn: the content stays a parts list, which is+ # what expand_*_images() reads. Pictures out of tool results come first,+ # right after the tool turns that produced them.+ messages.append({"role": "user", "content": carried + parts})elif text or not results:messages.append({"role": "user", "content": text})return messages@@ -5509,40 +5567,7 @@ class APIHandler(BaseHTTPRequestHandler):tools = body.get("tools") or body.get("functions") or Nonetool_choice = body.get("tool_choice")audio_clips = [] if ARCH == "inkling" else None- messages = body.get("messages")- # Le immagini diventano segnaposto PRIMA del rendering: il renderer- # tratta poi turni di solo testo, e il conto dei segnaposto e quello- # degli embedding non possono divergere.- image = None- if ARCH == "glm53":- messages, images = expand_glm53_images(- messages, getattr(self.server.engine, "model_dir", None))- if len(images) > 1:- raise APIError(400, "one image per request for now; the engine "- "holds a single pending image.", "messages")- image = images[0] if images else None- elif ARCH == "deepseek_v41":- ceiling = os.environ.get("V41_MAX_IMAGE_TOKENS")- messages, images = expand_dsv41_images(- messages, getattr(self.server.engine, "model_dir", None),- int(ceiling) if ceiling else None)- if len(images) > 1:- raise APIError(400, "one image per request for now; the engine "- "holds a single pending image.", "messages")- image = images[0] if images else None- elif ARCH == "qwen38" or (ARCH == "qwen36" and qwen36_has_vision(- getattr(self.server.engine, "model_dir", None))):- # Qwen3.5/3.6/3.8 share the tower and the preprocessor, so qwen36- # checkpoints converted with their tower take the same path (#1757).- ceiling = os.environ.get("Q38_MAX_IMAGE_TOKENS" if ARCH == "qwen38"- else "Q36_MAX_IMAGE_TOKENS")- messages, images = expand_qwen38_images(- messages, getattr(self.server.engine, "model_dir", None),- int(ceiling) if ceiling else None)- if len(images) > 1:- raise APIError(400, "one image per request for now; the engine "- "holds a single pending image.", "messages")- image = images[0] if images else None+ messages, image = self.expand_images(body.get("messages"))add_generation_prompt = resolve_generation_prompt(messages, body)prompt = render_chat_for_arch(messages, enable_thinking, reasoning_effort,tools, tool_choice, audio_out=audio_clips,@@ -5554,6 +5579,34 @@ class APIHandler(BaseHTTPRequestHandler):audio=b"".join(audio_clips) if audio_clips else None,image=image)+ def expand_images(self, messages):+ """(messages with placeholders, the one image or None) for this engine's family.++ Le immagini diventano segnaposto PRIMA del rendering: il renderer tratta+ poi turni di solo testo, e il conto dei segnaposto e quello degli+ embedding non possono divergere. Both chat endpoints come through here,+ so a picture is one thing whichever API delivered it."""+ model_dir = getattr(self.server.engine, "model_dir", None)+ if ARCH == "glm53":+ messages, images = expand_glm53_images(messages, model_dir)+ elif ARCH == "deepseek_v41":+ ceiling = os.environ.get("V41_MAX_IMAGE_TOKENS")+ messages, images = expand_dsv41_images(messages, model_dir,+ int(ceiling) if ceiling else None)+ elif ARCH == "qwen38" or (ARCH == "qwen36" and qwen36_has_vision(model_dir)):+ # Qwen3.5/3.6/3.8 share the tower and the preprocessor, so qwen36+ # checkpoints converted with their tower take the same path (#1757).+ ceiling = os.environ.get("Q38_MAX_IMAGE_TOKENS" if ARCH == "qwen38"+ else "Q36_MAX_IMAGE_TOKENS")+ messages, images = expand_qwen38_images(messages, model_dir,+ int(ceiling) if ceiling else None)+ else:+ return messages, None+ if len(images) > 1:+ raise APIError(400, "one image per request for now; the engine "+ "holds a single pending image.", "messages")+ return messages, images[0] if images else None+# ---- Anthropic /v1/messages (#343) ----------------------------------------------------ANTHROPIC_STOP = {"stop": "end_turn", "length": "max_tokens", "tool_calls": "tool_use"}@@ -5563,7 +5616,7 @@ class APIHandler(BaseHTTPRequestHandler):if body.get(unsupported) not in (None, [], ""):raise APIError(400, f"Colibri does not support `{unsupported}` ({why}) yet.",unsupported, "unsupported_value")- messages = anthropic_to_openai(body)+ messages, image = self.expand_images(anthropic_to_openai(body))tools, tool_choice = anthropic_tools(body)thinking = body.get("thinking")if thinking is not None and not isinstance(thinking, dict):@@ -5596,10 +5649,10 @@ class APIHandler(BaseHTTPRequestHandler):add_generation_prompt=add_generation_prompt,preserve_thinking=not enable_thinking)self.anthropic_generation(translated, prompt, request_id, tools, enable_thinking,- add_generation_prompt)+ add_generation_prompt, image)def anthropic_generation(self, body, prompt, request_id, tools, enable_thinking,- add_generation_prompt=True):+ add_generation_prompt=True, image=None):maximum, temperature, top_p, grammar, _stop_sequences = generation_options(body, self.server.max_tokens)# Same policy as /v1/chat/completions: `body` is the translated OpenAI-shaped@@ -5668,7 +5721,8 @@ class APIHandler(BaseHTTPRequestHandler):stats = self.server.generate(prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,self.client_disconnected, grammar=grammar, stopped=generation_stopped,- **({"on_tool": sideband.feed} if sideband.enabled else {}))+ **({"on_tool": sideband.feed} if sideband.enabled else {}),+ **({"image": image} if image is not None else {}))stop_filter.finish()sideband.finish()content, stop_reason = blocks_and_stop("".join(output), stats,@@ -5812,7 +5866,8 @@ class APIHandler(BaseHTTPRequestHandler):stats = self.server.generate(prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,lambda: not connected[0], grammar=grammar, stopped=generation_stopped,- **({"on_tool": sideband.feed} if sideband.enabled else {}))+ **({"on_tool": sideband.feed} if sideband.enabled else {}),+ **({"image": image} if image is not None else {}))stop_filter.finish()sideband.finish()if split:diff --git a/c/tests/test_anthropic_messages.py b/c/tests/test_anthropic_messages.pyindex 7d9bd6a5..cc408b67 100644--- a/c/tests/test_anthropic_messages.py+++ b/c/tests/test_anthropic_messages.py@@ -30,11 +30,13 @@ class FakeEngine:self.script = scriptself.length_limited = length_limitedself.prompts = []+ self.images = []def generate(self, prompt, maximum, temperature, top_p, on_text, cache_slot=0,cancelled=None, grammar=None, stopped=None, on_accept=None,- on_tool=None):+ on_tool=None, image=None):self.prompts.append(prompt)+ self.images.append(image)self.emitted = 0for chunk in self.script:on_text(chunk)@@ -117,6 +119,99 @@ class TranslationTest(unittest.TestCase):self.assertIn("not supported", error.message)self.assertRegex(error.message, re.compile(r"role .{0,2}system", re.IGNORECASE))+ # ---- image blocks -------------------------------------------------------------------+ # An Anthropic client sends a picture as {"type": "image", "source": {"type": "base64",+ # "media_type": ..., "data": ...}}; the OpenAI endpoint already takes the same picture+ # as an image_url data: URI, and expand_*_images() turns that into the engine's+ # placeholders. The translation must produce exactly that part, in place, so the two+ # endpoints cannot disagree about where a picture sits in the conversation.+ PNG = {"type": "image", "source": {"type": "base64", "media_type": "image/png",+ "data": "iVBORw0KGgo="}}+ PNG_PART = {"type": "image_url",+ "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}}++ def test_image_block_becomes_an_image_url_part_in_place(self):+ messages = anthropic_to_openai({"messages": [+ {"role": "user", "content": [{"type": "text", "text": "What is this?"},+ self.PNG,+ {"type": "text", "text": "Be brief."}]}]})+ self.assertEqual(messages, [{"role": "user", "content": [+ {"type": "text", "text": "What is this?"}, self.PNG_PART,+ {"type": "text", "text": "Be brief."}]}])++ def test_text_only_content_still_collapses_to_a_string(self):+ # Every existing renderer and test relies on this shape; a picture-free+ # message must not start arriving as a parts list.+ messages = anthropic_to_openai({"messages": [+ {"role": "user", "content": [{"type": "text", "text": "a"},+ {"type": "text", "text": "b"}]}]})+ self.assertEqual(messages, [{"role": "user", "content": "ab"}])++ def test_tool_result_image_rides_the_following_user_turn(self):+ # Claude Code's Read tool on a .png returns the picture INSIDE the tool_result.+ # The tool role carries text only, so the picture is carried onto the user+ # turn that follows the results -- the model still sees it right after the+ # tool answered, which is where the client put it.+ messages = anthropic_to_openai({"messages": [+ {"role": "user", "content": "look at it"},+ {"role": "assistant", "content": [+ {"type": "tool_use", "id": "t1", "name": "Read", "input": {"p": "x.png"}}]},+ {"role": "user", "content": [+ {"type": "tool_result", "tool_use_id": "t1",+ "content": [{"type": "text", "text": "(image)"}, self.PNG]},+ {"type": "text", "text": "and?"}]},+ ]})+ self.assertEqual([m["role"] for m in messages], ["user", "assistant", "tool", "user"])+ self.assertEqual(messages[2]["content"], "(image)")+ self.assertEqual(messages[3]["content"],+ [self.PNG_PART, {"type": "text", "text": "and?"}])++ def test_tool_result_image_alone_still_makes_a_user_turn(self):+ # ...even when the tool_result is the whole message (no user text): dropping+ # the picture here is exactly the silent loss this exists to prevent.+ messages = anthropic_to_openai({"messages": [+ {"role": "user", "content": "go"},+ {"role": "assistant", "content": [+ {"type": "tool_use", "id": "t1", "name": "Read", "input": {}}]},+ {"role": "user", "content": [+ {"type": "tool_result", "tool_use_id": "t1", "content": [self.PNG]}]},+ ]})+ self.assertEqual([m["role"] for m in messages], ["user", "assistant", "tool", "user"])+ self.assertEqual(messages[2]["content"], "")+ self.assertEqual(messages[3]["content"], [self.PNG_PART])++ def test_image_in_an_assistant_message_is_refused(self):+ for content in ([self.PNG],+ [{"type": "tool_result", "tool_use_id": "t1", "content": [self.PNG]}]):+ with self.assertRaises(APIError) as caught:+ anthropic_to_openai({"messages": [+ {"role": "user", "content": "hi"},+ {"role": "assistant", "content": content}]})+ self.assertEqual(caught.exception.status, 400)+ self.assertIn("user messages", caught.exception.message)+ self.assertEqual(caught.exception.param, "messages.1.content.0")++ def test_image_source_is_validated(self):+ for source in ({"type": "base64", "media_type": "image/png"}, # no data+ {"type": "base64", "data": "abc"}, # no media_type+ {"type": "file", "file_id": "f1"}, # not supported+ "not-an-object"):+ with self.assertRaises(APIError) as caught:+ anthropic_to_openai({"messages": [{"role": "user", "content": [+ {"type": "image", "source": source}]}]})+ self.assertEqual(caught.exception.status, 400)+ self.assertTrue(caught.exception.param.startswith("messages.0.content.0.source"),+ caught.exception.param)++ def test_url_source_is_handed_to_the_same_refusal_as_the_openai_path(self):+ # The OpenAI endpoint refuses to fetch a remote URL by policy; the same URL+ # arriving as an Anthropic `url` source must reach that same refusal, not a+ # different one and not a fetch.+ messages = anthropic_to_openai({"messages": [{"role": "user", "content": [+ {"type": "image", "source": {"type": "url", "url": "https://x/y.png"}}]}]})+ self.assertEqual(messages[0]["content"],+ [{"type": "image_url", "image_url": {"url": "https://x/y.png"}}])+class MessagesHTTPTest(unittest.TestCase):def setUp(self):@@ -444,6 +539,63 @@ class MessagesHTTPTest(unittest.TestCase):self.assertEqual(caught.exception.code, 400)self.assertIn(field, json.load(caught.exception)["error"]["message"])+ # ---- images end to end -----------------------------------------------------------+ # The translation alone is not the contract: the picture has to reach the engine.+ # The OpenAI path expands image parts into the family's placeholders and hands the+ # patches to Engine.generate(image=...) beside the prompt; /v1/messages goes+ # through the same expansion, so the placeholder count and the patches cannot+ # disagree between the two endpoints.+ IMAGE = {"type": "image", "source": {"type": "base64", "media_type": "image/png",+ "data": "iVBORw0KGgo="}}++ def fake_preprocess(self, data, model_dir):+ self.preprocessed.append(data)+ return b"\x00" * 16, 4, 4 # 2x2 = 4 placeholders after merging++ def test_image_block_reaches_the_glm53_engine(self):+ self.preprocessed = []+ messages = [{"role": "user", "content": [{"type": "text", "text": "what is this?"},+ self.IMAGE]}]+ with patch("openai_server.ARCH", "glm53"), \+ patch("openai_server._preprocess_image", self.fake_preprocess):+ with self.post(self.base_body(messages=messages)) as response:+ self.assertEqual(response.status, 200)+ self.assertEqual(self.preprocessed, [b"\x89PNG\r\n\x1a\n"])+ self.assertEqual(self.engine.images, [(b"\x00" * 16, 4, 4)])+ self.assertIn("<|user|>what is this?<|begin_of_image|>" + "<|image|>" * 4+ + "<|end_of_image|><|assistant|>", self.engine.prompts[-1])++ def test_tool_result_image_reaches_the_glm53_engine_after_the_observation(self):+ self.preprocessed = []+ messages = [+ {"role": "user", "content": "read it"},+ {"role": "assistant", "content": [+ {"type": "tool_use", "id": "t1", "name": "Read", "input": {"p": "x.png"}}]},+ {"role": "user", "content": [+ {"type": "tool_result", "tool_use_id": "t1", "content": [self.IMAGE]}]},+ ]+ with patch("openai_server.ARCH", "glm53"), \+ patch("openai_server._preprocess_image", self.fake_preprocess):+ with self.post(self.base_body(messages=messages)) as response:+ self.assertEqual(response.status, 200)+ self.assertEqual(self.engine.images, [(b"\x00" * 16, 4, 4)])+ prompt = self.engine.prompts[-1]+ self.assertLess(prompt.index("<|observation|>"), prompt.index("<|begin_of_image|>"))+ self.assertIn("<|user|><|begin_of_image|>" + "<|image|>" * 4 + "<|end_of_image|>",+ prompt)++ def test_two_images_are_refused_like_the_openai_path(self):+ self.preprocessed = []+ messages = [{"role": "user", "content": [self.IMAGE, self.IMAGE]}]+ with patch("openai_server.ARCH", "glm53"), \+ patch("openai_server._preprocess_image", self.fake_preprocess):+ with self.assertRaises(HTTPError) as caught:+ self.post(self.base_body(messages=messages))+ self.addCleanup(caught.exception.close)+ self.assertEqual(caught.exception.code, 400)+ self.assertIn("one image per request", json.load(caught.exception)["error"]["message"])+ self.assertEqual(self.engine.images, [])+if __name__ == "__main__":unittest.main()