Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

anthropic: image blocks are served, and a tool_result's picture is not dropped

Léon Avic Simmons6fcb03427baa
diff --git a/c/openai_server.py b/c/openai_server.py
index 9d595ac9..b9affdd2 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -2785,19 +2785,57 @@ def split_thinking_reply(text, enable_thinking=True, add_generation_prompt=True)
return "".join(thinking), "".join(answer)
-def _anthropic_block_text(blocks, param):
- """Text out of an Anthropic content array (tool_result content is the same shape)."""
+def _anthropic_image_part(block, where):
+ """An Anthropic `image` block as the OpenAI-shaped part expand_*_images() reads.
+
+ A base64 source becomes the same data: URI a client of /v1/chat/completions would
+ send, so both endpoints hand the expander one picture in one shape. A `url` source
+ is passed through as-is: the expander already refuses to fetch remote URLs, and
+ the Anthropic client must meet that refusal, not a second one."""
+ source = block.get("source")
+ if not isinstance(source, dict):
+ raise APIError(400, "`image` blocks require a `source` object.", f"{where}.source")
+ kind = source.get("type")
+ if kind == "base64":
+ media_type, data = source.get("media_type"), source.get("data")
+ if not isinstance(media_type, str) or not media_type:
+ raise APIError(400, "`image.source.media_type` is required for base64 sources.",
+ f"{where}.source.media_type")
+ if not isinstance(data, str) or not data:
+ raise APIError(400, "`image.source.data` is required for base64 sources.",
+ f"{where}.source.data")
+ return {"type": "image_url", "image_url": {"url": f"data:{media_type};base64,{data}"}}
+ if kind == "url":
+ url = source.get("url")
+ if not isinstance(url, str) or not url:
+ raise APIError(400, "`image.source.url` is required for url sources.",
+ f"{where}.source.url")
+ return {"type": "image_url", "image_url": {"url": url}}
+ raise APIError(400, "`image.source.type` must be base64 or url.", f"{where}.source.type",
+ "unsupported_content_type")
+
+
+def _anthropic_block_text(blocks, param, images=None):
+ """Text out of an Anthropic content array (tool_result content is the same shape).
+
+ With `images`, a list, `image` blocks are allowed and their OpenAI-shaped parts are
+ appended to it instead of raising: a tool_result can carry a picture (Claude Code's
+ Read on an image file does), and the caller decides which turn it rides on."""
if isinstance(blocks, str):
return blocks
if not isinstance(blocks, list):
raise APIError(400, "Content must be a string or an array of blocks.", param)
parts = []
for index, block in enumerate(blocks):
+ where = f"{param}.{index}"
+ if isinstance(block, dict) and block.get("type") == "image" and images is not None:
+ images.append(_anthropic_image_part(block, where))
+ continue
if not isinstance(block, dict) or block.get("type") != "text":
raise APIError(400, "Colibri currently supports text blocks only here.",
- f"{param}.{index}", "unsupported_content_type")
+ where, "unsupported_content_type")
if not isinstance(block.get("text"), str):
- raise APIError(400, "Text blocks require a string `text` field.", f"{param}.{index}.text")
+ raise APIError(400, "Text blocks require a string `text` field.", f"{where}.text")
parts.append(block["text"])
return "".join(parts)
@@ -2834,7 +2872,10 @@ def anthropic_to_openai(body):
if not isinstance(content, list):
raise APIError(400, "Message content must be a string or an array of blocks.",
f"messages.{index}.content")
- texts, reasoning, calls, results = [], [], [], []
+ # `parts` keeps the user's text and image blocks in the order the client put
+ # them; `carried` collects pictures found inside tool_result blocks, which
+ # ride the user turn that follows the tool turns (the tool role is text).
+ texts, parts, carried, reasoning, calls, results = [], [], [], [], [], []
for j, block in enumerate(content):
where = f"messages.{index}.content.{j}"
if not isinstance(block, dict):
@@ -2844,6 +2885,12 @@ def anthropic_to_openai(body):
if not isinstance(block.get("text"), str):
raise APIError(400, "Text blocks require a string `text` field.", f"{where}.text")
texts.append(block["text"])
+ parts.append({"type": "text", "text": block["text"]})
+ elif kind == "image":
+ if role != "user":
+ raise APIError(400, "`image` blocks are valid only in user messages.",
+ where, "unsupported_content_type")
+ parts.append(_anthropic_image_part(block, where))
elif kind == "thinking":
if role != "assistant":
raise APIError(400, "`thinking` blocks are valid only in assistant messages.",
@@ -2869,13 +2916,19 @@ def anthropic_to_openai(body):
"function": {"name": name,
"arguments": json.dumps(arguments, ensure_ascii=False)}})
elif kind == "tool_result":
+ pictures = []
results.append({"role": "tool",
"tool_call_id": block.get("tool_use_id") or "",
"content": _anthropic_block_text(block.get("content", ""),
- f"{where}.content")})
+ f"{where}.content", pictures)})
+ if pictures and role != "user":
+ raise APIError(400, "`image` blocks are valid only in user messages.",
+ where, "unsupported_content_type")
+ carried.extend(pictures)
else:
- raise APIError(400, "Colibri supports `text`, `tool_use` and `tool_result` "
- "content blocks only.", f"{where}.type", "unsupported_content_type")
+ raise APIError(400, "Colibri supports `text`, `image`, `tool_use` and "
+ "`tool_result` content blocks only.", f"{where}.type",
+ "unsupported_content_type")
# tool results precede the user's own text: they answer the previous assistant turn
messages.extend(results)
text = "".join(texts)
@@ -2887,6 +2940,11 @@ def anthropic_to_openai(body):
if calls:
entry["tool_calls"] = calls
messages.append(entry)
+ elif carried or any(part["type"] != "text" for part in parts):
+ # A picture is in this turn: the content stays a parts list, which is
+ # what expand_*_images() reads. Pictures out of tool results come first,
+ # right after the tool turns that produced them.
+ messages.append({"role": "user", "content": carried + parts})
elif text or not results:
messages.append({"role": "user", "content": text})
return messages
@@ -5509,40 +5567,7 @@ class APIHandler(BaseHTTPRequestHandler):
tools = body.get("tools") or body.get("functions") or None
tool_choice = body.get("tool_choice")
audio_clips = [] if ARCH == "inkling" else None
- messages = body.get("messages")
- # Le immagini diventano segnaposto PRIMA del rendering: il renderer
- # tratta poi turni di solo testo, e il conto dei segnaposto e quello
- # degli embedding non possono divergere.
- image = None
- if ARCH == "glm53":
- messages, images = expand_glm53_images(
- messages, getattr(self.server.engine, "model_dir", None))
- if len(images) > 1:
- raise APIError(400, "one image per request for now; the engine "
- "holds a single pending image.", "messages")
- image = images[0] if images else None
- elif ARCH == "deepseek_v41":
- ceiling = os.environ.get("V41_MAX_IMAGE_TOKENS")
- messages, images = expand_dsv41_images(
- messages, getattr(self.server.engine, "model_dir", None),
- int(ceiling) if ceiling else None)
- if len(images) > 1:
- raise APIError(400, "one image per request for now; the engine "
- "holds a single pending image.", "messages")
- image = images[0] if images else None
- elif ARCH == "qwen38" or (ARCH == "qwen36" and qwen36_has_vision(
- getattr(self.server.engine, "model_dir", None))):
- # Qwen3.5/3.6/3.8 share the tower and the preprocessor, so qwen36
- # checkpoints converted with their tower take the same path (#1757).
- ceiling = os.environ.get("Q38_MAX_IMAGE_TOKENS" if ARCH == "qwen38"
- else "Q36_MAX_IMAGE_TOKENS")
- messages, images = expand_qwen38_images(
- messages, getattr(self.server.engine, "model_dir", None),
- int(ceiling) if ceiling else None)
- if len(images) > 1:
- raise APIError(400, "one image per request for now; the engine "
- "holds a single pending image.", "messages")
- image = images[0] if images else None
+ messages, image = self.expand_images(body.get("messages"))
add_generation_prompt = resolve_generation_prompt(messages, body)
prompt = render_chat_for_arch(messages, enable_thinking, reasoning_effort,
tools, tool_choice, audio_out=audio_clips,
@@ -5554,6 +5579,34 @@ class APIHandler(BaseHTTPRequestHandler):
audio=b"".join(audio_clips) if audio_clips else None,
image=image)
+ def expand_images(self, messages):
+ """(messages with placeholders, the one image or None) for this engine's family.
+
+ Le immagini diventano segnaposto PRIMA del rendering: il renderer tratta
+ poi turni di solo testo, e il conto dei segnaposto e quello degli
+ embedding non possono divergere. Both chat endpoints come through here,
+ so a picture is one thing whichever API delivered it."""
+ model_dir = getattr(self.server.engine, "model_dir", None)
+ if ARCH == "glm53":
+ messages, images = expand_glm53_images(messages, model_dir)
+ elif ARCH == "deepseek_v41":
+ ceiling = os.environ.get("V41_MAX_IMAGE_TOKENS")
+ messages, images = expand_dsv41_images(messages, model_dir,
+ int(ceiling) if ceiling else None)
+ elif ARCH == "qwen38" or (ARCH == "qwen36" and qwen36_has_vision(model_dir)):
+ # Qwen3.5/3.6/3.8 share the tower and the preprocessor, so qwen36
+ # checkpoints converted with their tower take the same path (#1757).
+ ceiling = os.environ.get("Q38_MAX_IMAGE_TOKENS" if ARCH == "qwen38"
+ else "Q36_MAX_IMAGE_TOKENS")
+ messages, images = expand_qwen38_images(messages, model_dir,
+ int(ceiling) if ceiling else None)
+ else:
+ return messages, None
+ if len(images) > 1:
+ raise APIError(400, "one image per request for now; the engine "
+ "holds a single pending image.", "messages")
+ return messages, images[0] if images else None
+
# ---- Anthropic /v1/messages (#343) ----------------------------------------------------
ANTHROPIC_STOP = {"stop": "end_turn", "length": "max_tokens", "tool_calls": "tool_use"}
@@ -5563,7 +5616,7 @@ class APIHandler(BaseHTTPRequestHandler):
if body.get(unsupported) not in (None, [], ""):
raise APIError(400, f"Colibri does not support `{unsupported}` ({why}) yet.",
unsupported, "unsupported_value")
- messages = anthropic_to_openai(body)
+ messages, image = self.expand_images(anthropic_to_openai(body))
tools, tool_choice = anthropic_tools(body)
thinking = body.get("thinking")
if thinking is not None and not isinstance(thinking, dict):
@@ -5596,10 +5649,10 @@ class APIHandler(BaseHTTPRequestHandler):
add_generation_prompt=add_generation_prompt,
preserve_thinking=not enable_thinking)
self.anthropic_generation(translated, prompt, request_id, tools, enable_thinking,
- add_generation_prompt)
+ add_generation_prompt, image)
def anthropic_generation(self, body, prompt, request_id, tools, enable_thinking,
- add_generation_prompt=True):
+ add_generation_prompt=True, image=None):
maximum, temperature, top_p, grammar, _stop_sequences = generation_options(
body, self.server.max_tokens)
# Same policy as /v1/chat/completions: `body` is the translated OpenAI-shaped
@@ -5668,7 +5721,8 @@ class APIHandler(BaseHTTPRequestHandler):
stats = self.server.generate(
prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,
self.client_disconnected, grammar=grammar, stopped=generation_stopped,
- **({"on_tool": sideband.feed} if sideband.enabled else {}))
+ **({"on_tool": sideband.feed} if sideband.enabled else {}),
+ **({"image": image} if image is not None else {}))
stop_filter.finish()
sideband.finish()
content, stop_reason = blocks_and_stop("".join(output), stats,
@@ -5812,7 +5866,8 @@ class APIHandler(BaseHTTPRequestHandler):
stats = self.server.generate(
prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,
lambda: not connected[0], grammar=grammar, stopped=generation_stopped,
- **({"on_tool": sideband.feed} if sideband.enabled else {}))
+ **({"on_tool": sideband.feed} if sideband.enabled else {}),
+ **({"image": image} if image is not None else {}))
stop_filter.finish()
sideband.finish()
if split:
diff --git a/c/tests/test_anthropic_messages.py b/c/tests/test_anthropic_messages.py
index 7d9bd6a5..cc408b67 100644
--- a/c/tests/test_anthropic_messages.py
+++ b/c/tests/test_anthropic_messages.py
@@ -30,11 +30,13 @@ class FakeEngine:
self.script = script
self.length_limited = length_limited
self.prompts = []
+ self.images = []
def generate(self, prompt, maximum, temperature, top_p, on_text, cache_slot=0,
cancelled=None, grammar=None, stopped=None, on_accept=None,
- on_tool=None):
+ on_tool=None, image=None):
self.prompts.append(prompt)
+ self.images.append(image)
self.emitted = 0
for chunk in self.script:
on_text(chunk)
@@ -117,6 +119,99 @@ class TranslationTest(unittest.TestCase):
self.assertIn("not supported", error.message)
self.assertRegex(error.message, re.compile(r"role .{0,2}system", re.IGNORECASE))
+ # ---- image blocks -------------------------------------------------------------------
+ # An Anthropic client sends a picture as {"type": "image", "source": {"type": "base64",
+ # "media_type": ..., "data": ...}}; the OpenAI endpoint already takes the same picture
+ # as an image_url data: URI, and expand_*_images() turns that into the engine's
+ # placeholders. The translation must produce exactly that part, in place, so the two
+ # endpoints cannot disagree about where a picture sits in the conversation.
+ PNG = {"type": "image", "source": {"type": "base64", "media_type": "image/png",
+ "data": "iVBORw0KGgo="}}
+ PNG_PART = {"type": "image_url",
+ "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}}
+
+ def test_image_block_becomes_an_image_url_part_in_place(self):
+ messages = anthropic_to_openai({"messages": [
+ {"role": "user", "content": [{"type": "text", "text": "What is this?"},
+ self.PNG,
+ {"type": "text", "text": "Be brief."}]}]})
+ self.assertEqual(messages, [{"role": "user", "content": [
+ {"type": "text", "text": "What is this?"}, self.PNG_PART,
+ {"type": "text", "text": "Be brief."}]}])
+
+ def test_text_only_content_still_collapses_to_a_string(self):
+ # Every existing renderer and test relies on this shape; a picture-free
+ # message must not start arriving as a parts list.
+ messages = anthropic_to_openai({"messages": [
+ {"role": "user", "content": [{"type": "text", "text": "a"},
+ {"type": "text", "text": "b"}]}]})
+ self.assertEqual(messages, [{"role": "user", "content": "ab"}])
+
+ def test_tool_result_image_rides_the_following_user_turn(self):
+ # Claude Code's Read tool on a .png returns the picture INSIDE the tool_result.
+ # The tool role carries text only, so the picture is carried onto the user
+ # turn that follows the results -- the model still sees it right after the
+ # tool answered, which is where the client put it.
+ messages = anthropic_to_openai({"messages": [
+ {"role": "user", "content": "look at it"},
+ {"role": "assistant", "content": [
+ {"type": "tool_use", "id": "t1", "name": "Read", "input": {"p": "x.png"}}]},
+ {"role": "user", "content": [
+ {"type": "tool_result", "tool_use_id": "t1",
+ "content": [{"type": "text", "text": "(image)"}, self.PNG]},
+ {"type": "text", "text": "and?"}]},
+ ]})
+ self.assertEqual([m["role"] for m in messages], ["user", "assistant", "tool", "user"])
+ self.assertEqual(messages[2]["content"], "(image)")
+ self.assertEqual(messages[3]["content"],
+ [self.PNG_PART, {"type": "text", "text": "and?"}])
+
+ def test_tool_result_image_alone_still_makes_a_user_turn(self):
+ # ...even when the tool_result is the whole message (no user text): dropping
+ # the picture here is exactly the silent loss this exists to prevent.
+ messages = anthropic_to_openai({"messages": [
+ {"role": "user", "content": "go"},
+ {"role": "assistant", "content": [
+ {"type": "tool_use", "id": "t1", "name": "Read", "input": {}}]},
+ {"role": "user", "content": [
+ {"type": "tool_result", "tool_use_id": "t1", "content": [self.PNG]}]},
+ ]})
+ self.assertEqual([m["role"] for m in messages], ["user", "assistant", "tool", "user"])
+ self.assertEqual(messages[2]["content"], "")
+ self.assertEqual(messages[3]["content"], [self.PNG_PART])
+
+ def test_image_in_an_assistant_message_is_refused(self):
+ for content in ([self.PNG],
+ [{"type": "tool_result", "tool_use_id": "t1", "content": [self.PNG]}]):
+ with self.assertRaises(APIError) as caught:
+ anthropic_to_openai({"messages": [
+ {"role": "user", "content": "hi"},
+ {"role": "assistant", "content": content}]})
+ self.assertEqual(caught.exception.status, 400)
+ self.assertIn("user messages", caught.exception.message)
+ self.assertEqual(caught.exception.param, "messages.1.content.0")
+
+ def test_image_source_is_validated(self):
+ for source in ({"type": "base64", "media_type": "image/png"}, # no data
+ {"type": "base64", "data": "abc"}, # no media_type
+ {"type": "file", "file_id": "f1"}, # not supported
+ "not-an-object"):
+ with self.assertRaises(APIError) as caught:
+ anthropic_to_openai({"messages": [{"role": "user", "content": [
+ {"type": "image", "source": source}]}]})
+ self.assertEqual(caught.exception.status, 400)
+ self.assertTrue(caught.exception.param.startswith("messages.0.content.0.source"),
+ caught.exception.param)
+
+ def test_url_source_is_handed_to_the_same_refusal_as_the_openai_path(self):
+ # The OpenAI endpoint refuses to fetch a remote URL by policy; the same URL
+ # arriving as an Anthropic `url` source must reach that same refusal, not a
+ # different one and not a fetch.
+ messages = anthropic_to_openai({"messages": [{"role": "user", "content": [
+ {"type": "image", "source": {"type": "url", "url": "https://x/y.png"}}]}]})
+ self.assertEqual(messages[0]["content"],
+ [{"type": "image_url", "image_url": {"url": "https://x/y.png"}}])
+
class MessagesHTTPTest(unittest.TestCase):
def setUp(self):
@@ -444,6 +539,63 @@ class MessagesHTTPTest(unittest.TestCase):
self.assertEqual(caught.exception.code, 400)
self.assertIn(field, json.load(caught.exception)["error"]["message"])
+ # ---- images end to end -----------------------------------------------------------
+ # The translation alone is not the contract: the picture has to reach the engine.
+ # The OpenAI path expands image parts into the family's placeholders and hands the
+ # patches to Engine.generate(image=...) beside the prompt; /v1/messages goes
+ # through the same expansion, so the placeholder count and the patches cannot
+ # disagree between the two endpoints.
+ IMAGE = {"type": "image", "source": {"type": "base64", "media_type": "image/png",
+ "data": "iVBORw0KGgo="}}
+
+ def fake_preprocess(self, data, model_dir):
+ self.preprocessed.append(data)
+ return b"\x00" * 16, 4, 4 # 2x2 = 4 placeholders after merging
+
+ def test_image_block_reaches_the_glm53_engine(self):
+ self.preprocessed = []
+ messages = [{"role": "user", "content": [{"type": "text", "text": "what is this?"},
+ self.IMAGE]}]
+ with patch("openai_server.ARCH", "glm53"), \
+ patch("openai_server._preprocess_image", self.fake_preprocess):
+ with self.post(self.base_body(messages=messages)) as response:
+ self.assertEqual(response.status, 200)
+ self.assertEqual(self.preprocessed, [b"\x89PNG\r\n\x1a\n"])
+ self.assertEqual(self.engine.images, [(b"\x00" * 16, 4, 4)])
+ self.assertIn("<|user|>what is this?<|begin_of_image|>" + "<|image|>" * 4
+ + "<|end_of_image|><|assistant|>", self.engine.prompts[-1])
+
+ def test_tool_result_image_reaches_the_glm53_engine_after_the_observation(self):
+ self.preprocessed = []
+ messages = [
+ {"role": "user", "content": "read it"},
+ {"role": "assistant", "content": [
+ {"type": "tool_use", "id": "t1", "name": "Read", "input": {"p": "x.png"}}]},
+ {"role": "user", "content": [
+ {"type": "tool_result", "tool_use_id": "t1", "content": [self.IMAGE]}]},
+ ]
+ with patch("openai_server.ARCH", "glm53"), \
+ patch("openai_server._preprocess_image", self.fake_preprocess):
+ with self.post(self.base_body(messages=messages)) as response:
+ self.assertEqual(response.status, 200)
+ self.assertEqual(self.engine.images, [(b"\x00" * 16, 4, 4)])
+ prompt = self.engine.prompts[-1]
+ self.assertLess(prompt.index("<|observation|>"), prompt.index("<|begin_of_image|>"))
+ self.assertIn("<|user|><|begin_of_image|>" + "<|image|>" * 4 + "<|end_of_image|>",
+ prompt)
+
+ def test_two_images_are_refused_like_the_openai_path(self):
+ self.preprocessed = []
+ messages = [{"role": "user", "content": [self.IMAGE, self.IMAGE]}]
+ with patch("openai_server.ARCH", "glm53"), \
+ patch("openai_server._preprocess_image", self.fake_preprocess):
+ with self.assertRaises(HTTPError) as caught:
+ self.post(self.base_body(messages=messages))
+ self.addCleanup(caught.exception.close)
+ self.assertEqual(caught.exception.code, 400)
+ self.assertIn("one image per request", json.load(caught.exception)["error"]["message"])
+ self.assertEqual(self.engine.images, [])
+
if __name__ == "__main__":
unittest.main()