Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(api): render Anthropic prompts per architecture

Mario Gutierrez140d91c48591
diff --git a/c/openai_server.py b/c/openai_server.py
index f1df5847..f58f2cfb 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -385,9 +385,10 @@ class InklingStreamSplit:
answer). Buffers partial markers across chunk boundaries so a marker split
between two DATA frames never leaks."""
- def __init__(self, on_content, on_reasoning=None):
+ def __init__(self, on_content, on_reasoning=None, on_reasoning_end=None):
self.on_content = on_content
self.on_reasoning = on_reasoning
+ self.on_reasoning_end = on_reasoning_end
self.mode = "content"
self.buf = ""
@@ -404,6 +405,8 @@ class InklingStreamSplit:
return
i, m = min(hits)
self._emit(self.buf[:i])
+ if m == INK_TEXT and self.mode == "reasoning" and self.on_reasoning_end:
+ self.on_reasoning_end()
self.mode = "reasoning" if m == INK_THINK else "content"
self.buf = self.buf[i + len(m):]
@@ -870,6 +873,17 @@ def render_chat(messages, enable_thinking=False, reasoning_effort=None, tools=No
return "".join(prompt)
+def render_chat_for_arch(messages, enable_thinking=False, reasoning_effort=None, tools=None,
+ tool_choice=None, audio_out=None):
+ """Render a chat request with the active engine's native prompt contract."""
+ if ARCH == "inkling":
+ return render_chat_inkling(messages, enable_thinking, reasoning_effort, tools,
+ tool_choice, audio_out=audio_out)
+ renderer = (render_chat_kimi if ARCH == "kimi" else
+ render_chat_v4 if ARCH == "deepseek_v4" else render_chat)
+ return renderer(messages, enable_thinking, reasoning_effort, tools, tool_choice)
+
+
# ---- Anthropic Messages API (#343) --------------------------------------------------------
# A translation layer, NOT a second engine path: /v1/messages rewrites an Anthropic-shaped
# request into the exact OpenAI-shaped body the existing path already validates, so prompt
@@ -2513,16 +2527,9 @@ class APIHandler(BaseHTTPRequestHandler):
raise APIError(400, "`enable_thinking` must be a boolean.", "enable_thinking")
tools = body.get("tools") or body.get("functions") or None
tool_choice = body.get("tool_choice")
- renderer = (render_chat_inkling if ARCH == "inkling" else
- render_chat_kimi if ARCH == "kimi" else
- render_chat_v4 if ARCH == "deepseek_v4" else render_chat)
audio_clips = [] if ARCH == "inkling" else None
- if audio_clips is not None:
- prompt = renderer(body.get("messages"), enable_thinking, reasoning_effort, tools,
- tool_choice, audio_out=audio_clips)
- else:
- prompt = renderer(body.get("messages"), enable_thinking, reasoning_effort, tools,
- tool_choice)
+ prompt = render_chat_for_arch(body.get("messages"), enable_thinking, reasoning_effort,
+ tools, tool_choice, audio_out=audio_clips)
self.generation(body, prompt, request_id, True, tools, tool_choice,
enable_thinking=enable_thinking,
audio=b"".join(audio_clips) if audio_clips else None)
@@ -2556,8 +2563,9 @@ class APIHandler(BaseHTTPRequestHandler):
translated["tool_choice"] = tool_choice
if tool_choice == "none":
tools = None
- prompt = render_chat(messages, enable_thinking, "high" if enable_thinking else None,
- tools, tool_choice)
+ prompt = render_chat_for_arch(messages, enable_thinking,
+ "high" if enable_thinking else None,
+ tools, tool_choice)
self.anthropic_generation(translated, prompt, request_id, tools, enable_thinking)
def anthropic_generation(self, body, prompt, request_id, tools, enable_thinking):
@@ -2588,8 +2596,12 @@ class APIHandler(BaseHTTPRequestHandler):
def blocks_and_stop(text, stats):
"""Split a finished reply into Anthropic content blocks + stop_reason."""
content = []
- if enable_thinking:
+ reasoning = ""
+ if ARCH == "inkling":
+ text, reasoning = split_inkling(text)
+ elif enable_thinking:
reasoning, text = split_thinking_reply(text)
+ if enable_thinking:
content.append({"type": "thinking", "thinking": reasoning,
"signature": ANTHROPIC_LOCAL_SIGNATURE})
calls = []
@@ -2727,8 +2739,13 @@ class APIHandler(BaseHTTPRequestHandler):
"signature": ANTHROPIC_LOCAL_SIGNATURE}})
send_event("content_block_stop", {"type": "content_block_stop", "index": 0})
- split = (ThinkingStreamSplit(emit_thinking, emit_answer, close_thinking)
- if enable_thinking else None)
+ if ARCH == "inkling":
+ split = InklingStreamSplit(emit_answer,
+ emit_thinking if enable_thinking else None,
+ close_thinking if enable_thinking else None)
+ else:
+ split = (ThinkingStreamSplit(emit_thinking, emit_answer, close_thinking)
+ if enable_thinking else None)
def on_text(chunk):
raw.append(chunk)
@@ -2740,7 +2757,7 @@ class APIHandler(BaseHTTPRequestHandler):
lambda: not connected[0], grammar=grammar, stopped=stop_filter.stopped)
stop_filter.finish()
if split:
- split.finish()
+ split.close()
close_thinking() # budget exhaustion before </think>
if tools and not state["in_tool"] and state["buf"]:
emit_text(state["buf"])
diff --git a/c/tests/test_anthropic_messages.py b/c/tests/test_anthropic_messages.py
index 1118ac45..f3abe586 100644
--- a/c/tests/test_anthropic_messages.py
+++ b/c/tests/test_anthropic_messages.py
@@ -14,11 +14,12 @@ import json
import re
import threading
import unittest
+from unittest.mock import patch
from urllib.error import HTTPError
from urllib.request import Request, urlopen
from openai_server import (APIServer, anthropic_to_openai, anthropic_tools, APIError,
- render_chat)
+ render_chat, render_chat_inkling, render_chat_kimi, render_chat_v4)
class FakeEngine:
@@ -153,6 +154,30 @@ class MessagesHTTPTest(unittest.TestCase):
self.assertEqual(payload["usage"], {"input_tokens": 11, "output_tokens": 3})
self.assertTrue(payload["id"].startswith("msg_"))
+ def test_each_architecture_receives_its_native_chat_prompt(self):
+ messages = [{"role": "user", "content": "Hi"}]
+ renderers = {
+ "glm": render_chat,
+ "inkling": render_chat_inkling,
+ "kimi": render_chat_kimi,
+ "deepseek_v4": render_chat_v4,
+ }
+ for arch, renderer in renderers.items():
+ with self.subTest(arch=arch), patch("openai_server.ARCH", arch):
+ self.post(self.base_body()).close()
+ self.assertEqual(self.engine.prompts[-1], renderer(messages))
+
+ def test_non_glm_architectures_reject_tools_before_generation(self):
+ body = self.base_body(tools=[{"name": "f", "input_schema": {"type": "object"}}])
+ for arch in ("inkling", "kimi", "deepseek_v4"):
+ with self.subTest(arch=arch), patch("openai_server.ARCH", arch):
+ before = len(self.engine.prompts)
+ with self.assertRaises(HTTPError) as caught:
+ self.post(body)
+ self.assertEqual(caught.exception.code, 400)
+ self.assertIn("tool", json.load(caught.exception)["error"]["message"].lower())
+ self.assertEqual(len(self.engine.prompts), before)
+
def test_x_api_key_and_bearer_both_authenticate(self):
with self.post(self.base_body(), {"x-api-key": "secret"}) as response:
self.assertEqual(response.status, 200)
@@ -272,6 +297,35 @@ class MessagesHTTPTest(unittest.TestCase):
{"type": "text", "text": "answer"},
])
+ def test_inkling_thinking_uses_inkling_content_markers(self):
+ self.engine.script = ("<|content_thinking|>reason", "ing<|content_text|>answer",)
+ with patch("openai_server.ARCH", "inkling"):
+ with self.post(self.base_body(thinking={"type": "enabled"})) as response:
+ payload = json.load(response)
+ self.assertEqual(payload["content"], [
+ {"type": "thinking", "thinking": "reasoning", "signature": "colibri-local"},
+ {"type": "text", "text": "answer"},
+ ])
+
+ def test_streamed_inkling_thinking_uses_inkling_content_markers(self):
+ self.engine.script = ("<|content_think", "ing|>reasoning<|content_", "text|>answer",)
+ with patch("openai_server.ARCH", "inkling"):
+ with self.post(self.base_body(stream=True, thinking={"type": "enabled"})) as response:
+ raw = response.read().decode()
+ payloads = [json.loads(line[len("data: "):]) for line in raw.splitlines()
+ if line.startswith("data: ")]
+ deltas = [payload["delta"] for payload in payloads
+ if payload["type"] == "content_block_delta"]
+ self.assertEqual("".join(delta.get("thinking", "") for delta in deltas), "reasoning")
+ self.assertEqual("".join(delta.get("text", "") for delta in deltas), "answer")
+ thinking_stop = next(index for index, payload in enumerate(payloads)
+ if payload["type"] == "content_block_stop" and payload["index"] == 0)
+ text_start = next(index for index, payload in enumerate(payloads)
+ if payload["type"] == "content_block_start" and payload["index"] == 1)
+ self.assertLess(thinking_stop, text_start)
+ self.assertNotIn("content_thinking", raw)
+ self.assertNotIn("content_text", raw)
+
def test_streamed_thinking_marker_can_split_at_every_boundary(self):
marker = "</think>"
for boundary in range(1, len(marker)):
diff --git a/docs/api.md b/docs/api.md
index 7aac3f5c..45d48eb1 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -98,18 +98,27 @@ export ANTHROPIC_MODEL=glm-5.2-colibri
claude
```
-Supported: system prompts (string or text blocks), multi-turn `user`/`assistant`
-messages, `text` / `tool_use` / `tool_result` content blocks, tools with
-`input_schema`, every `tool_choice` mode, streaming with the full named-event
+Supported on every served architecture: system prompts (string or text blocks),
+multi-turn `user`/`assistant` messages, streaming with the full named-event
sequence (`message_start` β†’ `content_block_*` β†’ `message_delta` β†’ `message_stop`,
-plus protocol `ping` keepalives during long prefills), `stop_reason`
-(`end_turn` / `max_tokens` / `tool_use`), Anthropic `usage` field names, and
-`x-api-key` authentication (`Authorization: Bearer` also works). Extended
-thinking is enabled with `{"thinking": {"type": "enabled"}}`.
+plus protocol `ping` keepalives during long prefills), `stop_reason`, Anthropic
+`usage` field names, and `x-api-key` authentication (`Authorization: Bearer`
+also works). The gateway renders each request with the active engine's native
+chat template; GLM, Inkling, Kimi K3, and DeepSeek V4 prompts are not
+interchangeable. Extended thinking is enabled with
+`{"thinking": {"type": "enabled"}}` and is translated to that architecture's
+reasoning protocol.
+
+Tool use (`tool_use` / `tool_result`, `input_schema`, and every `tool_choice`
+mode) is currently GLM-only. Inkling, Kimi K3, and DeepSeek V4 reject tool
+requests explicitly instead of feeding GLM tool markers to an incompatible
+tokenizer.
Not supported, and refused explicitly rather than ignored: `stop_sequences`,
`top_k`, and non-text content blocks (images, documents). Errors use Anthropic's
-own `{"type":"error","error":{...}}` envelope on this path.
+own `{"type":"error","error":{...}}` envelope on this path. Architecture-local
+features that have not been wired to this protocol are likewise rejected with
+an explicit error.
> The prefill warning below applies here too, and applies *hardest* to Claude Code:
> its system prompt and tool catalog are large, and on a disk-streaming CPU path