Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
fix(api): render Anthropic prompts per architecture
Mario Gutierrez140d91c48591
diff --git a/c/openai_server.py b/c/openai_server.pyindex f1df5847..f58f2cfb 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -385,9 +385,10 @@ class InklingStreamSplit:answer). Buffers partial markers across chunk boundaries so a marker splitbetween two DATA frames never leaks."""- def __init__(self, on_content, on_reasoning=None):+ def __init__(self, on_content, on_reasoning=None, on_reasoning_end=None):self.on_content = on_contentself.on_reasoning = on_reasoning+ self.on_reasoning_end = on_reasoning_endself.mode = "content"self.buf = ""@@ -404,6 +405,8 @@ class InklingStreamSplit:returni, m = min(hits)self._emit(self.buf[:i])+ if m == INK_TEXT and self.mode == "reasoning" and self.on_reasoning_end:+ self.on_reasoning_end()self.mode = "reasoning" if m == INK_THINK else "content"self.buf = self.buf[i + len(m):]@@ -870,6 +873,17 @@ def render_chat(messages, enable_thinking=False, reasoning_effort=None, tools=Noreturn "".join(prompt)+def render_chat_for_arch(messages, enable_thinking=False, reasoning_effort=None, tools=None,+ tool_choice=None, audio_out=None):+ """Render a chat request with the active engine's native prompt contract."""+ if ARCH == "inkling":+ return render_chat_inkling(messages, enable_thinking, reasoning_effort, tools,+ tool_choice, audio_out=audio_out)+ renderer = (render_chat_kimi if ARCH == "kimi" else+ render_chat_v4 if ARCH == "deepseek_v4" else render_chat)+ return renderer(messages, enable_thinking, reasoning_effort, tools, tool_choice)++# ---- Anthropic Messages API (#343) --------------------------------------------------------# A translation layer, NOT a second engine path: /v1/messages rewrites an Anthropic-shaped# request into the exact OpenAI-shaped body the existing path already validates, so prompt@@ -2513,16 +2527,9 @@ class APIHandler(BaseHTTPRequestHandler):raise APIError(400, "`enable_thinking` must be a boolean.", "enable_thinking")tools = body.get("tools") or body.get("functions") or Nonetool_choice = body.get("tool_choice")- renderer = (render_chat_inkling if ARCH == "inkling" else- render_chat_kimi if ARCH == "kimi" else- render_chat_v4 if ARCH == "deepseek_v4" else render_chat)audio_clips = [] if ARCH == "inkling" else None- if audio_clips is not None:- prompt = renderer(body.get("messages"), enable_thinking, reasoning_effort, tools,- tool_choice, audio_out=audio_clips)- else:- prompt = renderer(body.get("messages"), enable_thinking, reasoning_effort, tools,- tool_choice)+ prompt = render_chat_for_arch(body.get("messages"), enable_thinking, reasoning_effort,+ tools, tool_choice, audio_out=audio_clips)self.generation(body, prompt, request_id, True, tools, tool_choice,enable_thinking=enable_thinking,audio=b"".join(audio_clips) if audio_clips else None)@@ -2556,8 +2563,9 @@ class APIHandler(BaseHTTPRequestHandler):translated["tool_choice"] = tool_choiceif tool_choice == "none":tools = None- prompt = render_chat(messages, enable_thinking, "high" if enable_thinking else None,- tools, tool_choice)+ prompt = render_chat_for_arch(messages, enable_thinking,+ "high" if enable_thinking else None,+ tools, tool_choice)self.anthropic_generation(translated, prompt, request_id, tools, enable_thinking)def anthropic_generation(self, body, prompt, request_id, tools, enable_thinking):@@ -2588,8 +2596,12 @@ class APIHandler(BaseHTTPRequestHandler):def blocks_and_stop(text, stats):"""Split a finished reply into Anthropic content blocks + stop_reason."""content = []- if enable_thinking:+ reasoning = ""+ if ARCH == "inkling":+ text, reasoning = split_inkling(text)+ elif enable_thinking:reasoning, text = split_thinking_reply(text)+ if enable_thinking:content.append({"type": "thinking", "thinking": reasoning,"signature": ANTHROPIC_LOCAL_SIGNATURE})calls = []@@ -2727,8 +2739,13 @@ class APIHandler(BaseHTTPRequestHandler):"signature": ANTHROPIC_LOCAL_SIGNATURE}})send_event("content_block_stop", {"type": "content_block_stop", "index": 0})- split = (ThinkingStreamSplit(emit_thinking, emit_answer, close_thinking)- if enable_thinking else None)+ if ARCH == "inkling":+ split = InklingStreamSplit(emit_answer,+ emit_thinking if enable_thinking else None,+ close_thinking if enable_thinking else None)+ else:+ split = (ThinkingStreamSplit(emit_thinking, emit_answer, close_thinking)+ if enable_thinking else None)def on_text(chunk):raw.append(chunk)@@ -2740,7 +2757,7 @@ class APIHandler(BaseHTTPRequestHandler):lambda: not connected[0], grammar=grammar, stopped=stop_filter.stopped)stop_filter.finish()if split:- split.finish()+ split.close()close_thinking() # budget exhaustion before </think>if tools and not state["in_tool"] and state["buf"]:emit_text(state["buf"])diff --git a/c/tests/test_anthropic_messages.py b/c/tests/test_anthropic_messages.pyindex 1118ac45..f3abe586 100644--- a/c/tests/test_anthropic_messages.py+++ b/c/tests/test_anthropic_messages.py@@ -14,11 +14,12 @@ import jsonimport reimport threadingimport unittest+from unittest.mock import patchfrom urllib.error import HTTPErrorfrom urllib.request import Request, urlopenfrom openai_server import (APIServer, anthropic_to_openai, anthropic_tools, APIError,- render_chat)+ render_chat, render_chat_inkling, render_chat_kimi, render_chat_v4)class FakeEngine:@@ -153,6 +154,30 @@ class MessagesHTTPTest(unittest.TestCase):self.assertEqual(payload["usage"], {"input_tokens": 11, "output_tokens": 3})self.assertTrue(payload["id"].startswith("msg_"))+ def test_each_architecture_receives_its_native_chat_prompt(self):+ messages = [{"role": "user", "content": "Hi"}]+ renderers = {+ "glm": render_chat,+ "inkling": render_chat_inkling,+ "kimi": render_chat_kimi,+ "deepseek_v4": render_chat_v4,+ }+ for arch, renderer in renderers.items():+ with self.subTest(arch=arch), patch("openai_server.ARCH", arch):+ self.post(self.base_body()).close()+ self.assertEqual(self.engine.prompts[-1], renderer(messages))++ def test_non_glm_architectures_reject_tools_before_generation(self):+ body = self.base_body(tools=[{"name": "f", "input_schema": {"type": "object"}}])+ for arch in ("inkling", "kimi", "deepseek_v4"):+ with self.subTest(arch=arch), patch("openai_server.ARCH", arch):+ before = len(self.engine.prompts)+ with self.assertRaises(HTTPError) as caught:+ self.post(body)+ self.assertEqual(caught.exception.code, 400)+ self.assertIn("tool", json.load(caught.exception)["error"]["message"].lower())+ self.assertEqual(len(self.engine.prompts), before)+def test_x_api_key_and_bearer_both_authenticate(self):with self.post(self.base_body(), {"x-api-key": "secret"}) as response:self.assertEqual(response.status, 200)@@ -272,6 +297,35 @@ class MessagesHTTPTest(unittest.TestCase):{"type": "text", "text": "answer"},])+ def test_inkling_thinking_uses_inkling_content_markers(self):+ self.engine.script = ("<|content_thinking|>reason", "ing<|content_text|>answer",)+ with patch("openai_server.ARCH", "inkling"):+ with self.post(self.base_body(thinking={"type": "enabled"})) as response:+ payload = json.load(response)+ self.assertEqual(payload["content"], [+ {"type": "thinking", "thinking": "reasoning", "signature": "colibri-local"},+ {"type": "text", "text": "answer"},+ ])++ def test_streamed_inkling_thinking_uses_inkling_content_markers(self):+ self.engine.script = ("<|content_think", "ing|>reasoning<|content_", "text|>answer",)+ with patch("openai_server.ARCH", "inkling"):+ with self.post(self.base_body(stream=True, thinking={"type": "enabled"})) as response:+ raw = response.read().decode()+ payloads = [json.loads(line[len("data: "):]) for line in raw.splitlines()+ if line.startswith("data: ")]+ deltas = [payload["delta"] for payload in payloads+ if payload["type"] == "content_block_delta"]+ self.assertEqual("".join(delta.get("thinking", "") for delta in deltas), "reasoning")+ self.assertEqual("".join(delta.get("text", "") for delta in deltas), "answer")+ thinking_stop = next(index for index, payload in enumerate(payloads)+ if payload["type"] == "content_block_stop" and payload["index"] == 0)+ text_start = next(index for index, payload in enumerate(payloads)+ if payload["type"] == "content_block_start" and payload["index"] == 1)+ self.assertLess(thinking_stop, text_start)+ self.assertNotIn("content_thinking", raw)+ self.assertNotIn("content_text", raw)+def test_streamed_thinking_marker_can_split_at_every_boundary(self):marker = "</think>"for boundary in range(1, len(marker)):diff --git a/docs/api.md b/docs/api.mdindex 7aac3f5c..45d48eb1 100644--- a/docs/api.md+++ b/docs/api.md@@ -98,18 +98,27 @@ export ANTHROPIC_MODEL=glm-5.2-colibriclaude```-Supported: system prompts (string or text blocks), multi-turn `user`/`assistant`-messages, `text` / `tool_use` / `tool_result` content blocks, tools with-`input_schema`, every `tool_choice` mode, streaming with the full named-event+Supported on every served architecture: system prompts (string or text blocks),+multi-turn `user`/`assistant` messages, streaming with the full named-eventsequence (`message_start` β `content_block_*` β `message_delta` β `message_stop`,-plus protocol `ping` keepalives during long prefills), `stop_reason`-(`end_turn` / `max_tokens` / `tool_use`), Anthropic `usage` field names, and-`x-api-key` authentication (`Authorization: Bearer` also works). Extended-thinking is enabled with `{"thinking": {"type": "enabled"}}`.+plus protocol `ping` keepalives during long prefills), `stop_reason`, Anthropic+`usage` field names, and `x-api-key` authentication (`Authorization: Bearer`+also works). The gateway renders each request with the active engine's native+chat template; GLM, Inkling, Kimi K3, and DeepSeek V4 prompts are not+interchangeable. Extended thinking is enabled with+`{"thinking": {"type": "enabled"}}` and is translated to that architecture's+reasoning protocol.++Tool use (`tool_use` / `tool_result`, `input_schema`, and every `tool_choice`+mode) is currently GLM-only. Inkling, Kimi K3, and DeepSeek V4 reject tool+requests explicitly instead of feeding GLM tool markers to an incompatible+tokenizer.Not supported, and refused explicitly rather than ignored: `stop_sequences`,`top_k`, and non-text content blocks (images, documents). Errors use Anthropic's-own `{"type":"error","error":{...}}` envelope on this path.+own `{"type":"error","error":{...}}` envelope on this path. Architecture-local+features that have not been wired to this protocol are likewise rejected with+an explicit error.> The prefill warning below applies here too, and applies *hardest* to Claude Code:> its system prompt and tool catalog are large, and on a disk-streaming CPU path