Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(deepseek-v41): tool calling, in the checkpoint's own DSML format

JustVugg084206256359
diff --git a/c/openai_server.py b/c/openai_server.py
index 24c95644..3b90bde1 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -21,6 +21,7 @@ import time
import uuid
import v4_dsml # vendored DeepSeek V4 DSML reference primitives
+import v41_dsml # ...and V4.1's, whose tag names differ by a space
from family_registry import (FamilyConfigError, UnknownFamilyError, family_by_id,
family_ids, resolve_model)
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
@@ -532,10 +533,34 @@ def parse_k3_tool_calls(reply, tools=None):
return text.strip(), calls
+def parse_dsv41_tool_calls(reply):
+ """Parse DeepSeek V4.1 DSML tool calls out of one assistant reply.
+
+ Same posture as the V4 path: the vendored reference parser decodes the block
+ strictly, and the gateway then cuts an incomplete block out of the visible
+ content so raw DSML never reaches the client, whatever the model did with
+ its token budget.
+ """
+ content, calls = v41_dsml.parse_completion_text(reply)
+ if not calls:
+ cut = len(content)
+ for marker in (v41_dsml.TOOL_CALLS_PREFIX, v41_dsml.TOOL_CALL_PREFIX):
+ pos = content.find(marker)
+ if 0 <= pos < cut:
+ cut = pos
+ if cut < len(content):
+ content = content[:cut]
+ for marker in (v41_dsml.eos_token, THINK_OPEN, THINK_CLOSE):
+ content = content.replace(marker, "")
+ return content.strip(), calls
+
+
def parse_arch_tool_calls(reply, tools, tool_reply=None):
"""Architecture-appropriate tool-call parser. Returns (content, tool_calls)."""
if ARCH == "deepseek_v4":
return parse_dsv4_tool_calls(reply)
+ if ARCH == "deepseek_v41":
+ return parse_dsv41_tool_calls(reply)
if ARCH == "kimi":
if tool_reply is not None:
_sideband_text, calls = parse_k3_tool_calls(tool_reply, tools)
@@ -550,6 +575,10 @@ def _tool_stream_markers():
"""Marker(s) that open a model tool-call block, in match order (arch-specific)."""
if ARCH == "deepseek_v4":
return ("<" + DSV4_DSML + "tool_calls", "<" + DSV4_DSML + "invoke")
+ if ARCH == "deepseek_v41":
+ # V4.1's tag names lead with a space (" calls", " invoke"): building these the
+ # V4 way would suppress nothing and leak the block into delta.content.
+ return (v41_dsml.TOOL_CALLS_PREFIX, v41_dsml.TOOL_CALL_PREFIX)
if ARCH == "kimi":
return (K3_TOOLS_OPEN,)
return (BOX_START,)
@@ -1982,6 +2011,15 @@ def render_chat_glm53(messages, enable_thinking=False, reasoning_effort=None, to
# <|User|> / <|Assistant|>
# <think> or </think> the generation cue: open in thinking mode, closed otherwise
# assistant turns end with <|end▁of▁sentence|>
+#
+# Two details the reference hides in its control flow, both reproduced below. First,
+# <think> and </think> straddle a turn boundary: the OPENING token is the transition the
+# reference appends after a user turn, the CLOSING one belongs to the assistant turn that
+# follows, so an assistant turn always starts with one or the other and there is no such
+# thing as a past assistant turn without a </think>. Second, the reference drops the
+# reasoning of turns before the last user message -- except when the conversation declares
+# tools, where it keeps every one of them (`if any(m.get("tools") ...)`), because a tool
+# call is only interpretable next to the reasoning that produced it.
DSV41_BOS = "<|begin▁of▁sentence|>"
DSV41_EOS = "<|end▁of▁sentence|>"
DSV41_SYSTEM = "<|System|>"
@@ -1991,55 +2029,149 @@ DSV41_ASSISTANT = "<|Assistant|>"
DSV41_EFFORT = {"minimal": 50, "low": 50, "medium": 75, "high": 75, "xhigh": 100}
+def _dsv41_tools_block(tools):
+ """V4.1 tool-declaration block, rendered by the vendored reference template."""
+ schemas = []
+ for tool in (tools or []):
+ fn = tool.get("function", tool) if isinstance(tool, dict) else {}
+ # Gateway-side scrub: OpenAI clients attach routing hints the model
+ # schema must not carry.
+ clean = {k: v for k, v in fn.items() if k not in ("defer_loading", "strict")}
+ if isinstance(tool, dict) and tool.get("namespace") is not None:
+ clean = dict(clean, namespace=tool["namespace"])
+ schemas.append(clean)
+ return v41_dsml.render_tools(v41_dsml.tools_from_openai_format(schemas))
+
+
+def _dsv41_merge_turns(messages):
+ """encoding.py merge_tool_messages + sort_tool_results_by_call_order.
+
+ V4.1 has no standalone tool role: a tool result is a <tool_result> block inside the
+ NEXT user turn, and consecutive user-side messages collapse into one turn joined by
+ a blank line. Returns turns of {"role", "parts"/"content", ...}, validating each
+ original message so field-level errors keep pointing at the client's own indices.
+ """
+ turns = []
+ for index, message in enumerate(messages):
+ if not isinstance(message, dict):
+ raise APIError(400, "Each message must be an object.", f"messages.{index}")
+ role = message.get("role")
+ if role == "developer":
+ role = "system"
+ if role not in ("system", "user", "assistant", "tool"):
+ raise APIError(400, f"Unsupported role {role!r}.", f"messages.{index}.role")
+ raw = message.get("content")
+ text = content_text(raw, f"messages.{index}.content") if raw is not None else ""
+ if role == "assistant":
+ reasoning = message.get("reasoning_content")
+ if reasoning is not None and not isinstance(reasoning, str):
+ raise APIError(400, "`reasoning_content` must be a string.",
+ f"messages.{index}.reasoning_content")
+ turns.append({"role": "assistant", "content": text,
+ "reasoning_content": reasoning,
+ "tool_calls": message.get("tool_calls")})
+ elif role in ("user", "tool"):
+ block = ({"kind": "tool_result", "id": message.get("tool_call_id") or "",
+ "text": v41_dsml.render_tool_result(text)} if role == "tool"
+ else {"kind": "text", "id": "", "text": text})
+ if turns and turns[-1]["role"] == "user":
+ turns[-1]["parts"].append(block)
+ else:
+ turns.append({"role": "user", "parts": [block]})
+ else:
+ turns.append({"role": "system", "content": text})
+ # A parallel-call round trip arrives in whatever order the client's tools finished;
+ # the model reads them positionally, so restore the order it called them in.
+ order = {}
+ for turn in turns:
+ if turn["role"] == "assistant" and turn.get("tool_calls"):
+ order = {}
+ for at, call in enumerate(turn["tool_calls"]):
+ identifier = call.get("id") if isinstance(call, dict) else None
+ if identifier:
+ order[identifier] = at
+ elif turn["role"] == "user" and order:
+ results = [p for p in turn["parts"] if p["kind"] == "tool_result"]
+ if len(results) > 1:
+ results.sort(key=lambda p: order.get(p["id"], 0))
+ feed = iter(results)
+ turn["parts"] = [next(feed) if p["kind"] == "tool_result" else p
+ for p in turn["parts"]]
+ return turns
+
+
def render_chat_dsv41(messages, enable_thinking=False, reasoning_effort=None, tools=None,
tool_choice=None):
"""encoding.py _encode_messages_text for one turn.
- `tools` and `tool_choice` are accepted and refused rather than ignored: V4.1 declares
- tools inside the system message as a DSML block, and half-rendering that would produce
- a model that announces tools it cannot be told the results of.
+ Tool use follows the checkpoint's own DSML format (encoding/encoding.py, vendored in
+ v41_dsml.py): schemas are declared at the end of the system message, assistant tool
+ calls are <|DSML| calls> blocks, and tool results are <tool_result> blocks merged
+ into the following user turn.
"""
- if tools or (isinstance(tool_choice, dict) and tool_choice):
- raise APIError(400, "DeepSeek V4.1 tool calling is not wired in this build; send "
- "the request without `tools`.", "tools")
if not isinstance(messages, list) or not messages:
raise APIError(400, "`messages` must be a non-empty array.", "messages")
+ forced = None
+ if isinstance(tool_choice, dict):
+ forced = ((tool_choice.get("function") or {}).get("name")
+ or tool_choice.get("name"))
+ if forced:
+ tools = [t for t in (tools or [])
+ if ((t.get("function", t) if isinstance(t, dict) else {}).get("name") == forced)]
+ elif tool_choice == "none":
+ tools = None # the client forbade tools: do not offer them
+ turns = _dsv41_merge_turns(messages)
+ if tools:
+ tools_text = _dsv41_tools_block(tools)
+ if forced:
+ tools_text += f"\n\nYou must call the function `{forced}`. Do not answer directly."
+ elif tool_choice == "required":
+ tools_text += "\n\nYou must call one of the functions above. Do not answer directly."
+ for turn in turns:
+ if turn["role"] == "system":
+ turn["content"] += "\n\n" + tools_text
+ break
+ else:
+ # No system message: the reference renders tools on an empty one, so the block
+ # arrives as <|System|> + "\n\n" + tools. Keep that exact byte layout.
+ turns.insert(0, {"role": "system", "content": "\n\n" + tools_text})
budget = DSV41_EFFORT.get(reasoning_effort or "high", 75)
- prompt = [DSV41_BOS]
+ effort = (f"Reasoning Effort: {budget} (range 1-100, the higher the value, the more "
+ f"thorough the reasoning)\n\n") if enable_thinking else ""
+ # find_last_user_index: a mid-conversation system message counts as a user turn,
+ # because it is one of the two things the reference puts an assistant header after.
last_user = -1
- for index, message in enumerate(messages):
- if isinstance(message, dict) and message.get("role") == "user":
+ for index, turn in enumerate(turns):
+ if turn["role"] == "user" or (turn["role"] == "system" and index > 0):
last_user = index
- for index, message in enumerate(messages):
- if not isinstance(message, dict):
- raise APIError(400, "Each message must be an object.", f"messages.{index}")
- role = message.get("role")
- if role == "developer":
- role = "system"
- if role not in ("system", "user", "assistant"):
- raise APIError(400, f"Unsupported role {role!r}.", f"messages.{index}.role")
- content = content_text(message.get("content"), f"messages.{index}.content")
+ # With tools on the table the reference keeps every turn's reasoning; without them it
+ # drops the reasoning of everything before the last user message.
+ drop_thinking = not tools
+ prompt = [DSV41_BOS]
+ for index, turn in enumerate(turns):
+ role = turn["role"]
if role == "system":
prompt.append(DSV41_SYSTEM)
- if index == 0 and enable_thinking:
- prompt.append(f"Reasoning Effort: {budget} (range 1-100, the higher the "
- f"value, the more thorough the reasoning)\n\n")
- prompt.append(content)
+ if index == 0:
+ prompt.append(effort)
+ prompt.append(turn["content"])
elif role == "user":
- if index == 0 and enable_thinking:
- prompt.append(DSV41_SYSTEM)
- prompt.append(f"Reasoning Effort: {budget} (range 1-100, the higher the "
- f"value, the more thorough the reasoning)\n\n")
- prompt.append(DSV41_USER + content)
+ if index == 0 and effort:
+ prompt.append(DSV41_SYSTEM + effort)
+ prompt.append(DSV41_USER)
+ prompt.append("\n\n".join(part["text"] for part in turn["parts"]))
else:
- reasoning = message.get("reasoning_content")
- opened = ""
- if isinstance(reasoning, str) and reasoning:
- opened = f"<think>{reasoning}</think>"
- prompt.append(DSV41_ASSISTANT + opened + content + DSV41_EOS)
+ keep = enable_thinking and (not drop_thinking or index > last_user)
+ prompt.append(DSV41_ASSISTANT)
+ prompt.append(f"<think>{turn.get('reasoning_content') or ''}</think>"
+ if keep else "</think>")
+ prompt.append(turn["content"])
+ if turn.get("tool_calls"):
+ prompt.append(v41_dsml.render_tool_calls(turn["tool_calls"]))
+ prompt.append(DSV41_EOS)
# the generation cue, exactly as render_message appends it after a user turn
prompt.append(DSV41_ASSISTANT)
- prompt.append("<think>" if enable_thinking and len(messages) - 1 >= last_user else "</think>")
+ prompt.append("<think>" if enable_thinking and len(turns) - 1 >= last_user else "</think>")
return "".join(prompt)
diff --git a/c/tests/fixtures/dsv41_encoding_case1.json b/c/tests/fixtures/dsv41_encoding_case1.json
new file mode 100644
index 00000000..c7435727
--- /dev/null
+++ b/c/tests/fixtures/dsv41_encoding_case1.json
@@ -0,0 +1,82 @@
+{
+ "thinking_mode": "thinking",
+ "tools": [
+ {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "description": "Get the weather for a specific location",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "location": {
+ "type": "string",
+ "description": "The city name"
+ },
+ "unit": {
+ "type": "string",
+ "enum": ["celsius", "fahrenheit"],
+ "description": "Temperature unit"
+ }
+ },
+ "required": ["location"]
+ }
+ }
+ },
+ {
+ "type": "function",
+ "function": {
+ "name": "search",
+ "description": "Search the web for information",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "query": {
+ "type": "string",
+ "description": "Search query"
+ },
+ "num_results": {
+ "type": "integer",
+ "description": "Number of results to return"
+ }
+ },
+ "required": ["query"]
+ }
+ }
+ }
+ ],
+ "messages": [
+ {
+ "role": "system",
+ "content": "You are a helpful assistant."
+ },
+ {
+ "role": "user",
+ "content": "What's the weather like in Beijing?"
+ },
+ {
+ "role": "assistant",
+ "reasoning_content": "The user wants the weather in Beijing. I should call get_weather.",
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "arguments": "{\"location\": \"Beijing\", \"unit\": \"celsius\"}"
+ }
+ }
+ ]
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "call_0",
+ "content": "{\"temperature\": 22, \"condition\": \"sunny\", \"humidity\": 45}"
+ },
+ {
+ "role": "assistant",
+ "reasoning_content": "Got the weather data. Let me format a nice response.",
+ "content": "The weather in Beijing is currently sunny with a temperature of 22\u00b0C and 45% humidity."
+ }
+ ]
+}
diff --git a/c/tests/fixtures/dsv41_encoding_case1.txt b/c/tests/fixtures/dsv41_encoding_case1.txt
new file mode 100644
index 00000000..3dc9bfe9
--- /dev/null
+++ b/c/tests/fixtures/dsv41_encoding_case1.txt
@@ -0,0 +1,38 @@
+<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)
+
+You are a helpful assistant.
+
+## Tools
+
+You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following:
+
+<|DSML| calls>
+<|DSML| invoke name="$TOOL_NAME">
+<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML| parameter>
+...
+</|DSML| invoke>
+<|DSML| invoke name="$TOOL_NAME2">
+...
+</|DSML| invoke>
+</|DSML| calls>
+
+String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
+
+If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
+
+Otherwise, output directly after </think> with tool calls or final response.
+
+### Available Tool Schemas
+
+{"name": "get_weather", "description": "Get the weather for a specific location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city name"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "Temperature unit"}}, "required": ["location"]}}
+{"name": "search", "description": "Search the web for information", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "Search query"}, "num_results": {"type": "integer", "description": "Number of results to return"}}, "required": ["query"]}}
+
+You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
+<|User|>What's the weather like in Beijing?<|Assistant|><think>The user wants the weather in Beijing. I should call get_weather.</think>
+
+<|DSML| calls>
+<|DSML| invoke name="get_weather">
+<|DSML| parameter name="location" string="true">Beijing</|DSML| parameter>
+<|DSML| parameter name="unit" string="true">celsius</|DSML| parameter>
+</|DSML| invoke>
+</|DSML| calls><|end▁of▁sentence|><|User|><tool_result>{"temperature": 22, "condition": "sunny", "humidity": 45}</tool_result><|Assistant|><think>Got the weather data. Let me format a nice response.</think>The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity.<|end▁of▁sentence|>
\ No newline at end of file
diff --git a/c/tests/fixtures/dsv41_encoding_case2.json b/c/tests/fixtures/dsv41_encoding_case2.json
new file mode 100644
index 00000000..132bb05a
--- /dev/null
+++ b/c/tests/fixtures/dsv41_encoding_case2.json
@@ -0,0 +1,24 @@
+[
+ {
+ "role": "system",
+ "content": "You are a helpful assistant."
+ },
+ {
+ "role": "user",
+ "content": "Hello"
+ },
+ {
+ "role": "assistant",
+ "reasoning_content": "The user said hello, I should greet back.",
+ "content": "Hi there! How can I help you?"
+ },
+ {
+ "role": "user",
+ "content": "What is the capital of France?"
+ },
+ {
+ "role": "assistant",
+ "reasoning_content": "The user asks about the capital of France. It is Paris.",
+ "content": "The capital of France is Paris."
+ }
+]
diff --git a/c/tests/fixtures/dsv41_encoding_case2.txt b/c/tests/fixtures/dsv41_encoding_case2.txt
new file mode 100644
index 00000000..78eb6be5
--- /dev/null
+++ b/c/tests/fixtures/dsv41_encoding_case2.txt
@@ -0,0 +1 @@
+<|begin▁of▁sentence|><|System|>You are a helpful assistant.<|User|>Hello<|Assistant|></think>Hi there! How can I help you?<|end▁of▁sentence|><|User|>What is the capital of France?<|Assistant|></think>The capital of France is Paris.<|end▁of▁sentence|>
\ No newline at end of file
diff --git a/c/tests/test_openai_tools_v41_e2e.py b/c/tests/test_openai_tools_v41_e2e.py
new file mode 100644
index 00000000..59cd3a37
--- /dev/null
+++ b/c/tests/test_openai_tools_v41_e2e.py
@@ -0,0 +1,271 @@
+"""End-to-end DeepSeek V4.1 tool-calling test for the OpenAI gateway.
+
+Mirrors test_openai_tools_v4_e2e.py against the deepseek_v41 arch: tool
+declaration in the checkpoint's native DSML encoding (v41_dsml.py, vendored
+from the DeepSeek-V4.1-Flash reference), DSML marker suppression in streamed
+deltas, tool_calls in both response shapes, and the <tool_result> round trip.
+Runs against a mock engine speaking the SERVE wire protocol; no real
+checkpoint needed.
+
+The second class needs no server at all: it renders the checkpoint's own
+encoding test cases and compares them byte for byte with the vendor's expected
+prompts, which is the only check that catches a tag name drifting by one
+character.
+"""
+import json
+import os
+import socket
+import subprocess
+import sys
+import tempfile
+import unittest
+import urllib.request
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+SERVER = HERE.parent / "openai_server.py"
+sys.path.insert(0, str(HERE.parent))
+
+ASSISTANT = "<|Assistant|>"
+
+# Mock engine: replies are keyed on the prompt so one process covers every case.
+# Prompts received are appended to MOCK_LOG for assertions on the rendering.
+# Note the space inside every tag name: that is V4.1's format, not a typo.
+MOCK_ENGINE = r'''#!/usr/bin/env python3
+import sys, os
+out, inp = sys.stdout.buffer, sys.stdin.buffer
+out.write(b"\x01\x01READY\x01\x01\n" + b"STAT 0 0 0 0 0\n"); out.flush()
+
+DSML = ("\n\n<|DSML| calls>\n"
+ "<|DSML| invoke name=\"get_weather\">\n"
+ "<|DSML| parameter name=\"location\" string=\"true\">Rome</|DSML| parameter>\n"
+ "<|DSML| parameter name=\"unit\" string=\"true\">celsius</|DSML| parameter>\n"
+ "</|DSML| invoke>\n"
+ "</|DSML| calls>")
+
+def reply(rid, text, chunks=1):
+ data = text.encode("utf-8")
+ n = max(1, len(data) // chunks)
+ for i in range(0, len(data), n):
+ part = data[i:i+n]
+ out.write(("DATA %s %d\n" % (rid, len(part))).encode() + part + b"\n"); out.flush()
+ out.write(("DONE %s STAT %d 1.0 50.0 10.0 42 0\n" % (rid, len(text.split()))).encode())
+ out.flush()
+
+while True:
+ line = inp.readline()
+ if not line: break
+ f = line.decode().strip().split()
+ if not f or f[0] != "SUBMIT": continue
+ rid, plen = f[1], int(f[3])
+ prompt = inp.read(plen).decode("utf-8", "replace"); inp.read(1)
+ with open(os.environ["MOCK_LOG"], "a") as log:
+ log.write(prompt + "\n\x00\n")
+ if "<tool_result>" in prompt:
+ reply(rid, "25 degrees and sunny in Rome.")
+ elif "weather in Rome" in prompt:
+ reply(rid, DSML, chunks=12) # split across chunks: suppression must hold
+ else:
+ reply(rid, "Hello from the mock V4.1 engine.")
+'''
+
+TOOLS = [{"type": "function", "function": {
+ "name": "get_weather",
+ "description": "Current weather for a city",
+ "parameters": {"type": "object",
+ "properties": {"location": {"type": "string"}, "unit": {"type": "string"}},
+ "required": ["location"]}}}]
+
+
[email protected](os.name == "posix",
+ "the mock engine is a shebang script the gateway execs directly")
+class ToolCallingV41E2E(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.tmp = tempfile.TemporaryDirectory()
+ (Path(cls.tmp.name) / "config.json").write_text(
+ json.dumps({"model_type": "deepseek_v41"}), encoding="utf-8")
+ mock = Path(cls.tmp.name) / "mock_engine.py"
+ mock.write_text(MOCK_ENGINE)
+ mock.chmod(0o755)
+ cls.mock_log = Path(cls.tmp.name) / "prompts.log"
+ cls.mock_log.touch()
+ with socket.socket() as probe:
+ probe.bind(("127.0.0.1", 0))
+ cls.port = probe.getsockname()[1]
+ env = dict(os.environ, MOCK_LOG=str(cls.mock_log))
+ env.pop("COLI_API_KEY", None)
+ cls.server = subprocess.Popen(
+ [sys.executable, str(SERVER), "--model", cls.tmp.name,
+ "--engine", str(mock), "--arch", "deepseek_v41", "--port", str(cls.port)],
+ env=env, stderr=subprocess.DEVNULL)
+ cls.base = f"http://127.0.0.1:{cls.port}/v1"
+ for _ in range(100):
+ try:
+ with urllib.request.urlopen(cls.base + "/models", timeout=2):
+ pass
+ return
+ except OSError:
+ if cls.server.poll() is not None:
+ raise RuntimeError("gateway exited during startup")
+ import time
+ time.sleep(0.1)
+ raise RuntimeError("gateway did not come up")
+
+ @classmethod
+ def tearDownClass(cls):
+ cls.server.terminate()
+ cls.server.wait(timeout=5)
+ cls.tmp.cleanup()
+
+ def post(self, body, path="/chat/completions"):
+ req = urllib.request.Request(
+ self.base + path, json.dumps(body).encode(),
+ {"Content-Type": "application/json"})
+ with urllib.request.urlopen(req, timeout=30) as resp:
+ return json.loads(resp.read())
+
+ def model_id(self):
+ with urllib.request.urlopen(self.base + "/models", timeout=5) as resp:
+ return json.loads(resp.read())["data"][0]["id"]
+
+ def test_tool_call_non_stream(self):
+ out = self.post({"model": self.model_id(),
+ "messages": [{"role": "user", "content": "weather in Rome?"}],
+ "tools": TOOLS, "temperature": 0, "max_tokens": 128})
+ choice = out["choices"][0]
+ self.assertEqual(choice["finish_reason"], "tool_calls")
+ calls = choice["message"].get("tool_calls") or []
+ self.assertEqual(len(calls), 1)
+ fn = calls[0]["function"]
+ self.assertEqual(fn["name"], "get_weather")
+ self.assertEqual(json.loads(fn["arguments"]), {"location": "Rome", "unit": "celsius"})
+
+ def test_tool_result_round_trip(self):
+ mid = self.model_id()
+ msgs = [{"role": "user", "content": "weather in Rome?"}]
+ out = self.post({"model": mid, "messages": msgs, "tools": TOOLS,
+ "temperature": 0, "max_tokens": 128})
+ calls = out["choices"][0]["message"]["tool_calls"]
+ msgs.append({"role": "assistant", "content": None, "tool_calls": calls})
+ msgs.append({"role": "tool", "tool_call_id": calls[0]["id"],
+ "content": json.dumps({"temp_c": 25})})
+ out2 = self.post({"model": mid, "messages": msgs, "tools": TOOLS,
+ "temperature": 0, "max_tokens": 128})
+ self.assertIn("25", out2["choices"][0]["message"]["content"] or "")
+ prompts = self.mock_log.read_text()
+ self.assertIn("## Tools", prompts) # DSML declaration
+ self.assertIn("<|DSML| calls>", prompts) # native block syntax
+ self.assertNotIn("<|DSML|calls>", prompts) # V4 spelling, without the space
+ self.assertIn("<tool_result>", prompts) # result merged into user turn
+
+ def test_tool_call_streamed_markers_suppressed(self):
+ req = urllib.request.Request(
+ self.base + "/chat/completions",
+ json.dumps({"model": self.model_id(),
+ "messages": [{"role": "user", "content": "weather in Rome?"}],
+ "tools": TOOLS, "temperature": 0, "max_tokens": 128,
+ "stream": True}).encode(),
+ {"Content-Type": "application/json"})
+ got_calls, leak = False, False
+ with urllib.request.urlopen(req, timeout=30) as resp:
+ for raw in resp:
+ line = raw.decode().strip()
+ if not line.startswith("data: ") or line == "data: [DONE]":
+ continue
+ for choice in json.loads(line[6:])["choices"]:
+ delta = choice.get("delta", {})
+ if delta.get("tool_calls"):
+ got_calls = True
+ elif "DSML" in (delta.get("content") or ""):
+ leak = True
+ self.assertTrue(got_calls, "no tool_calls deltas streamed")
+ self.assertFalse(leak, "DSML markers leaked into streamed content")
+
+ def test_no_tools_plain_text(self):
+ out = self.post({"model": self.model_id(),
+ "messages": [{"role": "user", "content": "hello"}],
+ "temperature": 0, "max_tokens": 64})
+ self.assertEqual(out["choices"][0]["message"]["content"],
+ "Hello from the mock V4.1 engine.")
+
+
+class Dsv41VendorEncodingCases(unittest.TestCase):
+ """The checkpoint's own encoding test cases, rendered by the gateway.
+
+ fixtures/dsv41_encoding_caseN.json and .txt are copies of
+ encoding/tests/test_input_N.json and test_output_N.txt from the
+ DeepSeek-V4.1-Flash reference: a conversation and the exact prompt its
+ encoder produces for it. The gateway renders a GENERATION prompt, so where
+ a case ends on an assistant turn the comparison drops that turn and
+ expects the assistant header plus the thinking cue the reference itself
+ appends after the preceding user turn.
+
+ Two of the five reference cases are not modelled by the gateway and are
+ not copied here: case 3 and case 4 use the `latest_reminder` role and the
+ internal classification `task` tokens, and case 5 carries multimodal
+ content blocks that the gateway expands into image placeholders upstream
+ of the renderer (expand_dsv41_images), not inside it.
+ """
+
+ @staticmethod
+ def _case(number):
+ payload = json.loads((HERE / f"fixtures/dsv41_encoding_case{number}.json").read_text())
+ if isinstance(payload, list):
+ payload = {"messages": payload}
+ vendor = (HERE / f"fixtures/dsv41_encoding_case{number}.txt").read_text()
+ return payload, vendor
+
+ def _assert_case(self, number, reasoning_effort=None):
+ import openai_server
+
+ payload, vendor = self._case(number)
+ messages = payload["messages"]
+ thinking = payload.get("thinking_mode") == "thinking"
+ expected = vendor
+ if messages[-1].get("role") == "assistant":
+ messages = messages[:-1]
+ head = vendor.rfind(ASSISTANT) + len(ASSISTANT)
+ expected = vendor[:head] + ("<think>" if thinking else "</think>")
+ rendered = openai_server.render_chat_dsv41(
+ messages, enable_thinking=thinking, reasoning_effort=reasoning_effort,
+ tools=payload.get("tools"))
+ self.assertEqual(rendered, expected)
+
+ def test_case1_tools_and_tool_result_in_thinking_mode(self):
+ # two tools declared, an assistant turn with a DSML call, a tool result
+ self._assert_case(1)
+
+ def test_case2_plain_multi_turn_chat(self):
+ self._assert_case(2)
+
+ def test_tag_names_carry_the_leading_space(self):
+ import v41_dsml
+
+ self.assertEqual(v41_dsml.TOOL_CALLS_OPEN, "<|DSML| calls>")
+ self.assertEqual(v41_dsml.TOOL_CALL_PREFIX, "<|DSML| invoke")
+ block = v41_dsml.render_tool_calls(
+ [{"type": "function",
+ "function": {"name": "f", "arguments": json.dumps({"a": "x", "b": 2})}}])
+ self.assertIn('<|DSML| parameter name="a" string="true">x', block)
+ self.assertIn('<|DSML| parameter name="b" string="false">2', block)
+ content, calls = v41_dsml.parse_completion_text("said it" + block)
+ self.assertEqual(content, "said it")
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(json.loads(calls[0]["function"]["arguments"]), {"a": "x", "b": 2})
+
+ def test_truncated_block_never_reaches_the_client(self):
+ import openai_server
+
+ openai_server.ARCH = "deepseek_v41"
+ try:
+ content, calls = openai_server.parse_dsv41_tool_calls(
+ "here you go\n\n<|DSML| calls>\n<|DSML| invoke name=\"f\"")
+ finally:
+ openai_server.ARCH = "glm"
+ self.assertEqual(calls, [])
+ self.assertEqual(content, "here you go")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/v41_dsml.py b/c/v41_dsml.py
new file mode 100644
index 00000000..799e64bc
--- /dev/null
+++ b/c/v41_dsml.py
@@ -0,0 +1,341 @@
+"""DeepSeek V4.1 DSML primitives, vendored from the checkpoint reference.
+
+Byte-exact excerpts of ``encoding/encoding.py`` (inside
+``deepseek-ai/DeepSeek-V4.1-Flash``) covering the DSML surfaces the gateway
+speaks: the tool-declaration template, the ``<|DSML| calls>`` block encoding,
+and the strict block parser. Keeping these as verbatim reference code (instead
+of a re-implementation) makes a checkpoint encoding revision a mechanical
+re-vendor + diff, exactly as ``v4_dsml.py`` does for V4.
+
+THE ONE DIFFERENCE THAT BITES: V4.1's DSML tag names carry a LEADING ASCII
+SPACE. V4 wrote ``<|DSML|tool_calls>`` / ``<|DSML|invoke>`` /
+``<|DSML|parameter>``; V4.1 writes ``<|DSML| calls>`` / ``<|DSML| invoke>``
+/ ``<|DSML| parameter>``, and the block name itself changed from
+``tool_calls`` to ``calls``. The space is part of the tag name, not separator
+whitespace: it is inside ``tool_calls_block_name`` and friends below, it is in
+every template, and the parameter regex anchors on it (``^ name="..."``). A
+tag built by concatenating the V4 way parses as a tool name of ``"calls"``
+with no parameters, silently, so the names live in one place here and every
+template interpolates them rather than spelling them out.
+
+What is deliberately NOT vendored:
+
+- ``encode_messages`` / ``render_message`` -- the gateway's turn structure
+ (``render_chat_dsv41``) is authoritative: it always terminates the prompt
+ with an assistant generation cue (the reference leaves a trailing assistant
+ turn uncued), and it models neither ``latest_reminder`` nor the internal
+ classification ``task`` tokens.
+- ``process_image_messages`` -- the gateway expands images itself
+ (``expand_dsv41_images``), because it must hand the engine ViT patches, not
+ a file path.
+- the ``REASONING_EFFORT_*`` block -- ``render_chat_dsv41`` renders the
+ numeric budget line from its own effort table.
+
+Gateway adaptations, each marked ADAPTED below:
+
+- ``tool_calls_from_openai_format`` accepts ``arguments`` as a JSON string OR
+ a dict (messages arrive from arbitrary OpenAI clients), and never asserts.
+- ``parse_completion_text`` wraps the strict reference parser so malformed or
+ truncated DSML degrades to plain content instead of raising, and re-shapes
+ calls to the gateway contract ({"id", "type", "function": {...}}).
+"""
+
+import json
+import re
+import sys
+import uuid
+
+# ============================================================
+# Special tokens and templates (byte-exact with the reference)
+# ============================================================
+
+bos_token: str = "<|begin▁of▁sentence|>"
+eos_token: str = "<|end▁of▁sentence|>"
+thinking_start_token: str = "<think>"
+thinking_end_token: str = "</think>"
+dsml_token: str = "|DSML|"
+
+# The leading spaces are load-bearing: see the module docstring.
+tool_calls_block_name: str = " calls"
+tool_call_tag_name: str = " invoke"
+tool_parameter_tag_name: str = " parameter"
+
+tool_call_template: str = (
+ "<{dsml_token}{tool_call_tag_name} name=\"{name}\">\n{arguments}\n"
+ "</{dsml_token}{tool_call_tag_name}>"
+)
+tool_calls_template = (
+ "<{dsml_token}{tc_block_name}>\n{tool_calls}\n</{dsml_token}{tc_block_name}>"
+)
+
+tool_output_template: str = "<tool_result>{content}</tool_result>"
+
+# Markers that open a tool_calls block in model output (used by the gateway's
+# streaming suppression as well as the parser).
+TOOL_CALLS_OPEN = f"<{dsml_token}{tool_calls_block_name}>" # full opener (streaming suppression)
+TOOL_CALLS_PREFIX = f"<{dsml_token}{tool_calls_block_name}" # reference parse anchor (no '>')
+TOOL_CALL_PREFIX = f"<{dsml_token}{tool_call_tag_name}" # a stray invoke, no calls block
+TOOL_CALLS_BLOCK_START = "\n\n" + TOOL_CALLS_PREFIX # as emitted in model output
+
+TOOLS_TEMPLATE = """## Tools
+
+You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}{tc_block_name}>" block like the following:
+
+<{dsml_token}{tc_block_name}>
+<{dsml_token}{tool_call_tag_name} name="$TOOL_NAME">
+<{dsml_token}{tool_parameter_tag_name} name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</{dsml_token}{tool_parameter_tag_name}>
+...
+</{dsml_token}{tool_call_tag_name}>
+<{dsml_token}{tool_call_tag_name} name="$TOOL_NAME2">
+...
+</{dsml_token}{tool_call_tag_name}>
+</{dsml_token}{tc_block_name}>
+
+String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
+
+If thinking_mode is enabled (triggered by {thinking_start_token}), you MUST output your complete reasoning inside {thinking_start_token}...{thinking_end_token} BEFORE any tool calls or final response.
+
+Otherwise, output directly after {thinking_end_token} with tool calls or final response.
+
+### Available Tool Schemas
+
+{tool_schemas}
+
+You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
+"""
+
+
+# ============================================================
+# Encoding helpers (byte-exact unless marked ADAPTED)
+# ============================================================
+
+def to_json(value):
+ try:
+ return json.dumps(value, ensure_ascii=False)
+ except Exception:
+ return json.dumps(value, ensure_ascii=True)
+
+
+def _split_tool_name(name, namespace=None):
+ """Split a qualified `namespace::name`. ADAPTED: a conflicting namespace is
+ dropped rather than asserted, so one odd client cannot 500 the request."""
+ prefix, separator, bare_name = (name or "").partition("::")
+ if separator:
+ namespace, name = prefix, bare_name
+ return namespace, name
+
+
+def _tool_name_for_encoding(tool):
+ """The wire name: `namespace::name` when the tool carries a namespace."""
+ namespace = tool.get("namespace")
+ if isinstance(namespace, dict):
+ namespace = namespace.get("name")
+ namespace, name = _split_tool_name(tool.get("name") or "", namespace)
+ return name if namespace is None else f"{namespace}::{name}"
+
+
+def tools_from_openai_format(tools):
+ """Unwrap {"type": "function", "function": {...}} and qualify the name."""
+ functions = []
+ for tool in tools or []:
+ function = dict(tool.get("function", tool) if isinstance(tool, dict) else {})
+ if isinstance(tool, dict) and tool.get("namespace") is not None:
+ function["namespace"] = tool["namespace"]
+ function["name"] = _tool_name_for_encoding(function)
+ namespace = function.pop("namespace", None)
+ if isinstance(namespace, dict) and namespace.get("description"):
+ function["description"] = (
+ namespace["description"] + "\n" + (function.get("description") or ""))
+ functions.append(function)
+ return functions
+
+
+def tool_calls_from_openai_format(tool_calls):
+ """ADAPTED: tolerate "arguments" given as a dict, not only a JSON string."""
+ calls = []
+ for tool_call in tool_calls or []:
+ fn = tool_call.get("function", tool_call) if isinstance(tool_call, dict) else {}
+ namespace, name = _split_tool_name(
+ fn.get("name") or "",
+ (tool_call.get("namespace") if isinstance(tool_call, dict) else None)
+ or fn.get("namespace"))
+ args = fn.get("arguments", "{}")
+ if not isinstance(args, str):
+ args = to_json(args)
+ call = {"name": name, "arguments": args}
+ if namespace is not None:
+ call["namespace"] = namespace
+ calls.append(call)
+ return calls
+
+
+def encode_arguments_to_dsml(tool_call):
+ """Encode one call's arguments into V4.1 DSML parameter tags."""
+ p_dsml_template = ('<{dsml_token}{tool_parameter_tag_name} name="{key}" string="{is_str}">'
+ '{value}</{dsml_token}{tool_parameter_tag_name}>')
+ p_dsml_strs = []
+ arguments = tool_call.get("arguments")
+ if not isinstance(arguments, dict):
+ # the reference tolerates JSON strings, including double-encoded ones
+ for _ in range(2):
+ if isinstance(arguments, str):
+ try:
+ arguments = json.loads(arguments)
+ except Exception:
+ break
+ else:
+ break
+ if not isinstance(arguments, dict):
+ arguments = {"arguments": tool_call.get("arguments")}
+ for key, value in arguments.items():
+ p_dsml_strs.append(p_dsml_template.format(
+ dsml_token=dsml_token,
+ tool_parameter_tag_name=tool_parameter_tag_name,
+ key=key,
+ is_str="true" if isinstance(value, str) else "false",
+ value=value if isinstance(value, str) else to_json(value)))
+ return "\n".join(p_dsml_strs)
+
+
+def decode_dsml_to_arguments(tool_name, tool_args):
+ """DSML parameters back to a call dict, arguments as a JSON string."""
+ def _decode_value(key, value, string):
+ if string == "true":
+ value = to_json(value)
+ return f"{to_json(key)}: {value}"
+ tool_args_json = ("{" + ", ".join(
+ [_decode_value(k, v, string=is_str) for k, (v, is_str) in tool_args.items()]) + "}")
+ namespace, name = _split_tool_name(tool_name)
+ tool_call = dict(name=name, arguments=tool_args_json)
+ if namespace is not None:
+ tool_call["namespace"] = namespace
+ return tool_call
+
+
+def render_tools(tools):
+ """The `## Tools` block a V4.1 system message carries. One JSON schema per
+ line, not a JSON array."""
+ tools_json = [to_json(t) for t in tools]
+ return TOOLS_TEMPLATE.format(
+ tool_schemas="\n".join(tools_json),
+ dsml_token=dsml_token,
+ tc_block_name=tool_calls_block_name,
+ tool_call_tag_name=tool_call_tag_name,
+ tool_parameter_tag_name=tool_parameter_tag_name,
+ thinking_start_token=thinking_start_token,
+ thinking_end_token=thinking_end_token,
+ )
+
+
+def render_tool_calls(tool_calls):
+ """OpenAI-format tool_calls -> the DSML block a V4.1 assistant turn carries,
+ including the reference's leading blank line."""
+ calls = tool_calls_from_openai_format(tool_calls)
+ rendered = [
+ tool_call_template.format(dsml_token=dsml_token,
+ tool_call_tag_name=tool_call_tag_name,
+ name=_tool_name_for_encoding(call),
+ arguments=encode_arguments_to_dsml(call))
+ for call in calls
+ ]
+ return "\n\n" + tool_calls_template.format(
+ dsml_token=dsml_token, tool_calls="\n".join(rendered),
+ tc_block_name=tool_calls_block_name)
+
+
+def render_tool_result(content):
+ """One tool message, as it appears inside the following user turn."""
+ return tool_output_template.format(content=content)
+
+
+# ============================================================
+# Parsing -- strict reference core + tolerant gateway wrapper
+# ============================================================
+
+def _read_until_stop(index, text, stop):
+ min_pos = len(text)
+ matched_stop = None
+ for s in stop:
+ pos = text.find(s, index)
+ if pos != -1 and pos < min_pos:
+ min_pos = pos
+ matched_stop = s
+ if matched_stop:
+ return min_pos + len(matched_stop), text[index:min_pos], matched_stop
+ return len(text), text[index:], None
+
+
+def _parse_tool_calls_strict(index, text):
+ """Reference parse of a <|DSML| calls> block. Raises ValueError on malformed."""
+ tool_calls = []
+ stop_token = None
+ tool_calls_end_token = f"</{dsml_token}{tool_calls_block_name}>"
+ tool_call_start_token = f"<{dsml_token}{tool_call_tag_name}"
+ tool_call_end_token = f"</{dsml_token}{tool_call_tag_name}"
+ tool_parameter_start_token = f"<{dsml_token}{tool_parameter_tag_name}"
+ tool_parameter_end_token = f"/{dsml_token}{tool_parameter_tag_name}"
+ while index < len(text):
+ index, sep, stop_token = _read_until_stop(
+ index, text, [tool_call_start_token, tool_calls_end_token])
+ if sep != ">\n":
+ raise ValueError(f"Tool call format error: expected '>\\n' but got '{sep}'")
+ if stop_token == tool_calls_end_token:
+ break
+ if stop_token is None:
+ raise ValueError("Missing special token in tool calls")
+ index, tool_name_content, stop_token = _read_until_stop(
+ index, text, [tool_parameter_start_token, tool_call_end_token])
+ p_tool_name = re.findall(r'^\s*name="(.*?)">\n$', tool_name_content, flags=re.DOTALL)
+ if len(p_tool_name) != 1:
+ raise ValueError(f"Tool name format error: '{tool_name_content}'")
+ tool_name = p_tool_name[0]
+ tool_args = {}
+ while stop_token == tool_parameter_start_token:
+ index, param_content, stop_token = _read_until_stop(
+ index, text, [tool_parameter_end_token])
+ param_kv = re.findall(r'^ name="(.*?)" string="(true|false)">(.*?)<$',
+ param_content, flags=re.DOTALL)
+ if len(param_kv) != 1:
+ raise ValueError(f"Parameter format error: '{param_content}'")
+ param_name, string, param_value = param_kv[0]
+ if param_name in tool_args:
+ raise ValueError(f"Duplicate parameter name: '{param_name}'")
+ tool_args[param_name] = (param_value, string)
+ index, content, stop_token = _read_until_stop(
+ index, text, [tool_parameter_start_token, tool_call_end_token])
+ if content != ">\n":
+ raise ValueError(f"Parameter format error: expected '>\\n' but got '{content}'")
+ tool_calls.append(decode_dsml_to_arguments(tool_name=tool_name, tool_args=tool_args))
+ return index, stop_token, tool_calls
+
+
+def parse_completion_text(text):
+ """ADAPTED: split `content [+ blank line + DSML calls block]` from a
+ finished assistant turn. Never raises.
+
+ Returns (content, tool_calls) where tool_calls use the gateway shape
+ [{"id": "call_...", "type": "function", "function": {"name", "arguments"}}].
+ Malformed or truncated DSML degrades to plain content (logged to stderr);
+ the caller decides whether raw markers may stay visible.
+
+ V4.1 puts no id on the wire, so ids are minted here -- the same contract as
+ every other family, where the id only has to round-trip within one
+ conversation.
+ """
+ cut = text.find(TOOL_CALLS_BLOCK_START)
+ parse_at = cut + len(TOOL_CALLS_BLOCK_START) if cut >= 0 else -1
+ if cut < 0 and text.startswith(TOOL_CALLS_PREFIX):
+ cut, parse_at = 0, len(TOOL_CALLS_PREFIX) # reply opening directly with the block
+ if cut < 0:
+ return text, []
+ try:
+ _index, _stop, calls = _parse_tool_calls_strict(parse_at, text)
+ except ValueError as exc:
+ sys.stderr.write(f"[api] deepseek_v41: DSML tool calls parse failed ({exc}) "
+ "-- treating as plain content\n")
+ sys.stderr.flush()
+ return text, []
+ shaped = [{"id": "call_" + uuid.uuid4().hex[:24], "type": "function",
+ "function": {"name": c["name"], "arguments": c["arguments"]}}
+ for c in calls]
+ return text[:cut], shaped