Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

refactor(v4): vendor the checkpoint's DSML primitives behind #948's tool calling

dcutugno3d1c3bec3535
diff --git a/c/openai_server.py b/c/openai_server.py
index 68111a95..0c46aa2c 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -19,6 +19,8 @@ import sys
import threading
import time
import uuid
+
+import v4_dsml # vendored DeepSeek V4 DSML reference primitives
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import unquote, urlsplit
@@ -380,42 +382,8 @@ def parse_tool_calls(reply, tools=None):
# <|DSML|parameter name="k" string="true">v</|DSML|parameter>\n</|DSML|invoke>\n</|DSML|tool_calls>
# preceded by "\n\n". There is no standalone "tool" role: results are <tool_result>{content}
# </tool_result> blocks merged into the following user turn. DSML = U+FF5C (|) + ASCII.
-DSV4_DSML = "|DSML|"
-DSV4_EOS = "<\uff5cend\u2581of\u2581sentence\uff5c>"
-_DSV4_BLOCK_RE = re.compile(
- r"<" + DSV4_DSML + r"tool_calls>(.*?)</" + DSV4_DSML + r"tool_calls>", re.DOTALL)
-_DSV4_INVOKE_RE = re.compile(
- r"<" + DSV4_DSML + r"invoke name=\"(.*?)\">(.*?)</" + DSV4_DSML + r"invoke>", re.DOTALL)
-_DSV4_PARAM_RE = re.compile(
- r"<" + DSV4_DSML + r"parameter name=\"(.*?)\" string=\"(true|false)\">(.*?)</" +
- DSV4_DSML + r"parameter>", re.DOTALL)
-
-DSV4_TOOLS_TEMPLATE = """## Tools
-
-You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml}tool_calls>" block like the following:
-
-<{dsml}tool_calls>
-<{dsml}invoke name="$TOOL_NAME">
-<{dsml}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</{dsml}parameter>
-...
-</{dsml}invoke>
-<{dsml}invoke name="$TOOL_NAME2">
-...
-</{dsml}invoke>
-</{dsml}tool_calls>
-
-String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
-
-If thinking_mode is enabled (triggered by {think_open}), you MUST output your complete reasoning inside {think_open}...{think_close} BEFORE any tool calls or final response.
-
-Otherwise, output directly after {think_close} with tool calls or final response.
-
-### Available Tool Schemas
-
-{tool_schemas}
-
-You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
-"""
+DSV4_DSML = v4_dsml.dsml_token
+DSV4_EOS = v4_dsml.eos_token
# OpenAI-style reasoning_effort levels -> the V4 vocabulary (encoding_dsv4.py has only
# low/high/max; `low` adds nothing). In thinking mode the level's prompt is prepended at the
@@ -438,75 +406,41 @@ DSV4_REASONING_EFFORT_PROMPTS = {
def _dsv4_tools_block(tools):
- """V4 tool-declaration block (byte-matches encoding_dsv4.py's TOOLS_TEMPLATE)."""
+ """V4 tool-declaration block, rendered by the vendored reference template."""
schemas = []
for tool in (tools or []):
fn = tool.get("function", tool) if isinstance(tool, dict) else {}
- clean = {k: v for k, v in fn.items() if k not in ("defer_loading", "strict")}
- schemas.append(json.dumps(clean, ensure_ascii=False))
- return DSV4_TOOLS_TEMPLATE.format(dsml=DSV4_DSML, think_open=THINK_OPEN,
- think_close=THINK_CLOSE, tool_schemas="\n".join(schemas))
+ # Gateway-side scrub: OpenAI clients attach routing hints the model
+ # schema must not carry.
+ schemas.append({k: v for k, v in fn.items() if k not in ("defer_loading", "strict")})
+ return v4_dsml.render_tools(schemas)
def _dsv4_tool_calls(tool_calls):
- """Render OpenAI-format tool_calls into a V4 DSML block (incl. the leading \\n\\n)."""
- calls = []
- for tc in tool_calls:
- fn = tc.get("function", tc) if isinstance(tc, dict) else {}
- name = fn.get("name") or ""
- args = fn.get("arguments")
- try:
- arguments = json.loads(args) if isinstance(args, str) else (args or {})
- except (json.JSONDecodeError, TypeError, ValueError):
- arguments = {}
- params = []
- for key, value in (arguments or {}).items():
- is_str = "true" if isinstance(value, str) else "false"
- text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
- params.append('<%sparameter name="%s" string="%s">%s</%sparameter>'
- % (DSV4_DSML, key, is_str, text, DSV4_DSML))
- calls.append('<%sinvoke name="%s">\n%s\n</%sinvoke>'
- % (DSV4_DSML, name, "\n".join(params), DSV4_DSML))
- return '\n\n<%stool_calls>\n%s\n</%stool_calls>' % (DSV4_DSML, "\n".join(calls), DSV4_DSML)
+ """Render OpenAI-format tool_calls into a V4 DSML block (incl. the leading
+
+)."""
+ return v4_dsml.render_tool_calls(tool_calls)
def parse_dsv4_tool_calls(reply):
"""Parse DeepSeek V4 DSML tool calls out of one assistant reply.
- Returns (content, tool_calls): the visible answer text with the DSML block (and any
- thinking/eos markers) removed, plus OpenAI-format tool_calls. Tolerant of truncated
- output: an incomplete block yields no calls rather than an error.
+ The block itself is decoded by the vendored reference parser (strict, so a
+ malformed block degrades to no calls instead of half-parsed arguments).
+ Gateway hardening on top: an incomplete block (e.g. length-truncated
+ output) is cut from the visible content so raw DSML syntax never leaks,
+ and any thinking/eos markers around the block are scrubbed.
"""
- content, calls = reply, []
- block = _DSV4_BLOCK_RE.search(reply)
- if block:
- content = reply[:block.start()].rstrip("\n")
- for m in _DSV4_INVOKE_RE.finditer(block.group(1)):
- name = m.group(1).strip()
- if not name:
- continue
- arguments = {}
- for p in _DSV4_PARAM_RE.finditer(m.group(2)):
- key, is_str, value = p.group(1), p.group(2), p.group(3)
- if is_str != "true":
- try:
- value = json.loads(value)
- except (json.JSONDecodeError, TypeError, ValueError):
- pass
- arguments[key] = value
- calls.append({"id": "call_" + uuid.uuid4().hex[:24], "type": "function",
- "function": {"name": name,
- "arguments": json.dumps(arguments, ensure_ascii=False)}})
- else:
- # No complete block (e.g. length-truncated output): drop everything from the first
- # V4 tool marker so raw DSML syntax never leaks into the visible content.
- cut = len(reply)
+ content, calls = v4_dsml.parse_completion_text(reply)
+ if not calls:
+ cut = len(content)
for marker in ("<" + DSV4_DSML + "tool_calls", "<" + DSV4_DSML + "invoke"):
- pos = reply.find(marker)
- if pos >= 0 and pos < cut:
+ pos = content.find(marker)
+ if 0 <= pos < cut:
cut = pos
- if cut < len(reply):
- content = reply[:cut]
+ if cut < len(content):
+ content = content[:cut]
for marker in (DSV4_EOS, THINK_OPEN, THINK_CLOSE):
content = content.replace(marker, "")
return content.strip(), calls
diff --git a/c/v4_dsml.py b/c/v4_dsml.py
new file mode 100644
index 00000000..8e135090
--- /dev/null
+++ b/c/v4_dsml.py
@@ -0,0 +1,249 @@
+"""DeepSeek V4 DSML primitives, vendored from the checkpoint reference.
+
+Byte-exact excerpts of ``encoding/encoding_dsv4.py`` (inside
+``deepseek-ai/DeepSeek-V4-Flash-0731``) covering the DSML surfaces the gateway
+speaks: the tool-declaration template, the ``<|DSML|tool_calls>`` block
+encoding, and the strict block parser. Keeping these as verbatim reference
+code (instead of a re-implementation) makes a checkpoint encoding revision a
+mechanical re-vendor + diff.
+
+What is deliberately NOT vendored:
+
+- ``encode_messages`` / ``render_message`` — the gateway's turn structure
+ (``render_chat_v4``) is authoritative: it always terminates the prompt with
+ an assistant generation cue (the reference leaves a trailing assistant
+ ``tool_calls`` turn uncued), renders ``reasoning_content`` for every turn
+ that carries one, and merges ``tool`` messages itself.
+- ``REASONING_EFFORT_PROMPTS`` — the gateway ships tuned effort prompts with
+ an explicit token budget; the reference's unbounded "no shortcuts
+ permitted" variants routinely consumed the whole completion budget with
+ reasoning.
+
+Gateway adaptations, each marked ADAPTED below:
+
+- ``tool_calls_from_openai_format`` accepts ``arguments`` as a JSON string OR
+ a dict (messages arrive from arbitrary OpenAI clients).
+- ``parse_completion_text`` wraps the strict reference parser so malformed or
+ truncated DSML degrades to plain content instead of raising, and re-shapes
+ calls to the gateway contract ({"id", "type", "function": {...}}).
+"""
+
+import json
+import re
+import sys
+import uuid
+
+# ============================================================
+# Special tokens and templates (byte-exact with the reference)
+# ============================================================
+
+bos_token: str = "<|begin▁of▁sentence|>"
+eos_token: str = "<|end▁of▁sentence|>"
+thinking_start_token: str = "<think>"
+thinking_end_token: str = "</think>"
+dsml_token: str = "|DSML|"
+
+tool_call_template: str = (
+ "<{dsml_token}invoke name=\"{name}\">\n{arguments}\n</{dsml_token}invoke>"
+)
+tool_calls_template = (
+ "<{dsml_token}{tc_block_name}>\n{tool_calls}\n</{dsml_token}{tc_block_name}>"
+)
+tool_calls_block_name: str = "tool_calls"
+
+tool_output_template: str = "<tool_result>{content}</tool_result>"
+
+# Marker that opens a tool_calls block in model output (used by the gateway's
+# streaming suppression as well as the parser).
+TOOL_CALLS_OPEN = f"<{dsml_token}{tool_calls_block_name}>" # full opener (streaming suppression)
+TOOL_CALLS_PREFIX = f"<{dsml_token}{tool_calls_block_name}" # reference parse anchor (no '>')
+TOOL_CALLS_BLOCK_START = "\n\n" + TOOL_CALLS_PREFIX # as emitted in model output
+
+TOOLS_TEMPLATE = """## Tools
+
+You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}tool_calls>" block like the following:
+
+<{dsml_token}tool_calls>
+<{dsml_token}invoke name="$TOOL_NAME">
+<{dsml_token}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</{dsml_token}parameter>
+...
+</{dsml_token}invoke>
+<{dsml_token}invoke name="$TOOL_NAME2">
+...
+</{dsml_token}invoke>
+</{dsml_token}tool_calls>
+
+String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
+
+If thinking_mode is enabled (triggered by {thinking_start_token}), you MUST output your complete reasoning inside {thinking_start_token}...{thinking_end_token} BEFORE any tool calls or final response.
+
+Otherwise, output directly after {thinking_end_token} with tool calls or final response.
+
+### Available Tool Schemas
+
+{tool_schemas}
+
+You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
+"""
+
+# ============================================================
+# Encoding helpers (byte-exact unless marked ADAPTED)
+# ============================================================
+
+def to_json(value):
+ try:
+ return json.dumps(value, ensure_ascii=False)
+ except Exception:
+ return json.dumps(value, ensure_ascii=True)
+
+
+def tools_from_openai_format(tools):
+ return [tool.get("function", tool) for tool in tools]
+
+
+def tool_calls_from_openai_format(tool_calls):
+ """ADAPTED: tolerate "arguments" given as a dict, not only a JSON string."""
+ out = []
+ for tool_call in tool_calls:
+ fn = tool_call.get("function", tool_call) if isinstance(tool_call, dict) else {}
+ args = fn.get("arguments", "{}")
+ if not isinstance(args, str):
+ args = to_json(args)
+ out.append({"name": fn.get("name"), "arguments": args})
+ return out
+
+
+def encode_arguments_to_dsml(tool_call):
+ p_dsml_template = '<{dsml_token}parameter name="{key}" string="{is_str}">{value}</{dsml_token}parameter>'
+ p_dsml_strs = []
+ try:
+ arguments = json.loads(tool_call["arguments"])
+ except Exception:
+ arguments = {"arguments": tool_call["arguments"]}
+ if not isinstance(arguments, dict):
+ arguments = {"arguments": arguments}
+ for k, v in arguments.items():
+ p_dsml_strs.append(p_dsml_template.format(
+ dsml_token=dsml_token, key=k,
+ is_str="true" if isinstance(v, str) else "false",
+ value=v if isinstance(v, str) else to_json(v)))
+ return "\n".join(p_dsml_strs)
+
+
+def decode_dsml_to_arguments(tool_name, tool_args):
+ def _decode_value(key, value, string):
+ if string == "true":
+ value = to_json(value)
+ return f"{to_json(key)}: {value}"
+ tool_args_json = ("{" + ", ".join(
+ [_decode_value(k, v, string=is_str) for k, (v, is_str) in tool_args.items()]) + "}")
+ return dict(name=tool_name, arguments=tool_args_json)
+
+
+def render_tools(tools):
+ tools_json = [to_json(t) for t in tools]
+ return TOOLS_TEMPLATE.format(
+ tool_schemas="\n".join(tools_json),
+ dsml_token=dsml_token,
+ thinking_start_token=thinking_start_token,
+ thinking_end_token=thinking_end_token,
+ )
+
+
+def render_tool_calls(tool_calls):
+ """OpenAI-format tool_calls -> the DSML block a V4 assistant turn carries,
+ including the reference's leading blank line."""
+ calls = tool_calls_from_openai_format(tool_calls)
+ rendered = [
+ tool_call_template.format(dsml_token=dsml_token,
+ name=tc.get("name") or "",
+ arguments=encode_arguments_to_dsml(tc))
+ for tc in calls
+ ]
+ return "\n\n" + tool_calls_template.format(
+ dsml_token=dsml_token, tool_calls="\n".join(rendered),
+ tc_block_name=tool_calls_block_name)
+
+
+# ============================================================
+# Parsing — strict reference core + tolerant gateway wrapper
+# ============================================================
+
+def _read_until_stop(index, text, stop):
+ min_pos = len(text)
+ matched_stop = None
+ for s in stop:
+ pos = text.find(s, index)
+ if pos != -1 and pos < min_pos:
+ min_pos = pos
+ matched_stop = s
+ if matched_stop:
+ return min_pos + len(matched_stop), text[index:min_pos], matched_stop
+ return len(text), text[index:], None
+
+
+def _parse_tool_calls_strict(index, text):
+ """Reference parse of a <dsml>tool_calls block. Raises ValueError on malformed."""
+ tool_calls = []
+ tool_calls_end_token = f"</{dsml_token}{tool_calls_block_name}>"
+ while index < len(text):
+ index, sep, stop_token = _read_until_stop(
+ index, text, [f"<{dsml_token}invoke", tool_calls_end_token])
+ if sep != ">\n":
+ raise ValueError(f"Tool call format error: expected '>\\n' but got '{sep}'")
+ if stop_token == tool_calls_end_token:
+ break
+ if stop_token is None:
+ raise ValueError("Missing special token in tool calls")
+ index, tool_name_content, stop_token = _read_until_stop(
+ index, text, [f"<{dsml_token}parameter", f"</{dsml_token}invoke"])
+ p_tool_name = re.findall(r'^\s*name="(.*?)">\n$', tool_name_content, flags=re.DOTALL)
+ if len(p_tool_name) != 1:
+ raise ValueError(f"Tool name format error: '{tool_name_content}'")
+ tool_name = p_tool_name[0]
+ tool_args = {}
+ while stop_token == f"<{dsml_token}parameter":
+ index, param_content, stop_token = _read_until_stop(
+ index, text, [f"/{dsml_token}parameter"])
+ param_kv = re.findall(r'^ name="(.*?)" string="(true|false)">(.*?)<$',
+ param_content, flags=re.DOTALL)
+ if len(param_kv) != 1:
+ raise ValueError(f"Parameter format error: '{param_content}'")
+ param_name, string, param_value = param_kv[0]
+ if param_name in tool_args:
+ raise ValueError(f"Duplicate parameter name: '{param_name}'")
+ tool_args[param_name] = (param_value, string)
+ index, content, stop_token = _read_until_stop(
+ index, text, [f"<{dsml_token}parameter", f"</{dsml_token}invoke"])
+ if content != ">\n":
+ raise ValueError(f"Parameter format error: expected '>\\n' but got '{content}'")
+ tool_calls.append(decode_dsml_to_arguments(tool_name=tool_name, tool_args=tool_args))
+ return index, stop_token, tool_calls
+
+
+def parse_completion_text(text):
+ """ADAPTED: split `content [+ blank line + DSML tool_calls block]` from a
+ finished assistant turn. Never raises.
+
+ Returns (content, tool_calls) where tool_calls use the gateway shape
+ [{"id": "call_...", "type": "function", "function": {"name", "arguments"}}].
+ Malformed or truncated DSML degrades to plain content (logged to stderr);
+ the caller decides whether raw markers may stay visible.
+ """
+ cut = text.find(TOOL_CALLS_BLOCK_START)
+ parse_at = cut + len(TOOL_CALLS_BLOCK_START) if cut >= 0 else -1
+ if cut < 0 and text.startswith(TOOL_CALLS_PREFIX):
+ cut, parse_at = 0, len(TOOL_CALLS_PREFIX) # reply opening directly with the block
+ if cut < 0:
+ return text, []
+ try:
+ _index, _stop, calls = _parse_tool_calls_strict(parse_at, text)
+ except ValueError as exc:
+ sys.stderr.write(f"[api] deepseek_v4: DSML tool_calls parse failed ({exc}) "
+ "-- treating as plain content\n")
+ sys.stderr.flush()
+ return text, []
+ shaped = [{"id": "call_" + uuid.uuid4().hex[:24], "type": "function",
+ "function": {"name": c["name"], "arguments": c["arguments"]}}
+ for c in calls]
+ return text[:cut], shaped