Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
refactor(v4): vendor the checkpoint's DSML primitives behind #948's tool calling
dcutugno3d1c3bec3535
diff --git a/c/openai_server.py b/c/openai_server.pyindex 68111a95..0c46aa2c 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -19,6 +19,8 @@ import sysimport threadingimport timeimport uuid++import v4_dsml # vendored DeepSeek V4 DSML reference primitivesfrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServerfrom pathlib import Pathfrom urllib.parse import unquote, urlsplit@@ -380,42 +382,8 @@ def parse_tool_calls(reply, tools=None):# <|DSML|parameter name="k" string="true">v</|DSML|parameter>\n</|DSML|invoke>\n</|DSML|tool_calls># preceded by "\n\n". There is no standalone "tool" role: results are <tool_result>{content}# </tool_result> blocks merged into the following user turn. DSML = U+FF5C (|) + ASCII.-DSV4_DSML = "|DSML|"-DSV4_EOS = "<\uff5cend\u2581of\u2581sentence\uff5c>"-_DSV4_BLOCK_RE = re.compile(- r"<" + DSV4_DSML + r"tool_calls>(.*?)</" + DSV4_DSML + r"tool_calls>", re.DOTALL)-_DSV4_INVOKE_RE = re.compile(- r"<" + DSV4_DSML + r"invoke name=\"(.*?)\">(.*?)</" + DSV4_DSML + r"invoke>", re.DOTALL)-_DSV4_PARAM_RE = re.compile(- r"<" + DSV4_DSML + r"parameter name=\"(.*?)\" string=\"(true|false)\">(.*?)</" +- DSV4_DSML + r"parameter>", re.DOTALL)--DSV4_TOOLS_TEMPLATE = """## Tools--You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml}tool_calls>" block like the following:--<{dsml}tool_calls>-<{dsml}invoke name="$TOOL_NAME">-<{dsml}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</{dsml}parameter>-...-</{dsml}invoke>-<{dsml}invoke name="$TOOL_NAME2">-...-</{dsml}invoke>-</{dsml}tool_calls>--String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.--If thinking_mode is enabled (triggered by {think_open}), you MUST output your complete reasoning inside {think_open}...{think_close} BEFORE any tool calls or final response.--Otherwise, output directly after {think_close} with tool calls or final response.--### Available Tool Schemas--{tool_schemas}--You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.-"""+DSV4_DSML = v4_dsml.dsml_token+DSV4_EOS = v4_dsml.eos_token# OpenAI-style reasoning_effort levels -> the V4 vocabulary (encoding_dsv4.py has only# low/high/max; `low` adds nothing). In thinking mode the level's prompt is prepended at the@@ -438,75 +406,41 @@ DSV4_REASONING_EFFORT_PROMPTS = {def _dsv4_tools_block(tools):- """V4 tool-declaration block (byte-matches encoding_dsv4.py's TOOLS_TEMPLATE)."""+ """V4 tool-declaration block, rendered by the vendored reference template."""schemas = []for tool in (tools or []):fn = tool.get("function", tool) if isinstance(tool, dict) else {}- clean = {k: v for k, v in fn.items() if k not in ("defer_loading", "strict")}- schemas.append(json.dumps(clean, ensure_ascii=False))- return DSV4_TOOLS_TEMPLATE.format(dsml=DSV4_DSML, think_open=THINK_OPEN,- think_close=THINK_CLOSE, tool_schemas="\n".join(schemas))+ # Gateway-side scrub: OpenAI clients attach routing hints the model+ # schema must not carry.+ schemas.append({k: v for k, v in fn.items() if k not in ("defer_loading", "strict")})+ return v4_dsml.render_tools(schemas)def _dsv4_tool_calls(tool_calls):- """Render OpenAI-format tool_calls into a V4 DSML block (incl. the leading \\n\\n)."""- calls = []- for tc in tool_calls:- fn = tc.get("function", tc) if isinstance(tc, dict) else {}- name = fn.get("name") or ""- args = fn.get("arguments")- try:- arguments = json.loads(args) if isinstance(args, str) else (args or {})- except (json.JSONDecodeError, TypeError, ValueError):- arguments = {}- params = []- for key, value in (arguments or {}).items():- is_str = "true" if isinstance(value, str) else "false"- text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)- params.append('<%sparameter name="%s" string="%s">%s</%sparameter>'- % (DSV4_DSML, key, is_str, text, DSV4_DSML))- calls.append('<%sinvoke name="%s">\n%s\n</%sinvoke>'- % (DSV4_DSML, name, "\n".join(params), DSV4_DSML))- return '\n\n<%stool_calls>\n%s\n</%stool_calls>' % (DSV4_DSML, "\n".join(calls), DSV4_DSML)+ """Render OpenAI-format tool_calls into a V4 DSML block (incl. the leading++)."""+ return v4_dsml.render_tool_calls(tool_calls)def parse_dsv4_tool_calls(reply):"""Parse DeepSeek V4 DSML tool calls out of one assistant reply.- Returns (content, tool_calls): the visible answer text with the DSML block (and any- thinking/eos markers) removed, plus OpenAI-format tool_calls. Tolerant of truncated- output: an incomplete block yields no calls rather than an error.+ The block itself is decoded by the vendored reference parser (strict, so a+ malformed block degrades to no calls instead of half-parsed arguments).+ Gateway hardening on top: an incomplete block (e.g. length-truncated+ output) is cut from the visible content so raw DSML syntax never leaks,+ and any thinking/eos markers around the block are scrubbed."""- content, calls = reply, []- block = _DSV4_BLOCK_RE.search(reply)- if block:- content = reply[:block.start()].rstrip("\n")- for m in _DSV4_INVOKE_RE.finditer(block.group(1)):- name = m.group(1).strip()- if not name:- continue- arguments = {}- for p in _DSV4_PARAM_RE.finditer(m.group(2)):- key, is_str, value = p.group(1), p.group(2), p.group(3)- if is_str != "true":- try:- value = json.loads(value)- except (json.JSONDecodeError, TypeError, ValueError):- pass- arguments[key] = value- calls.append({"id": "call_" + uuid.uuid4().hex[:24], "type": "function",- "function": {"name": name,- "arguments": json.dumps(arguments, ensure_ascii=False)}})- else:- # No complete block (e.g. length-truncated output): drop everything from the first- # V4 tool marker so raw DSML syntax never leaks into the visible content.- cut = len(reply)+ content, calls = v4_dsml.parse_completion_text(reply)+ if not calls:+ cut = len(content)for marker in ("<" + DSV4_DSML + "tool_calls", "<" + DSV4_DSML + "invoke"):- pos = reply.find(marker)- if pos >= 0 and pos < cut:+ pos = content.find(marker)+ if 0 <= pos < cut:cut = pos- if cut < len(reply):- content = reply[:cut]+ if cut < len(content):+ content = content[:cut]for marker in (DSV4_EOS, THINK_OPEN, THINK_CLOSE):content = content.replace(marker, "")return content.strip(), callsdiff --git a/c/v4_dsml.py b/c/v4_dsml.pynew file mode 100644index 00000000..8e135090--- /dev/null+++ b/c/v4_dsml.py@@ -0,0 +1,249 @@+"""DeepSeek V4 DSML primitives, vendored from the checkpoint reference.++Byte-exact excerpts of ``encoding/encoding_dsv4.py`` (inside+``deepseek-ai/DeepSeek-V4-Flash-0731``) covering the DSML surfaces the gateway+speaks: the tool-declaration template, the ``<|DSML|tool_calls>`` block+encoding, and the strict block parser. Keeping these as verbatim reference+code (instead of a re-implementation) makes a checkpoint encoding revision a+mechanical re-vendor + diff.++What is deliberately NOT vendored:++- ``encode_messages`` / ``render_message`` — the gateway's turn structure+ (``render_chat_v4``) is authoritative: it always terminates the prompt with+ an assistant generation cue (the reference leaves a trailing assistant+ ``tool_calls`` turn uncued), renders ``reasoning_content`` for every turn+ that carries one, and merges ``tool`` messages itself.+- ``REASONING_EFFORT_PROMPTS`` — the gateway ships tuned effort prompts with+ an explicit token budget; the reference's unbounded "no shortcuts+ permitted" variants routinely consumed the whole completion budget with+ reasoning.++Gateway adaptations, each marked ADAPTED below:++- ``tool_calls_from_openai_format`` accepts ``arguments`` as a JSON string OR+ a dict (messages arrive from arbitrary OpenAI clients).+- ``parse_completion_text`` wraps the strict reference parser so malformed or+ truncated DSML degrades to plain content instead of raising, and re-shapes+ calls to the gateway contract ({"id", "type", "function": {...}}).+"""++import json+import re+import sys+import uuid++# ============================================================+# Special tokens and templates (byte-exact with the reference)+# ============================================================++bos_token: str = "<|begin▁of▁sentence|>"+eos_token: str = "<|end▁of▁sentence|>"+thinking_start_token: str = "<think>"+thinking_end_token: str = "</think>"+dsml_token: str = "|DSML|"++tool_call_template: str = (+ "<{dsml_token}invoke name=\"{name}\">\n{arguments}\n</{dsml_token}invoke>"+)+tool_calls_template = (+ "<{dsml_token}{tc_block_name}>\n{tool_calls}\n</{dsml_token}{tc_block_name}>"+)+tool_calls_block_name: str = "tool_calls"++tool_output_template: str = "<tool_result>{content}</tool_result>"++# Marker that opens a tool_calls block in model output (used by the gateway's+# streaming suppression as well as the parser).+TOOL_CALLS_OPEN = f"<{dsml_token}{tool_calls_block_name}>" # full opener (streaming suppression)+TOOL_CALLS_PREFIX = f"<{dsml_token}{tool_calls_block_name}" # reference parse anchor (no '>')+TOOL_CALLS_BLOCK_START = "\n\n" + TOOL_CALLS_PREFIX # as emitted in model output++TOOLS_TEMPLATE = """## Tools++You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}tool_calls>" block like the following:++<{dsml_token}tool_calls>+<{dsml_token}invoke name="$TOOL_NAME">+<{dsml_token}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</{dsml_token}parameter>+...+</{dsml_token}invoke>+<{dsml_token}invoke name="$TOOL_NAME2">+...+</{dsml_token}invoke>+</{dsml_token}tool_calls>++String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.++If thinking_mode is enabled (triggered by {thinking_start_token}), you MUST output your complete reasoning inside {thinking_start_token}...{thinking_end_token} BEFORE any tool calls or final response.++Otherwise, output directly after {thinking_end_token} with tool calls or final response.++### Available Tool Schemas++{tool_schemas}++You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.+"""++# ============================================================+# Encoding helpers (byte-exact unless marked ADAPTED)+# ============================================================++def to_json(value):+ try:+ return json.dumps(value, ensure_ascii=False)+ except Exception:+ return json.dumps(value, ensure_ascii=True)+++def tools_from_openai_format(tools):+ return [tool.get("function", tool) for tool in tools]+++def tool_calls_from_openai_format(tool_calls):+ """ADAPTED: tolerate "arguments" given as a dict, not only a JSON string."""+ out = []+ for tool_call in tool_calls:+ fn = tool_call.get("function", tool_call) if isinstance(tool_call, dict) else {}+ args = fn.get("arguments", "{}")+ if not isinstance(args, str):+ args = to_json(args)+ out.append({"name": fn.get("name"), "arguments": args})+ return out+++def encode_arguments_to_dsml(tool_call):+ p_dsml_template = '<{dsml_token}parameter name="{key}" string="{is_str}">{value}</{dsml_token}parameter>'+ p_dsml_strs = []+ try:+ arguments = json.loads(tool_call["arguments"])+ except Exception:+ arguments = {"arguments": tool_call["arguments"]}+ if not isinstance(arguments, dict):+ arguments = {"arguments": arguments}+ for k, v in arguments.items():+ p_dsml_strs.append(p_dsml_template.format(+ dsml_token=dsml_token, key=k,+ is_str="true" if isinstance(v, str) else "false",+ value=v if isinstance(v, str) else to_json(v)))+ return "\n".join(p_dsml_strs)+++def decode_dsml_to_arguments(tool_name, tool_args):+ def _decode_value(key, value, string):+ if string == "true":+ value = to_json(value)+ return f"{to_json(key)}: {value}"+ tool_args_json = ("{" + ", ".join(+ [_decode_value(k, v, string=is_str) for k, (v, is_str) in tool_args.items()]) + "}")+ return dict(name=tool_name, arguments=tool_args_json)+++def render_tools(tools):+ tools_json = [to_json(t) for t in tools]+ return TOOLS_TEMPLATE.format(+ tool_schemas="\n".join(tools_json),+ dsml_token=dsml_token,+ thinking_start_token=thinking_start_token,+ thinking_end_token=thinking_end_token,+ )+++def render_tool_calls(tool_calls):+ """OpenAI-format tool_calls -> the DSML block a V4 assistant turn carries,+ including the reference's leading blank line."""+ calls = tool_calls_from_openai_format(tool_calls)+ rendered = [+ tool_call_template.format(dsml_token=dsml_token,+ name=tc.get("name") or "",+ arguments=encode_arguments_to_dsml(tc))+ for tc in calls+ ]+ return "\n\n" + tool_calls_template.format(+ dsml_token=dsml_token, tool_calls="\n".join(rendered),+ tc_block_name=tool_calls_block_name)+++# ============================================================+# Parsing — strict reference core + tolerant gateway wrapper+# ============================================================++def _read_until_stop(index, text, stop):+ min_pos = len(text)+ matched_stop = None+ for s in stop:+ pos = text.find(s, index)+ if pos != -1 and pos < min_pos:+ min_pos = pos+ matched_stop = s+ if matched_stop:+ return min_pos + len(matched_stop), text[index:min_pos], matched_stop+ return len(text), text[index:], None+++def _parse_tool_calls_strict(index, text):+ """Reference parse of a <dsml>tool_calls block. Raises ValueError on malformed."""+ tool_calls = []+ tool_calls_end_token = f"</{dsml_token}{tool_calls_block_name}>"+ while index < len(text):+ index, sep, stop_token = _read_until_stop(+ index, text, [f"<{dsml_token}invoke", tool_calls_end_token])+ if sep != ">\n":+ raise ValueError(f"Tool call format error: expected '>\\n' but got '{sep}'")+ if stop_token == tool_calls_end_token:+ break+ if stop_token is None:+ raise ValueError("Missing special token in tool calls")+ index, tool_name_content, stop_token = _read_until_stop(+ index, text, [f"<{dsml_token}parameter", f"</{dsml_token}invoke"])+ p_tool_name = re.findall(r'^\s*name="(.*?)">\n$', tool_name_content, flags=re.DOTALL)+ if len(p_tool_name) != 1:+ raise ValueError(f"Tool name format error: '{tool_name_content}'")+ tool_name = p_tool_name[0]+ tool_args = {}+ while stop_token == f"<{dsml_token}parameter":+ index, param_content, stop_token = _read_until_stop(+ index, text, [f"/{dsml_token}parameter"])+ param_kv = re.findall(r'^ name="(.*?)" string="(true|false)">(.*?)<$',+ param_content, flags=re.DOTALL)+ if len(param_kv) != 1:+ raise ValueError(f"Parameter format error: '{param_content}'")+ param_name, string, param_value = param_kv[0]+ if param_name in tool_args:+ raise ValueError(f"Duplicate parameter name: '{param_name}'")+ tool_args[param_name] = (param_value, string)+ index, content, stop_token = _read_until_stop(+ index, text, [f"<{dsml_token}parameter", f"</{dsml_token}invoke"])+ if content != ">\n":+ raise ValueError(f"Parameter format error: expected '>\\n' but got '{content}'")+ tool_calls.append(decode_dsml_to_arguments(tool_name=tool_name, tool_args=tool_args))+ return index, stop_token, tool_calls+++def parse_completion_text(text):+ """ADAPTED: split `content [+ blank line + DSML tool_calls block]` from a+ finished assistant turn. Never raises.++ Returns (content, tool_calls) where tool_calls use the gateway shape+ [{"id": "call_...", "type": "function", "function": {"name", "arguments"}}].+ Malformed or truncated DSML degrades to plain content (logged to stderr);+ the caller decides whether raw markers may stay visible.+ """+ cut = text.find(TOOL_CALLS_BLOCK_START)+ parse_at = cut + len(TOOL_CALLS_BLOCK_START) if cut >= 0 else -1+ if cut < 0 and text.startswith(TOOL_CALLS_PREFIX):+ cut, parse_at = 0, len(TOOL_CALLS_PREFIX) # reply opening directly with the block+ if cut < 0:+ return text, []+ try:+ _index, _stop, calls = _parse_tool_calls_strict(parse_at, text)+ except ValueError as exc:+ sys.stderr.write(f"[api] deepseek_v4: DSML tool_calls parse failed ({exc}) "+ "-- treating as plain content\n")+ sys.stderr.flush()+ return text, []+ shaped = [{"id": "call_" + uuid.uuid4().hex[:24], "type": "function",+ "function": {"name": c["name"], "arguments": c["arguments"]}}+ for c in calls]+ return text[:cut], shaped