Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

serve: DeepSeek-V4.1 through the server (deepseek-v3 pre-tokenizer, DSML tool calls, effort words, end token, card sampling)

Niko12215a52007005a0
diff --git a/serve/dsml.py b/serve/dsml.py
new file mode 100644
index 000000000..c4a1166a5
--- /dev/null
+++ b/serve/dsml.py
@@ -0,0 +1,212 @@
+"""serve/dsml.py - DeepSeek-V4.1's tool-call text (DSML) for the server.
+
+DeepSeek-V4.1 writes a tool call as (the model's own chat template, the same bytes as DeepSeek's encoding.py)
+
+ <|DSML| calls>
+ <|DSML| invoke name="get_weather">
+ <|DSML| parameter name="city" string="true">Zagreb</|DSML| parameter>
+ <|DSML| parameter name="days" string="false">3</|DSML| parameter>
+ </|DSML| invoke>
+ </|DSML| calls>
+
+`|DSML|` is one token (id 128825, a user-defined one, so it is also text when a request is tokenized). String
+values are written as they are; every other type is JSON (string="false").
+
+The server's OutputParser (serve/frontend.py) reads the Qwen form `<tool_call><function=NAME><parameter=P>...`, with
+its streaming of tool arguments, its calls inside the thinking and its recovery rules. DsmlTranslator rewrites the
+DSML text into that form as it streams (holding back only a tag that is still arriving), and DsmlOutputParser is the
+OutputParser behind it. So the OpenAI and Anthropic paths, tool_choice, streaming arguments and the thinking/answer
+split behave for DeepSeek as they do for the other families, and there is one place that knows the DSML spelling.
+"""
+from __future__ import annotations
+
+import re
+
+from serve.frontend import OutputParser
+
+DSML = "|DSML|"
+_OPEN = "<" + DSML
+_CLOSE = "</" + DSML
+_TAG_MAX = 600 # a tag (a tool name, a parameter name) longer than this is text, not held any further
+
+_CALLS = re.compile(r"<|DSML|\s*(?:tool_|function_)?calls\s*>")
+_CALLS_END = re.compile(r"</|DSML|\s*(?:tool_|function_)?calls\s*>")
+_INVOKE = re.compile(r'<|DSML|\s*invoke\s+name\s*=\s*"([^"]*)"\s*>')
+_INVOKE_END = re.compile(r"</|DSML|\s*invoke\s*>")
+_PARAM = re.compile(r'<|DSML|\s*parameter\s+name\s*=\s*"([^"]*)"(?:\s+string\s*=\s*"(true|false)")?\s*>')
+_PARAM_END = re.compile(r"</|DSML|\s*parameter\s*>")
+# a tag cut short is held while it can still become one of those: "<", "<|DS", '<|DSML| invoke name="get_we'
+_PARTIAL = re.compile(r"^</?(?:|(?:D(?:S(?:M(?:L(?:|[^<>]*)?)?)?)?)?)?$")
+
+
+def _could_still_be_a_tag(s: str) -> bool:
+ """`s` starts with '<' and has no '>' yet: it may be the start of a DSML tag."""
+ return len(s) <= _TAG_MAX and _PARTIAL.match(s) is not None
+
+
+class DsmlTranslator:
+ """DSML text -> the Qwen tool-call text, a delta at a time. Text outside a call passes through unchanged.
+
+ on_param(tool, name, is_string) is called when a parameter tag is read, before its text is returned, so the parser
+ behind it can treat a string="true" value as the string it is."""
+
+ def __init__(self, on_param=None):
+ self.buf = ""
+ self.state = "text" # text | calls (inside the block) | invoke | param (inside a value)
+ self.tool = None
+ self.on_param = on_param
+
+ @property
+ def held(self) -> str:
+ """Text held back because it may be the start of a tag."""
+ return self.buf
+
+ def feed(self, delta: str) -> str:
+ self.buf += delta
+ return self._run(final=False)
+
+ def flush(self) -> str:
+ """The end of the output: whatever is held is text."""
+ return self._run(final=True)
+
+ # ------------------------------------------------------------------------------------------------
+ def _run(self, final: bool) -> str:
+ out: list[str] = []
+ while self.buf:
+ lt = self.buf.find("<")
+ if lt < 0:
+ self._text(self.buf, out)
+ self.buf = ""
+ break
+ if lt:
+ self._text(self.buf[:lt], out)
+ self.buf = self.buf[lt:]
+ tag = self._tag(out)
+ if tag == "wait":
+ if final:
+ self._text(self.buf, out)
+ self.buf = ""
+ break
+ if tag is None: # an ordinary '<'
+ self._text("<", out)
+ self.buf = self.buf[1:]
+ return "".join(out)
+
+ def _text(self, text: str, out: list[str]) -> None:
+ if self.state in ("calls", "invoke"):
+ if text.strip() == "":
+ return # the layout between tags
+ # stray text inside the block: it is shown as text, and the block goes on
+ out.append(text)
+
+ def _tag(self, outl: list[str]):
+ """Handle the tag at the start of self.buf. Returns True (a tag was consumed), "wait" (it may still become
+ one) or None (it is not one)."""
+ s = self.buf
+ gt = s.find(">")
+ if gt < 0:
+ return "wait" if _could_still_be_a_tag(s) else None
+ head = s[:gt + 1]
+ if not (head.startswith(_OPEN) or head.startswith(_CLOSE)):
+ return None
+ out = outl.append
+ m = _PARAM_END.fullmatch(head)
+ if m:
+ if self.state == "param":
+ self.buf = s[gt + 1:]
+ self.state = "invoke"
+ out("\n</parameter>\n")
+ return True
+ return None
+ m = _INVOKE_END.fullmatch(head) or _CALLS_END.fullmatch(head)
+ if m:
+ closing_invoke = bool(_INVOKE_END.fullmatch(head))
+ if self.state == "param": # the value's closing tag was left out
+ out("\n</parameter>\n")
+ self.state = "invoke"
+ if closing_invoke and self.state == "invoke":
+ self.buf = s[gt + 1:]
+ self.state = "calls"
+ out("</function>\n</tool_call>\n")
+ return True
+ if not closing_invoke and self.state in ("calls", "invoke"):
+ if self.state == "invoke":
+ out("</function>\n</tool_call>\n")
+ self.buf = s[gt + 1:]
+ self.state = "text"
+ return True
+ return None
+ if self.state == "param":
+ return None # inside a value only a closing tag means anything
+ m = _CALLS.fullmatch(head)
+ if m:
+ self.buf = s[gt + 1:]
+ self.state = "calls"
+ return True
+ m = _INVOKE.fullmatch(head)
+ if m:
+ if self.state == "invoke": # the previous call's closing tag was left out
+ out("</function>\n</tool_call>\n")
+ self.buf = s[gt + 1:]
+ self.state, self.tool = "invoke", m.group(1)
+ out(f"<tool_call>\n<function={m.group(1)}>\n")
+ return True
+ m = _PARAM.fullmatch(head)
+ if m and self.state in ("calls", "invoke"):
+ self.buf = s[gt + 1:]
+ self.state = "param"
+ if self.on_param is not None:
+ self.on_param(self.tool, m.group(1), m.group(2) != "false")
+ out(f"<parameter={m.group(1)}>\n")
+ return True
+ return None
+
+
+class DsmlOutputParser(OutputParser):
+ """OutputParser for DeepSeek-V4.1: the model's DSML calls are read as the template-form calls they are."""
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.dsml = DsmlTranslator(self._on_param)
+
+ def _on_param(self, tool, name, is_string) -> None:
+ """string="true": the value is a string whatever it looks like ("123", "true"), as the template wrote it."""
+ if not is_string:
+ return
+ spec = self.schemas.get(tool)
+ if spec is None:
+ return
+ params = dict(spec.get("parameters") or {})
+ props = dict(params.get("properties") or {})
+ if (props.get(name) or {}).get("type") != "string":
+ props[name] = {**(props.get(name) or {}), "type": "string"}
+ params["properties"] = props
+ self.schemas[tool] = {**spec, "parameters": params}
+ if self.scall is not None and self.scall.name == tool:
+ self.sdeclared[name] = "string"
+
+ def feed(self, delta: str):
+ return super().feed(self.dsml.feed(delta))
+
+ def finish(self, reason: str | None = None):
+ rest = self.dsml.flush()
+ out = super().feed(rest) if rest else []
+ return out + super().finish(reason)
+
+ @property
+ def clean(self) -> bool:
+ """No tag is held back anywhere: a clean point to stop the thinking at."""
+ return not self.buf and not self.dsml.held
+
+
+def to_dsml_force(force: str | None) -> str | None:
+ """forced_call's opening of a call in the Qwen form -> the same opening in DSML (the model's own spelling)."""
+ if not force:
+ return force
+ head = _OPEN + " calls>\n"
+ marker = "<function="
+ at = force.find(marker)
+ name = force[at + len(marker):] if at >= 0 else ""
+ if name.endswith(">\n"):
+ return head + f'{_OPEN} invoke name="{name[:-2]}">\n'
+ return head + f'{_OPEN} invoke name="{name}'
diff --git a/serve/fixtures/deepseek41/chat_template.jinja b/serve/fixtures/deepseek41/chat_template.jinja
new file mode 100644
index 000000000..ecbc24b07
--- /dev/null
+++ b/serve/fixtures/deepseek41/chat_template.jinja
@@ -0,0 +1,291 @@
+{#- Unsloth port of DS41 #}
+{%- set IMG = "<|deepseek_image|>" -%}
+{%- set DSML = "|DSML|" -%}
+
+{%- if thinking_mode is not defined or thinking_mode is none -%}
+ {%- if enable_thinking is defined and enable_thinking is not none -%}
+ {%- set thinking_mode = "thinking" if enable_thinking else "chat" -%}
+ {%- elif thinking is defined and thinking is not none and thinking is not string -%}
+ {%- set thinking_mode = "thinking" if thinking else "chat" -%}
+ {%- else -%}
+ {%- set thinking_mode = "thinking" -%}
+ {%- endif -%}
+{%- endif -%}
+{%- if thinking_mode not in ["chat", "thinking"] -%}
+ {{- raise_exception("Invalid thinking_mode `" ~ thinking_mode ~ "`, should be \"chat\" or \"thinking\"") -}}
+{%- endif -%}
+{%- if drop_thinking is not defined or drop_thinking is none -%}{%- set drop_thinking = true -%}{%- endif -%}
+{%- if add_generation_prompt is not defined -%}{%- set add_generation_prompt = false -%}{%- endif -%}
+{%- if tools is not defined -%}{%- set tools = none -%}{%- endif -%}
+{%- if response_format is not defined -%}{%- set response_format = none -%}{%- endif -%}
+
+{%- if reasoning_effort is not defined or reasoning_effort is none -%}{%- set reasoning_effort = "high" -%}{%- endif -%}
+{%- set effort_map = {"low": 50, "medium": 62, "high": 75, "max": 100} -%}
+{%- if reasoning_effort is string and reasoning_effort in effort_map -%}
+ {%- set budget = effort_map[reasoning_effort] -%}
+{%- elif reasoning_effort is integer and reasoning_effort >= 1 and reasoning_effort <= 100 -%}
+ {%- set budget = reasoning_effort -%}
+{%- else -%}
+ {{- raise_exception("Invalid reasoning effort: " ~ reasoning_effort ~ ", should be int within [1,100] or one of \"low\", \"medium\", \"high\", \"max\"") -}}
+{%- endif -%}
+
+{#- Splits/validates "ns::name" against an explicit namespace and returns the qualified name. -#}
+{%- macro qualified_name(name, ns) -%}
+ {%- if ns is mapping -%}{%- set ns = ns.name -%}{%- endif -%}
+ {%- set parts = name.split("::", 1) -%}
+ {%- if parts | length == 2 -%}
+ {%- if ns is not none and ns != parts[0] -%}{{- raise_exception("Conflicting tool namespaces: " ~ ns ~ " != " ~ parts[0]) -}}{%- endif -%}
+ {%- set ns = parts[0] -%}
+ {%- set name = parts[1] -%}
+ {%- endif -%}
+ {%- if "::" in name -%}{{- raise_exception("Tool name must not contain '::': " ~ name) -}}{%- endif -%}
+ {%- if ns is not none and "::" in ns -%}{{- raise_exception("Tool namespace must not contain '::': " ~ ns) -}}{%- endif -%}
+ {{- name if ns is none else ns ~ "::" ~ name -}}
+{%- endmacro -%}
+
+{%- macro render_tools(tool_list) -%}
+ {%- set ts = namespace(schemas=[]) -%}
+ {%- for tool in tool_list -%}
+ {%- set fn = tool.function if tool.function is defined else tool -%}
+ {%- set ns = tool.namespace if tool.namespace is defined and tool.namespace is not none else fn.get("namespace") -%}
+ {#- JSON assembled by hand (key order kept, name/description replaced in place): llama.cpp has no ** unpacking. -#}
+ {%- set qname = qualified_name(fn.name, ns) -%}
+ {%- set sch = namespace(parts=[]) -%}
+ {%- for k, v in fn.items() if k != "namespace" -%}
+ {%- if k == "name" -%}{%- set v = qname -%}
+ {%- elif k == "description" and ns is mapping and ns.description -%}{%- set v = ns.description ~ "\n" ~ (fn.description or "") -%}
+ {%- endif -%}
+ {%- set sch.parts = sch.parts + [(k | tojson) ~ ": " ~ (v | tojson)] -%}
+ {%- endfor -%}
+ {%- if "description" not in fn and ns is mapping and ns.description -%}
+ {%- set sch.parts = sch.parts + ['"description": ' ~ ((ns.description ~ "\n") | tojson)] -%}
+ {%- endif -%}
+ {%- set ts.schemas = ts.schemas + ["{" ~ sch.parts | join(", ") ~ "}"] -%}
+ {%- endfor -%}
+ {{- "## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<|DSML| calls>\" block like the following:\n\n<|DSML| calls>\n<|DSML| invoke name=\"$TOOL_NAME\">\n<|DSML| parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</|DSML| parameter>\n...\n</|DSML| invoke>\n<|DSML| invoke name=\"$TOOL_NAME2\">\n...\n</|DSML| invoke>\n</|DSML| calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n" -}}
+ {{- ts.schemas | join("\n") -}}
+ {{- "\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n" -}}
+{%- endmacro -%}
+
+{#- 1. Merge tool results (and consecutive user turns) into user messages made of content blocks. -#}
+{%- set mns = namespace(msgs=[]) -%}
+{%- for msg in messages -%}
+ {%- set content = msg.get("content") -%}
+ {%- set is_blocks = content is iterable and content is not string and content is not mapping -%}
+ {%- if content is string and IMG in content -%}
+ {{- raise_exception("Message content contains image special token " ~ IMG ~ ". Images should be provided as image content blocks.") -}}
+ {%- endif -%}
+ {%- set rc = msg.reasoning_content if msg.get("reasoning_content") is not none else msg.get("thinking") -%}
+ {%- if rc is string and IMG in rc -%}
+ {{- raise_exception("reasoning_content contains image special token " ~ IMG) -}}
+ {%- endif -%}
+ {%- set prev = mns.msgs[-1] if mns.msgs else none -%}
+ {%- if msg.role == "tool" -%}
+ {%- set tblock = {"type": "tool_result", "tool_use_id": msg.get("tool_call_id", ""), "content": content if content is not none else ""} -%}
+ {%- if prev is not none and prev.role == "user" and prev.content_blocks is defined -%}
+ {%- set mns.msgs = mns.msgs[:-1] + [dict(prev, content_blocks=prev.content_blocks + [tblock])] -%}
+ {%- else -%}
+ {%- set mns.msgs = mns.msgs + [{"role": "user", "content_blocks": [tblock]}] -%}
+ {%- endif -%}
+ {%- elif msg.role == "user" -%}
+ {%- set blocks = msg.content_blocks if msg.get("content_blocks") is not none else (content if is_blocks else [{"type": "text", "text": content or ""}]) -%}
+ {%- if prev is not none and prev.role == "user" and prev.content_blocks is defined and prev.get("task") is none -%}
+ {%- set mns.msgs = mns.msgs[:-1] + [dict(prev, content_blocks=prev.content_blocks + blocks)] -%}
+ {%- else -%}
+ {%- set mns.msgs = mns.msgs + [dict(msg, content_blocks=blocks)] -%}
+ {%- endif -%}
+ {%- else -%}
+ {%- set mns.msgs = mns.msgs + [msg] -%}
+ {%- endif -%}
+{%- endfor -%}
+
+{#- Top-level tools / response_format attach to a leading system message, or to a new empty one. -#}
+{%- if tools or response_format -%}
+ {%- if not (mns.msgs and mns.msgs[0].role == "system") -%}
+ {%- set mns.msgs = [{"role": "system", "content": ""}] + mns.msgs -%}
+ {%- endif -%}
+ {%- set first = mns.msgs[0] -%}
+ {%- if tools and not first.get("tools") -%}{%- set first = dict(first, tools=tools) -%}{%- endif -%}
+ {%- if response_format and not first.get("response_format") -%}{%- set first = dict(first, response_format=response_format) -%}{%- endif -%}
+ {%- set mns.msgs = [first] + mns.msgs[1:] -%}
+{%- endif -%}
+
+{#- 2. Reorder tool results to match the tool_calls order of the preceding assistant message. -#}
+{%- set sns = namespace(msgs=[], ids=[]) -%}
+{%- for m in mns.msgs -%}
+ {%- if m.role == "assistant" and m.get("tool_calls") -%}
+ {%- set sns.ids = [] -%}
+ {%- for tc in m.tool_calls -%}
+ {%- set sns.ids = sns.ids + [tc.get("id") or (tc.function.get("id", "") if tc.function is defined else "")] -%}
+ {%- endfor -%}
+ {%- set sns.msgs = sns.msgs + [m] -%}
+ {%- elif m.role == "user" and m.content_blocks -%}
+ {%- set tb = namespace(blocks=[]) -%}
+ {%- for b in m.content_blocks if b.type == "tool_result" -%}
+ {%- set order = namespace(idx=0) -%}
+ {%- for tc_id in sns.ids -%}
+ {%- if tc_id and tc_id == b.get("tool_use_id", "") -%}{%- set order.idx = loop.index0 -%}{%- endif -%}
+ {%- endfor -%}
+ {%- set tb.blocks = tb.blocks + [dict(b, _order=order.idx)] -%}
+ {%- endfor -%}
+ {%- if tb.blocks | length > 1 and sns.ids -%}
+ {%- set sorted_blocks = tb.blocks | sort(attribute="_order") -%}
+ {%- set rb = namespace(blocks=[], i=0) -%}
+ {%- for b in m.content_blocks -%}
+ {%- if b.type == "tool_result" -%}
+ {%- set rb.blocks = rb.blocks + [sorted_blocks[rb.i]] -%}
+ {%- set rb.i = rb.i + 1 -%}
+ {%- else -%}
+ {%- set rb.blocks = rb.blocks + [b] -%}
+ {%- endif -%}
+ {%- endfor -%}
+ {%- set sns.msgs = sns.msgs + [dict(m, content_blocks=rb.blocks)] -%}
+ {%- else -%}
+ {%- set sns.msgs = sns.msgs + [m] -%}
+ {%- endif -%}
+ {%- else -%}
+ {%- set sns.msgs = sns.msgs + [m] -%}
+ {%- endif -%}
+{%- endfor -%}
+{%- set msgs = sns.msgs -%}
+
+{#- Last user turn: users and mid-conversation systems count, except user turns made only of tool results.
+ Tools anywhere disable drop_thinking. -#}
+{%- set state = namespace(last_user=-1, has_tools=false) -%}
+{%- for m in msgs -%}
+ {%- if m.role == "user" or (m.role == "system" and loop.index0 > 0) -%}
+ {%- set real = namespace(v=true) -%}
+ {%- if m.get("content_blocks") -%}
+ {%- set real.v = false -%}
+ {%- for b in m.content_blocks if b.type in ["text", "image", "image_url"] -%}{%- set real.v = true -%}{%- endfor -%}
+ {%- endif -%}
+ {%- if real.v -%}{%- set state.last_user = loop.index0 -%}{%- endif -%}
+ {%- endif -%}
+ {%- if m.get("tools") -%}{%- set state.has_tools = true -%}{%- endif -%}
+{%- endfor -%}
+{%- set effective_drop = drop_thinking and not state.has_tools -%}
+{%- set is_thinking = thinking_mode == "thinking" -%}
+{#- Like encoding.py, thinking mode with drop_thinking silently drops messages of other roles before the last user turn. -#}
+{%- if is_thinking and effective_drop -%}
+ {%- set kept = namespace(msgs=[]) -%}
+ {%- for m in msgs -%}
+ {%- if m.role in ["user", "system", "tool", "latest_reminder", "direct_search_results", "assistant"] or loop.index0 >= state.last_user -%}
+ {%- set kept.msgs = kept.msgs + [m] -%}
+ {%- endif -%}
+ {%- endfor -%}
+ {%- if kept.msgs | length != msgs | length -%}
+ {%- set msgs = kept.msgs -%}
+ {%- set state.last_user = -1 -%}
+ {%- for m in msgs -%}
+ {%- if m.role == "user" or (m.role == "system" and loop.index0 > 0) -%}
+ {%- set real = namespace(v=true) -%}
+ {%- if m.get("content_blocks") -%}
+ {%- set real.v = false -%}
+ {%- for b in m.content_blocks if b.type in ["text", "image", "image_url"] -%}{%- set real.v = true -%}{%- endfor -%}
+ {%- endif -%}
+ {%- if real.v -%}{%- set state.last_user = loop.index0 -%}{%- endif -%}
+ {%- endif -%}
+ {%- endfor -%}
+ {%- endif -%}
+{%- endif -%}
+
+{{- "<|begin▁of▁sentence|>" -}}
+{#- 3. Render each message. -#}
+{%- for m in msgs -%}
+ {%- set idx = loop.index0 -%}
+ {%- set content = m.get("content") -%}
+ {%- if content is iterable and content is not string and content is not mapping -%}
+ {%- set sp = namespace(parts=[]) -%}
+ {%- for b in content -%}
+ {%- if b.type == "text" -%}{%- set sp.parts = sp.parts + [b.text or ""] -%}
+ {%- elif b.type in ["image", "image_url"] -%}{%- set sp.parts = sp.parts + [IMG] -%}
+ {%- endif -%}
+ {%- endfor -%}
+ {%- set content = sp.parts | join("\n\n") -%}
+ {%- endif -%}
+
+ {%- if idx == 0 -%}
+ {%- if is_thinking or m.role == "system" -%}{{- "<|System|>" -}}{%- endif -%}
+ {%- if is_thinking -%}{{- "Reasoning Effort: " ~ budget ~ " (range 1-100, the higher the value, the more thorough the reasoning)\n\n" -}}{%- endif -%}
+ {%- endif -%}
+
+ {%- if m.role == "system" -%}
+ {%- if idx > 0 -%}{{- "<|System|>" -}}{%- endif -%}
+ {{- content or "" -}}
+ {%- if m.get("tools") -%}{{- "\n\n" ~ render_tools(m.tools) -}}{%- endif -%}
+ {%- if m.get("response_format") -%}
+ {{- "\n\n## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n" ~ (m.response_format | tojson) -}}
+ {%- endif -%}
+
+ {%- elif m.role == "user" -%}
+ {{- "<|User|>" -}}
+ {%- set up = namespace(parts=[]) -%}
+ {%- for b in m.content_blocks -%}
+ {%- if b.type == "text" -%}
+ {%- if IMG in (b.text or "") -%}{{- raise_exception("Text block contains image placeholder " ~ IMG ~ ". Images should be separate content blocks.") -}}{%- endif -%}
+ {%- set up.parts = up.parts + [b.text or ""] -%}
+ {%- elif b.type in ["image", "image_url"] -%}
+ {%- set up.parts = up.parts + [IMG] -%}
+ {%- elif b.type == "tool_result" -%}
+ {%- set tc = b.get("content", "") -%}
+ {%- if tc is iterable and tc is not string and tc is not mapping -%}
+ {%- set sub = namespace(parts=[]) -%}
+ {%- for s in tc -%}
+ {%- if s.type == "text" -%}
+ {%- if IMG in (s.text or "") -%}{{- raise_exception("Text block contains image placeholder " ~ IMG ~ ". Images should be separate content blocks.") -}}{%- endif -%}
+ {%- set sub.parts = sub.parts + [s.get("text", "")] -%}
+ {%- elif s.type in ["image", "image_url"] -%}{%- set sub.parts = sub.parts + [IMG] -%}
+ {%- else -%}{%- set sub.parts = sub.parts + ["[Unsupported " ~ s.type ~ "]"] -%}
+ {%- endif -%}
+ {%- endfor -%}
+ {%- set tc = sub.parts | join("\n\n") -%}
+ {%- endif -%}
+ {%- set up.parts = up.parts + ["<tool_result>" ~ tc ~ "</tool_result>"] -%}
+ {%- else -%}
+ {%- set up.parts = up.parts + ["[Unsupported " ~ b.type ~ "]"] -%}
+ {%- endif -%}
+ {%- endfor -%}
+ {{- up.parts | join("\n\n") -}}
+
+ {%- elif m.role == "latest_reminder" -%}
+ {{- "<|latest_reminder|>" ~ (content or "") -}}
+
+ {%- elif m.role == "assistant" -%}
+ {%- set prev_has_task = idx > 0 and msgs[idx - 1].get("task") is not none -%}
+ {%- if is_thinking and not prev_has_task and (not effective_drop or idx > state.last_user) -%}
+ {{- ((m.reasoning_content if m.get("reasoning_content") is not none else m.get("thinking")) or "") ~ "</think>" -}}
+ {%- endif -%}
+ {{- content or "" -}}
+ {%- if m.get("tool_calls") -%}
+ {%- set tcs = namespace(lines=[]) -%}
+ {%- for tc in m.tool_calls -%}
+ {%- set fn = tc.function if tc.function is defined else tc -%}
+ {%- set params = namespace(lines=[]) -%}
+ {%- set args = fn.arguments if fn.arguments is mapping else {"arguments": fn.arguments} -%}
+ {%- for k, v in args.items() -%}
+ {%- set params.lines = params.lines + ["<" ~ DSML ~ " parameter name=\"" ~ k ~ "\" string=\"" ~ ("true" if v is string else "false") ~ "\">" ~ (v if v is string else v | tojson) ~ "</" ~ DSML ~ " parameter>"] -%}
+ {%- endfor -%}
+ {%- set name = qualified_name(fn.name, tc.get("namespace") or fn.get("namespace")) -%}
+ {%- set tcs.lines = tcs.lines + ["<" ~ DSML ~ " invoke name=\"" ~ name ~ "\">\n" ~ params.lines | join("\n") ~ "\n</" ~ DSML ~ " invoke>"] -%}
+ {%- endfor -%}
+ {{- "\n\n<" ~ DSML ~ " calls>\n" ~ tcs.lines | join("\n") ~ "\n</" ~ DSML ~ " calls>" -}}
+ {%- endif -%}
+ {%- if not m.get("wo_eos", false) -%}{{- "<|end▁of▁sentence|>" -}}{%- endif -%}
+
+ {%- else -%}
+ {{- raise_exception("Unknown role: " ~ m.role) -}}
+ {%- endif -%}
+
+ {#- Transition tokens, only when the next message is an assistant/latest_reminder turn or this is the last one. -#}
+ {%- if loop.last or msgs[idx + 1].role in ["assistant", "latest_reminder"] -%}
+ {%- set task = m.get("task") -%}
+ {%- if task is not none -%}
+ {%- set task_tokens = {"action": "<|action|>", "query": "<|query|>", "authority": "<|authority|>", "domain": "<|domain|>", "title": "<|title|>", "read_url": "<|read_url|>"} -%}
+ {%- if task not in task_tokens -%}{{- raise_exception("Invalid task: '" ~ task ~ "'") -}}{%- endif -%}
+ {%- if task == "action" -%}{{- "<|Assistant|>" ~ ("<think>" if is_thinking else "</think>") -}}{%- endif -%}
+ {{- task_tokens[task] -}}
+ {%- elif (m.role == "user" or (m.role == "system" and idx > 0)) and (add_generation_prompt or not loop.last) -%}
+ {{- "<|Assistant|>" ~ ("<think>" if is_thinking and (not effective_drop or idx >= state.last_user) else "</think>") -}}
+ {%- endif -%}
+ {%- endif -%}
+{%- endfor -%}
diff --git a/serve/fixtures/deepseek41/golden.json b/serve/fixtures/deepseek41/golden.json
new file mode 100644
index 000000000..01276d80f
--- /dev/null
+++ b/serve/fixtures/deepseek41/golden.json
@@ -0,0 +1,965 @@
+[
+ {
+ "id": "sanity-I1",
+ "mode": "chat",
+ "request": {
+ "messages": [
+ {
+ "role": "user",
+ "content": "[case sanity-I1]\nReturn only a JSON object with ok set to true and count set to the integer 3. No other keys."
+ }
+ ],
+ "chat_template_kwargs": {
+ "thinking": false,
+ "enable_thinking": false
+ }
+ },
+ "thinking": false,
+ "text": "<|begin▁of▁sentence|><|User|>[case sanity-I1]\nReturn only a JSON object with ok set to true and count set to the integer 3. No other keys.<|Assistant|></think>",
+ "ids": [
+ 0,
+ 128803,
+ 61,
+ 11675,
+ 102630,
+ 20448,
+ 19,
+ 2296,
+ 25529,
+ 1353,
+ 260,
+ 26639,
+ 2873,
+ 418,
+ 9109,
+ 1341,
+ 304,
+ 3292,
+ 305,
+ 1895,
+ 1341,
+ 304,
+ 270,
+ 9132,
+ 223,
+ 21,
+ 16,
+ 3011,
+ 915,
+ 18222,
+ 16,
+ 128804,
+ 128822
+ ],
+ "expect_text": "{\"ok\": true, \"count\": 3}<|end▁of▁sentence|>",
+ "finish": "stop"
+ },
+ {
+ "id": "sanity-I1",
+ "mode": "thinking",
+ "request": {
+ "messages": [
+ {
+ "role": "user",
+ "content": "[case sanity-I1]\nReturn only a JSON object with ok set to true and count set to the integer 3. No other keys."
+ }
+ ]
+ },
+ "thinking": true,
+ "text": "<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>[case sanity-I1]\nReturn only a JSON object with ok set to true and count set to the integer 3. No other keys.<|Assistant|><think>",
+ "ids": [
+ 0,
+ 128799,
+ 77666,
+ 288,
+ 9504,
+ 482,
+ 28,
+ 223,
+ 2402,
+ 343,
+ 16801,
+ 223,
+ 19,
+ 15,
+ 1457,
+ 14,
+ 270,
+ 4077,
+ 270,
+ 1990,
+ 14,
+ 270,
+ 850,
+ 15146,
+ 270,
+ 22805,
+ 868,
+ 128803,
+ 61,
+ 11675,
+ 102630,
+ 20448,
+ 19,
+ 2296,
+ 25529,
+ 1353,
+ 260,
+ 26639,
+ 2873,
+ 418,
+ 9109,
+ 1341,
+ 304,
+ 3292,
+ 305,
+ 1895,
+ 1341,
+ 304,
+ 270,
+ 9132,
+ 223,
+ 21,
+ 16,
+ 3011,
+ 915,
+ 18222,
+ 16,
+ 128804,
+ 128821
+ ],
+ "expect_text": "We need answer only JSON object with ok true and count integer 3. No other keys. Need comply.</think>{\"ok\":true,\"count\":3}<|end▁of▁sentence|>",
+ "finish": "stop"
+ },
+ {
+ "id": "sanity-T1",
+ "mode": "chat",
+ "request": {
+ "messages": [
+ {
+ "role": "user",
+ "content": "[case sanity-T1]\nUse lookup_record to look up record A-17."
+ }
+ ],
+ "tools": [
+ {
+ "type": "function",
+ "function": {
+ "name": "lookup_record",
+ "description": "Synthetic benchmark lookup; no external action is performed.",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "id"
+ ],
+ "additionalProperties": false
+ }
+ }
+ }
+ ],
+ "chat_template_kwargs": {
+ "thinking": false,
+ "enable_thinking": false
+ }
+ },
+ "thinking": false,
+ "text": "<|begin▁of▁sentence|><|System|>\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<|DSML| calls>\" block like the following:\n\n<|DSML| calls>\n<|DSML| invoke name=\"$TOOL_NAME\">\n<|DSML| parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</|DSML| parameter>\n...\n</|DSML| invoke>\n<|DSML| invoke name=\"$TOOL_NAME2\">\n...\n</|DSML| invoke>\n</|DSML| calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"name\": \"lookup_record\", \"description\": \"Synthetic benchmark lookup; no external action is performed.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"id\": {\"type\": \"string\"}}, \"required\": [\"id\"], \"additionalProperties\": false}}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<|User|>[case sanity-T1]\nUse lookup_record to look up record A-17.<|Assistant|></think>",
+ "ids": [
+ 0,
+ 128799,
+ 271,
+ 372,
+ 27193,
+ 271,
+ 3476,
+ 611,
+ 3278,
+ 304,
+ 260,
+ 1341,
+ 294,
+ 6704,
+ 304,
+ 1694,
+ 3287,
+ 270,
+ 3967,
+ 734,
+ 3417,
+ 16,
+ 2042,
+ 588,
+ 34756,
+ 6704,
+ 513,
+ 4985,
+ 260,
+ 38322,
+ 128825,
+ 10699,
+ 48902,
+ 5603,
+ 1277,
+ 270,
+ 2502,
+ 979,
+ 30,
+ 128825,
+ 10699,
+ 1018,
+ 30,
+ 128825,
+ 34756,
+ 2329,
+ 84476,
+ 6792,
+ 5896,
+ 35044,
+ 3816,
+ 30,
+ 128825,
+ 10767,
+ 2329,
+ 84476,
+ 46630,
+ 4130,
+ 61729,
+ 35044,
+ 4,
+ 3418,
+ 1281,
+ 11476,
+ 94,
+ 19836,
+ 3320,
+ 6,
+ 46630,
+ 4130,
+ 61729,
+ 56909,
+ 1718,
+ 128825,
+ 10767,
+ 1018,
+ 7835,
+ 1718,
+ 128825,
+ 34756,
+ 1018,
+ 30,
+ 128825,
+ 34756,
+ 2329,
+ 84476,
+ 6792,
+ 5896,
+ 35044,
+ 20,
+ 3816,
+ 7835,
+ 1718,
+ 128825,
+ 34756,
+ 1018,
+ 1718,
+ 128825,
+ 10699,
+ 4697,
+ 3524,
+ 8252,
+ 1531,
+ 366,
+ 12038,
+ 412,
+ 344,
+ 305,
+ 1341,
+ 3608,
+ 4463,
+ 1281,
+ 11476,
+ 4,
+ 37419,
+ 1884,
+ 710,
+ 915,
+ 4815,
+ 343,
+ 62896,
+ 14,
+ 2631,
+ 2065,
+ 634,
+ 14,
+ 31939,
+ 14,
+ 8435,
+ 754,
+ 2281,
+ 270,
+ 1990,
+ 295,
+ 26639,
+ 8786,
+ 305,
+ 1341,
+ 3608,
+ 4463,
+ 1281,
+ 19836,
+ 4,
+ 108526,
+ 3575,
+ 6892,
+ 71144,
+ 344,
+ 22104,
+ 343,
+ 89944,
+ 284,
+ 513,
+ 223,
+ 128821,
+ 754,
+ 440,
+ 74366,
+ 5238,
+ 782,
+ 5553,
+ 22805,
+ 6352,
+ 223,
+ 128821,
+ 1613,
+ 128822,
+ 119907,
+ 1117,
+ 4105,
+ 10699,
+ 469,
+ 4087,
+ 4256,
+ 339,
+ 13079,
+ 6922,
+ 14,
+ 5238,
+ 6578,
+ 1561,
+ 223,
+ 128822,
+ 418,
+ 4105,
+ 10699,
+ 469,
+ 4087,
+ 4256,
+ 339,
+ 795,
+ 17829,
+ 28249,
+ 13178,
+ 8380,
+ 271,
+ 24313,
+ 2852,
+ 3362,
+ 582,
+ 12533,
+ 1425,
+ 119326,
+ 1760,
+ 582,
+ 20855,
+ 3362,
+ 582,
+ 53,
+ 48819,
+ 35312,
+ 64018,
+ 29,
+ 1119,
+ 9390,
+ 4271,
+ 344,
+ 7451,
+ 66910,
+ 582,
+ 46172,
+ 3362,
+ 28612,
+ 4611,
+ 3362,
+ 582,
+ 10325,
+ 1760,
+ 582,
+ 68838,
+ 3362,
+ 28612,
+ 329,
+ 3362,
+ 28612,
+ 4611,
+ 3362,
+ 582,
+ 4463,
+ 4,
+ 55695,
+ 582,
+ 24486,
+ 3362,
+ 20584,
+ 329,
+ 45390,
+ 582,
+ 102970,
+ 34520,
+ 3362,
+ 5062,
+ 33236,
+ 3476,
+ 74366,
+ 29851,
+ 1605,
+ 270,
+ 3554,
+ 6428,
+ 4105,
+ 2329,
+ 305,
+ 10767,
+ 5815,
+ 8380,
+ 304,
+ 34756,
+ 4105,
+ 10699,
+ 603,
+ 128803,
+ 61,
+ 11675,
+ 102630,
+ 8139,
+ 19,
+ 2296,
+ 14885,
+ 64018,
+ 119326,
+ 304,
+ 1747,
+ 890,
+ 3676,
+ 334,
+ 15,
+ 1002,
+ 16,
+ 128804,
+ 128822
+ ],
+ "expect_text": "\n\n<|DSML| calls>\n<|DSML| invoke name=\"lookup_record\">\n<|DSML| parameter name=\"id\" string=\"true\">A-17</|DSML| parameter>\n</|DSML| invoke>\n</|DSML| calls><|end▁of▁sentence|>",
+ "finish": "tool_calls"
+ },
+ {
+ "id": "sanity-T2",
+ "mode": "thinking",
+ "request": {
+ "messages": [
+ {
+ "role": "user",
+ "content": "[case sanity-T2]\nUse get_weather for Zagreb in celsius."
+ }
+ ],
+ "tools": [
+ {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "description": "Synthetic benchmark lookup; no external action is performed.",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "city": {
+ "type": "string"
+ },
+ "unit": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "city",
+ "unit"
+ ],
+ "additionalProperties": false
+ }
+ }
+ }
+ ]
+ },
+ "thinking": true,
+ "text": "<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<|DSML| calls>\" block like the following:\n\n<|DSML| calls>\n<|DSML| invoke name=\"$TOOL_NAME\">\n<|DSML| parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</|DSML| parameter>\n...\n</|DSML| invoke>\n<|DSML| invoke name=\"$TOOL_NAME2\">\n...\n</|DSML| invoke>\n</|DSML| calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"name\": \"get_weather\", \"description\": \"Synthetic benchmark lookup; no external action is performed.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}, \"unit\": {\"type\": \"string\"}}, \"required\": [\"city\", \"unit\"], \"additionalProperties\": false}}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<|User|>[case sanity-T2]\nUse get_weather for Zagreb in celsius.<|Assistant|><think>",
+ "ids": [
+ 0,
+ 128799,
+ 77666,
+ 288,
+ 9504,
+ 482,
+ 28,
+ 223,
+ 2402,
+ 343,
+ 16801,
+ 223,
+ 19,
+ 15,
+ 1457,
+ 14,
+ 270,
+ 4077,
+ 270,
+ 1990,
+ 14,
+ 270,
+ 850,
+ 15146,
+ 270,
+ 22805,
+ 86262,
+ 372,
+ 27193,
+ 271,
+ 3476,
+ 611,
+ 3278,
+ 304,
+ 260,
+ 1341,
+ 294,
+ 6704,
+ 304,
+ 1694,
+ 3287,
+ 270,
+ 3967,
+ 734,
+ 3417,
+ 16,
+ 2042,
+ 588,
+ 34756,
+ 6704,
+ 513,
+ 4985,
+ 260,
+ 38322,
+ 128825,
+ 10699,
+ 48902,
+ 5603,
+ 1277,
+ 270,
+ 2502,
+ 979,
+ 30,
+ 128825,
+ 10699,
+ 1018,
+ 30,
+ 128825,
+ 34756,
+ 2329,
+ 84476,
+ 6792,
+ 5896,
+ 35044,
+ 3816,
+ 30,
+ 128825,
+ 10767,
+ 2329,
+ 84476,
+ 46630,
+ 4130,
+ 61729,
+ 35044,
+ 4,
+ 3418,
+ 1281,
+ 11476,
+ 94,
+ 19836,
+ 3320,
+ 6,
+ 46630,
+ 4130,
+ 61729,
+ 56909,
+ 1718,
+ 128825,
+ 10767,
+ 1018,
+ 7835,
+ 1718,
+ 128825,
+ 34756,
+ 1018,
+ 30,
+ 128825,
+ 34756,
+ 2329,
+ 84476,
+ 6792,
+ 5896,
+ 35044,
+ 20,
+ 3816,
+ 7835,
+ 1718,
+ 128825,
+ 34756,
+ 1018,
+ 1718,
+ 128825,
+ 10699,
+ 4697,
+ 3524,
+ 8252,
+ 1531,
+ 366,
+ 12038,
+ 412,
+ 344,
+ 305,
+ 1341,
+ 3608,
+ 4463,
+ 1281,
+ 11476,
+ 4,
+ 37419,
+ 1884,
+ 710,
+ 915,
+ 4815,
+ 343,
+ 62896,
+ 14,
+ 2631,
+ 2065,
+ 634,
+ 14,
+ 31939,
+ 14,
+ 8435,
+ 754,
+ 2281,
+ 270,
+ 1990,
+ 295,
+ 26639,
+ 8786,
+ 305,
+ 1341,
+ 3608,
+ 4463,
+ 1281,
+ 19836,
+ 4,
+ 108526,
+ 3575,
+ 6892,
+ 71144,
+ 344,
+ 22104,
+ 343,
+ 89944,
+ 284,
+ 513,
+ 223,
+ 128821,
+ 754,
+ 440,
+ 74366,
+ 5238,
+ 782,
+ 5553,
+ 22805,
+ 6352,
+ 223,
+ 128821,
+ 1613,
+ 128822,
+ 119907,
+ 1117,
+ 4105,
+ 10699,
+ 469,
+ 4087,
+ 4256,
+ 339,
+ 13079,
+ 6922,
+ 14,
+ 5238,
+ 6578,
+ 1561,
+ 223,
+ 128822,
+ 418,
+ 4105,
+ 10699,
+ 469,
+ 4087,
+ 4256,
+ 339,
+ 795,
+ 17829,
+ 28249,
+ 13178,
+ 8380,
+ 271,
+ 24313,
+ 2852,
+ 3362,
+ 582,
+ 1133,
+ 65,
+ 50219,
+ 1760,
+ 582,
+ 20855,
+ 3362,
+ 582,
+ 53,
+ 48819,
+ 35312,
+ 64018,
+ 29,
+ 1119,
+ 9390,
+ 4271,
+ 344,
+ 7451,
+ 66910,
+ 582,
+ 46172,
+ 3362,
+ 28612,
+ 4611,
+ 3362,
+ 582,
+ 10325,
+ 1760,
+ 582,
+ 68838,
+ 3362,
+ 28612,
+ 37399,
+ 3362,
+ 28612,
+ 4611,
+ 3362,
+ 582,
+ 4463,
+ 87928,
+ 582,
+ 15165,
+ 3362,
+ 28612,
+ 4611,
+ 3362,
+ 582,
+ 4463,
+ 4,
+ 55695,
+ 582,
+ 24486,
+ 3362,
+ 20584,
+ 37399,
+ 1760,
+ 582,
+ 15165,
+ 45390,
+ 582,
+ 102970,
+ 34520,
+ 3362,
+ 5062,
+ 33236,
+ 3476,
+ 74366,
+ 29851,
+ 1605,
+ 270,
+ 3554,
+ 6428,
+ 4105,
+ 2329,
+ 305,
+ 10767,
+ 5815,
+ 8380,
+ 304,
+ 34756,
+ 4105,
+ 10699,
+ 603,
+ 128803,
+ 61,
+ 11675,
+ 102630,
+ 8139,
+ 20,
+ 2296,
+ 14885,
+ 1178,
+ 65,
+ 50219,
+ 362,
+ 121648,
+ 295,
+ 124281,
+ 16,
+ 128804,
+ 128821
+ ],
+ "expect_text": "The user wants weather for Zagreb in celsius. Call the tool.</think>\n\n<|DSML| calls>\n<|DSML| invoke name=\"get_weather\">\n<|DSML| parameter name=\"city\" string=\"true\">Zagreb</|DSML| parameter>\n<|DSML| parameter name=\"unit\" string=\"true\">celsius</|DSML| parameter>\n</|DSML| invoke>\n</|DSML| calls><|end▁of▁sentence|>",
+ "finish": "tool_calls"
+ },
+ {
+ "id": "sanity-I3",
+ "mode": "thinking",
+ "request": {
+ "messages": [
+ {
+ "role": "user",
+ "content": "[case sanity-I3]\nProject: Orion. Build: 17. Return only a JSON object with keys project (string) and build (integer)."
+ }
+ ]
+ },
+ "thinking": true,
+ "text": "<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>[case sanity-I3]\nProject: Orion. Build: 17. Return only a JSON object with keys project (string) and build (integer).<|Assistant|><think>",
+ "ids": [
+ 0,
+ 128799,
+ 77666,
+ 288,
+ 9504,
+ 482,
+ 28,
+ 223,
+ 2402,
+ 343,
+ 16801,
+ 223,
+ 19,
+ 15,
+ 1457,
+ 14,
+ 270,
+ 4077,
+ 270,
+ 1990,
+ 14,
+ 270,
+ 850,
+ 15146,
+ 270,
+ 22805,
+ 868,
+ 128803,
+ 61,
+ 11675,
+ 102630,
+ 20448,
+ 21,
+ 2296,
+ 20811,
+ 28,
+ 84706,
+ 16,
+ 23718,
+ 28,
+ 223,
+ 1002,
+ 16,
+ 20870,
+ 1353,
+ 260,
+ 26639,
+ 2873,
+ 418,
+ 18222,
+ 2775,
+ 343,
+ 4463,
+ 11,
+ 305,
+ 2906,
+ 343,
+ 39648,
+ 797,
+ 128804,
+ 128821
+ ],
+ "expect_text": "We need answer only JSON object keys project string build integer. Need infer from prompt: Project: Orion. Build: 17. So {\"project\":\"Orion\",\"build\":17}. Ensure only JSON.</think>{\"project\":\"Orion\",\"build\":17}<|end▁of▁sentence|>",
+ "finish": "stop"
+ },
+ {
+ "id": "performance-short-r1",
+ "mode": "chat",
+ "request": {
+ "messages": [
+ {
+ "role": "user",
+ "content": "[case performance-short-r1]\nExplain how a hash table finds a value, including collisions and one small example."
+ }
+ ],
+ "chat_template_kwargs": {
+ "thinking": false,
+ "enable_thinking": false
+ }
+ },
+ "thinking": false,
+ "text": "<|begin▁of▁sentence|><|User|>[case performance-short-r1]\nExplain how a hash table finds a value, including collisions and one small example.<|Assistant|></think>",
+ "ids": [
+ 0,
+ 128803,
+ 61,
+ 11675,
+ 4197,
+ 10565,
+ 482,
+ 9954,
+ 19,
+ 2296,
+ 65106,
+ 1192,
+ 260,
+ 19657,
+ 4184,
+ 15249,
+ 260,
+ 1990,
+ 14,
+ 2622,
+ 54415,
+ 305,
+ 834,
+ 2395,
+ 2511,
+ 16,
+ 128804,
+ 128822
+ ],
+ "expect_text": "# How a Hash Table Finds a Value\n\nA hash table stores key–value pairs and uses a **hash function** to map each key to an index in an underlying array. Lookup is fast because it jumps almost directly to the right slot instead of scanning.\n\n## The lookup process\n\n1. **Hash the key.** Compute `h = hash(key)`, then reduce it to a valid array index, e.g. `index = h % capacity`.\n2. **Go to that bucket.** The array slot at `index` holds the entry (or a chain/list of entries).\n3. **Compare keys.** If the bucket holds the exact key you asked for, return its value.\n4. **Handle collisions.** If the bucket holds a *different* key that happened to hash to the same index, keep searching within that bucket (see below).\n\n## Collisions\n\nA **collision** happens when two different keys map to the same index. Two common resolutions:\n\n- **Separate chaining:** each bucket is a small list (or tree). On lookup, walk the list comparing keys until you find a match or reach the end (key absent).\n- **Open addressing:** if the slot is occupied by a different key, probe other slots (linear probing:",
+ "finish": "length"
+ }
+]
\ No newline at end of file
diff --git a/serve/frontend.py b/serve/frontend.py
index 3fda9c56f..33f4bbd6e 100644
--- a/serve/frontend.py
+++ b/serve/frontend.py
@@ -135,15 +135,36 @@ EFFORT = {"none": None, "off": None, "minimal": None, "disabled": None, "false":
"low": "low", "medium": "medium", "high": "xhigh", "xhigh": "xhigh", "max": "xhigh", "maximum": "xhigh"}
+# DeepSeek-V4.1's template takes low, medium, high (its default) and max, or a whole number 1..100 (the "Reasoning
+# Effort: N" line of its prompt). use_family("deepseek41") switches the table; the other families keep the one above.
+EFFORT_DEEPSEEK = {"none": None, "off": None, "minimal": None, "disabled": None, "false": None,
+ "low": "low", "medium": "medium", "high": "high", "xhigh": "max", "max": "max", "maximum": "max"}
+_EFFORT_QWEN = dict(EFFORT)
+_NUMERIC_EFFORT = False
+
+
+def use_family(family: str | None) -> None:
+ """The reasoning-effort words of the model family this server runs (module state, like ENV_TO_USER)."""
+ global _NUMERIC_EFFORT
+ EFFORT.clear()
+ EFFORT.update(EFFORT_DEEPSEEK if family == "deepseek41" else _EFFORT_QWEN)
+ _NUMERIC_EFFORT = family == "deepseek41"
+
+
def effort_kwargs(value) -> dict:
"""A reasoning effort as given by a client -> the template's kwargs. Unknown values are a 400, not a crash."""
if value is None or value == "":
return {}
if value is False:
return {"enable_thinking": False}
+ if _NUMERIC_EFFORT and isinstance(value, int) and not isinstance(value, bool):
+ if not 1 <= value <= 100:
+ raise ValueError(f"unknown reasoning effort {value!r}: use none, low, medium, high, max or 1..100")
+ return {"reasoning_effort": value}
key = str(value).strip().lower()
if key not in EFFORT:
- raise ValueError(f"unknown reasoning effort {value!r}: use none, low, medium or high")
+ raise ValueError(f"unknown reasoning effort {value!r}: use none, low, medium or high"
+ + (", max or 1..100" if _NUMERIC_EFFORT else ""))
level = EFFORT[key]
return {"enable_thinking": False} if level is None else {"reasoning_effort": level}
@@ -154,7 +175,7 @@ def budget_effort(tokens) -> dict:
n = int(tokens)
except (TypeError, ValueError):
return {}
- return {"reasoning_effort": "low" if n < 2048 else "medium" if n < 8192 else "xhigh"}
+ return {"reasoning_effort": "low" if n < 2048 else "medium" if n < 8192 else EFFORT["high"]}
def _has_image(content) -> bool:
@@ -905,6 +926,12 @@ class OutputParser:
self.batch = False # the call being finished is followed by another in the same wrapper
self._reset_scan()
+ @property
+ def clean(self) -> bool:
+ """Nothing is held back: a clean point to stop the thinking at (a parser that rewrites the model's text, as
+ serve/dsml.py's does, also counts what it holds)."""
+ return not self.buf
+
def _ok_at(self, p: int) -> bool:
"""self.buf[p] would open a call: at the start of a line, outside a code fence and inline code."""
snap = (self.fence, self.line, self.ticks)
diff --git a/serve/server.py b/serve/server.py
index 0635a715f..f41d96dfc 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -3583,6 +3583,8 @@ class Service:
self.restore_after_load = None # a session file an image job saved before it unloaded
self.stop_ids = set(tokenizer.encode(IM_END, parse_special=True) +
tokenizer.encode("<|endoftext|>", parse_special=True))
+ self.family = None # the model family ("deepseek41"): set_family() below
+ self.parser_cls = OutputParser # reads the model's output (reasoning, content, tool calls)
# #567: a prompt re-encodes only what follows the last special token it shares with a recent prompt (the
# same ids as a full encode: tools/strata_tokenizer.py PromptEncoder). Tokenizers without resume points
# encode in full.
@@ -3595,6 +3597,25 @@ class Service:
self.prompts = PromptEncoder(tokenizer, keep=slots) if slots is not None and slots > 0 \
else PromptEncoder(tokenizer)
+ def set_family(self, family: str | None) -> None:
+ """A family whose text differs from the Qwen form this server was written for (DeepSeek-V4.1: its own end
+ token, DSML tool calls, reasoning-effort words). The others keep the defaults."""
+ self.family = family
+ if family == "deepseek41":
+ from serve import dsml
+ eos = (getattr(self.tok, "special_ids", None) or {}).get("eos", 1)
+ self.stop_ids = {int(eos)}
+ self.parser_cls = dsml.DsmlOutputParser
+ self._force_text = dsml.to_dsml_force
+ import serve.frontend as _frontend
+ _frontend.use_family(family)
+
+ _force_text = None
+
+ def force_text(self, force):
+ """forced_call's opening of the call the reply must make, in this model's spelling."""
+ return self._force_text(force) if self._force_text is not None else force
+
def loaded(self) -> bool:
return not hasattr(self.engine, "alive") or self.engine.alive()
@@ -4718,7 +4739,7 @@ class Service:
print(f"[strata] thinking capped at {max_new - reserve} of max_tokens {max_new} to leave room for "
f"the answer (budget {budget})", flush=True)
budget = max(1, max_new - reserve)
- parser = OutputParser(thinking=thinking, tools=tools, stream_tools=True, recover=self.tool_call_recovery)
+ parser = self.parser_cls(thinking=thinking, tools=tools, stream_tools=True, recover=self.tool_call_recovery)
detok, n, finish = Detokenizer(self.tok), 0, "length"
run_tok, run_len, repeated = None, 0, False # #606: the current run of one repeated token
thinking_n = 0 # tokens written while thinking (Responses' reasoning_tokens)
@@ -4943,13 +4964,13 @@ class Service:
if budget and parser.state == "reasoning":
thought += 1
# at a clean point: no tag held back, no character split across tokens
- if thought >= budget and not parser.buf and not detok.pending():
+ if thought >= budget and parser.clean and not detok.pending():
wrap = True
break
if force:
if any(ev.kind in ("tool_start", "tool_call") for ev in evs):
force = None # the model is writing a call of its own
- elif parser.state == "content" and not parser.buf and not detok.pending():
+ elif parser.state == "content" and parser.clean and not detok.pending():
opens = True
break
except EngineNonFinite:
@@ -6731,7 +6752,7 @@ def make_handler(svc: Service):
self._no_local_images(messages)
if tool_choice_of(req.get("tool_choice"))[0] == "none": # as the Responses route: no tools are offered
tools = None
- force = forced_call(req.get("tool_choice"), tools) # a bad value is a 400 before anything is sent
+ force = svc.force_text(forced_call(req.get("tool_choice"), tools)) # a bad value is a 400 before anything is sent
messages, validator = prepare_format(req.get("response_format"), messages)
if validator is not None and (tools or req.get("strata_mcp")):
raise ValueError("structured response_format with tools/MCP is not supported")
@@ -6983,7 +7004,7 @@ def make_handler(svc: Service):
self._no_local_images(messages)
if tool_choice_of(req.get("tool_choice"))[0] == "none": # Anthropic's {"type": "none"}: no tools offered
tools = None
- force = forced_call(req.get("tool_choice"), tools) # "any" / {"type": "tool", "name": N}
+ force = svc.force_text(forced_call(req.get("tool_choice"), tools)) # "any" / {"type": "tool", "name": N}
max_new = int(req.get("max_tokens") or 0) # 0/-1: the rest of the context
svc.reasoning_budget(req) # a bad value is a 400 before anything is sent
stop_strings(req) # ... and so is a bad stop / stop_sequences
@@ -7636,7 +7657,8 @@ def check_request_sampling(req) -> None:
# the API's model name per model family (the GGUF architecture, "arch" in the config) when the config names none
-FAMILY_MODEL_NAMES = {"qwen4exp": "qwen3.8-flash-next", "qwen35moe": "qwen3.6-35b-a3b"}
+FAMILY_MODEL_NAMES = {"qwen4exp": "qwen3.8-flash-next", "qwen35moe": "qwen3.6-35b-a3b",
+ "deepseek41": "deepseek-v4.1-flash"}
# The sampling each model card recommends ("Best Practices"), used when neither the request, the Chat settings nor the
@@ -7649,6 +7671,8 @@ MODEL_CARD_SAMPLING = {
# Qwen3.6-35B-A3B: the general-task sets (the card's precise-coding set is temperature 0.6, presence_penalty 0)
"qwen35moe": {"thinking": {"temperature": 1.0, "top_p": 0.95, "top_k": 20, "presence_penalty": 1.5},
"instruct": {"temperature": 0.7, "top_p": 0.8, "top_k": 20, "presence_penalty": 1.5}},
+ # DeepSeek-V4.1-Flash: temperature 1.0, top_p 1.0 for local use (thinking or not), no penalties
+ "deepseek41": {"thinking": {"temperature": 1.0, "top_p": 1.0}, "instruct": {"temperature": 1.0, "top_p": 1.0}},
}
@@ -7909,7 +7933,12 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> in
tokens[i] = t
merges = (tpath / "merges.txt").read_text(encoding="utf-8").split("\n")
types = json.loads((tpath / "token_type.json").read_text())
- tok = ST.Tokenizer(tokens, merges, types)
+ tcfg = {}
+ try: # the pack's pre-tokenizer name and special ids (DeepSeek's differ)
+ tcfg = json.loads((tpath / "tokenizer.json").read_text(encoding="utf-8"))
+ except (OSError, ValueError):
+ pass
+ tok = ST.Tokenizer(tokens, merges, types, tcfg.get("pre") or "qwen35", tcfg.get("special_ids") or None)
hub = hub_from_config(cfg, a.mcp_config) # before the minutes of loading: a bad entry stops here
if a.offline and (hub is not None or cfg.get("before_load") or a.before_load):
ap.error("--offline requires MCP servers and before_load commands to be disabled in this profile")
@@ -8015,6 +8044,7 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> in
svc.queue_timeout_s = float(queue_timeout) if queue_timeout is not None else None
if a.engine == "strata": # the model card's sampling when nothing else is said (see MODEL_CARD_SAMPLING)
svc.model_arch = cfg.get("arch") or "qwen4exp"
+ svc.set_family(svc.model_arch if svc.model_arch == "deepseek41" else None)
requested = cfg.get("parallel")
if requested is None:
args = cfg.get("args", [])
diff --git a/serve/test_dsv41.py b/serve/test_dsv41.py
new file mode 100644
index 000000000..71c6a12fe
--- /dev/null
+++ b/serve/test_dsv41.py
@@ -0,0 +1,465 @@
+"""DeepSeek-V4.1 through the server: its tokenizer, its chat template, the DSML tool calls, the effort words, the end
+token and the model-card defaults, over both APIs against a fake engine (no GPU, no pack, no model).
+
+ python -m unittest serve.test_dsv41 -v
+
+STRATA_DSV41_PACK=<dir> (the pack's tokenizer folder: vocab.json, merges.txt, token_type.json, tokenizer.json) adds
+the check of the real tokenizer against the ids llama.cpp (PR 30817 b72050d0) made of the same rendered prompts.
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+import unittest
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+sys.path.insert(0, str(ROOT / "tools"))
+import strata_tokenizer as ST # noqa: E402
+from serve import frontend # noqa: E402
+from serve.dsml import DsmlOutputParser, DsmlTranslator, to_dsml_force # noqa: E402
+from serve.frontend import ChatTemplate, OutputParser, openai_to_messages # noqa: E402
+from serve.server import (FAMILY_MODEL_NAMES, MockEngine, Service, default_model_name, model_card_sampling, # noqa: E402
+ serve)
+
+FIX = ROOT / "serve" / "fixtures" / "deepseek41"
+TEMPLATE = FIX / "chat_template.jinja"
+GOLDEN = json.loads((FIX / "golden.json").read_text(encoding="utf-8"))
+
+BOS, EOS_TEXT = "<|begin▁of▁sentence|>", "<|end▁of▁sentence|>"
+SPECIALS = [(BOS, 3), (EOS_TEXT, 3), ("<|User|>", 3), ("<|Assistant|>", 3), ("<|System|>", 3),
+ ("<|latest_reminder|>", 3), ("<think>", 4), ("</think>", 4), ("|DSML|", 4), ("<|deepseek_image|>", 3)]
+EOS = 1
+
+
+def mini_tokenizer() -> ST.Tokenizer:
+ """The real Tokenizer class over a tiny vocabulary: DeepSeek's special tokens, then the 256 byte tokens, no merges."""
+ b2u = ST.bytes_to_unicode()
+ tokens = [s for s, _ in SPECIALS] + [b2u[b] for b in range(256)]
+ types = [t for _, t in SPECIALS] + [1] * 256
+ return ST.Tokenizer(tokens, [], types, "deepseek3", {"bos": 0, "eos": EOS})
+
+
+class DsEngine(MockEngine):
+ """Replays scripted completions as ids, then DeepSeek's end token; keeps what it was asked."""
+
+ def __init__(self, tok, script, max_context=8192):
+ super().__init__(tok, script, max_context=max_context)
+ self.scripts = [tok.encode(x, parse_special=True) + [EOS] for x in ([script] if isinstance(script, str) else script)]
+ self.script = self.scripts[0]
+ self.last_sampling = None
+ self.last = {}
+
+ def generate(self, ids, max_new, sampling, cancel, embeddings=None):
+ self.last_sampling = dict(sampling or {})
+ got = 0
+ try:
+ for t in super().generate(ids, max_new, sampling, cancel, embeddings):
+ got += 1
+ yield t
+ finally: # (the server stops reading at the end token: the engine's DONE comes while it is closed)
+ self.last = {"generated": got, "prompt_tokens": len(ids), "prompt_ms": 50.0, "decode_ms": got * 10.0,
+ "finish": "stop"}
+
+
+def call(*invokes, string="true") -> str:
+ """The text DeepSeek writes for calls: [(name, [(param, value, is_string)...])...]."""
+ out = ["\n\n<|DSML| calls>"]
+ for name, params in invokes:
+ out.append(f'<|DSML| invoke name="{name}">')
+ for p, v, is_str in params:
+ out.append(f'<|DSML| parameter name="{p}" string="{"true" if is_str else "false"}">{v}</|DSML| parameter>')
+ out.append("</|DSML| invoke>")
+ out.append("</|DSML| calls>")
+ return "\n".join(out)
+
+
+WEATHER = {"type": "function", "function": {"name": "get_weather", "description": "weather",
+ "parameters": {"type": "object", "properties": {
+ "city": {"type": "string"}, "days": {"type": "integer"}},
+ "required": ["city"]}}}
+
+
+class Pre(unittest.TestCase):
+ """The pre-tokenizer is llama.cpp's DEEPSEEK3_LLM list: a sequence of splits, not one pattern."""
+
+ def split(self, text):
+ return ST._sequence_findall([ST.regex.compile(p) for p in ST.DEEPSEEK3_PATTERNS], text)
+
+ def test_digits_in_runs_of_three(self):
+ self.assertEqual(self.split("a12345678b"), ["a", "123", "456", "78", "b"])
+
+ def test_cjk_runs(self):
+ self.assertEqual(self.split("你好, world"), ["你好", ",", " world"])
+
+ def test_punctuation_then_letters_and_whitespace(self):
+ self.assertEqual(self.split("x_y.z \n\nq"), ["x", "_y", ".z", " \n\n", "q"])
+
+ def test_every_character_is_kept(self):
+ text = "Hello, 世界! 123\t\r\n é \U0001f600 |DSML|"
+ self.assertEqual("".join(self.split(text)), text)
+
+
+class Tokenizer(unittest.TestCase):
+ def test_a_special_token_decodes_to_its_text(self):
+ tok = mini_tokenizer()
+ ids = tok.encode("<|User|>hi|DSML|", parse_special=True)
+ self.assertEqual(ids[0], 2)
+ self.assertEqual(tok.decode(ids), "<|User|>hi|DSML|")
+ self.assertEqual(tok.token_bytes(8), "|DSML|".encode())
+
+ def test_the_dsml_token_is_matched_without_parse_special_and_a_control_token_is_not(self):
+ tok = mini_tokenizer()
+ self.assertEqual(tok.encode("|DSML|").__len__(), 1)
+ self.assertGreater(len(tok.encode("<|User|>")), 1)
+
+ def test_qwen_tokens_are_unchanged(self):
+ b2u = ST.bytes_to_unicode()
+ tok = ST.Tokenizer([b2u[b] for b in range(256)] + ["<|im_end|>"], [], [1] * 256 + [3])
+ self.assertEqual(tok.token_bytes(256), b"<|im_end|>")
+ self.assertEqual(tok.encode("a<|im_end|>", parse_special=True), [97, 256])
+
+ @unittest.skipUnless(os.environ.get("STRATA_DSV41_PACK"), "STRATA_DSV41_PACK (the pack's tokenizer folder) not set")
+ def test_the_real_tokenizer_makes_the_ids_llama_cpp_made(self):
+ d = Path(os.environ["STRATA_DSV41_PACK"])
+ vocab = json.loads((d / "vocab.json").read_text(encoding="utf-8"))
+ tokens = [None] * len(vocab)
+ for t, i in vocab.items():
+ tokens[i] = t
+ cfg = json.loads((d / "tokenizer.json").read_text())
+ tok = ST.Tokenizer(tokens, (d / "merges.txt").read_text(encoding="utf-8").split("\n"),
+ json.loads((d / "token_type.json").read_text()), cfg["pre"], cfg["special_ids"])
+ for case in GOLDEN:
+ self.assertEqual(tok.encode(case["text"], parse_special=True), case["ids"], case["id"] + " " + case["mode"])
+
+
+class Template(unittest.TestCase):
+ """The GGUF's own chat template (the same bytes as DeepSeek's encoding.py on the cases checked), through the
+ server's request reader and renderer: the text llama.cpp tokenized for the reference run."""
+
+ @classmethod
+ def setUpClass(cls):
+ cls.tpl = ChatTemplate(TEMPLATE)
+
+ def test_the_golden_prompts(self):
+ for case in GOLDEN:
+ messages, tools, kw = openai_to_messages(case["request"])
+ self.assertEqual(self.tpl.render(messages, tools=tools, **kw), case["text"], case["id"] + " " + case["mode"])
+
+ def test_thinking_is_the_default(self):
+ text = self.tpl.render([{"role": "user", "content": "hi"}])
+ self.assertTrue(text.endswith("<|Assistant|><think>"))
+ text = self.tpl.render([{"role": "user", "content": "hi"}], enable_thinking=False)
+ self.assertTrue(text.endswith("<|Assistant|></think>"))
+
+ def test_a_tool_call_and_its_result_render_in_dsml(self):
+ msgs = [{"role": "user", "content": "weather?"},
+ {"role": "assistant", "content": "", "reasoning_content": "call it",
+ "tool_calls": [{"function": {"name": "get_weather", "arguments": {"city": "Zagreb", "days": 3}}}]},
+ {"role": "tool", "content": "sunny"}]
+ text = self.tpl.render(msgs, tools=[WEATHER["function"]], enable_thinking=False)
+ self.assertIn('<|DSML| invoke name="get_weather">', text)
+ self.assertIn('<|DSML| parameter name="city" string="true">Zagreb</|DSML| parameter>', text)
+ self.assertIn('<|DSML| parameter name="days" string="false">3</|DSML| parameter>', text)
+ self.assertIn("<tool_result>sunny</tool_result>", text)
+
+
+class Translator(unittest.TestCase):
+ RAW = call(("get_weather", [("city", "Zagreb", True), ("days", "3", False)]), ("ping", []))
+
+ def all_at_once(self, text):
+ t = DsmlTranslator()
+ return t.feed(text) + t.flush()
+
+ def test_a_call_becomes_the_qwen_form(self):
+ got = self.all_at_once("Sure." + self.RAW)
+ self.assertEqual(got, "Sure.\n\n<tool_call>\n<function=get_weather>\n<parameter=city>\nZagreb\n</parameter>\n"
+ "<parameter=days>\n3\n</parameter>\n</function>\n</tool_call>\n"
+ "<tool_call>\n<function=ping>\n</function>\n</tool_call>\n")
+
+ def test_the_split_of_the_stream_changes_nothing(self):
+ want = self.all_at_once("a < b and 1<2" + self.RAW + "tail")
+ for step in (1, 2, 3, 5, 8, 13):
+ t = DsmlTranslator()
+ text = "a < b and 1<2" + self.RAW + "tail"
+ got = "".join(t.feed(text[i:i + step]) for i in range(0, len(text), step)) + t.flush()
+ self.assertEqual(got, want, step)
+
+ def test_text_with_angle_brackets_passes(self):
+ text = "x <div> y </|DSML| nope> <|User|> <|DSML| unknown>"
+ self.assertEqual(self.all_at_once(text), text)
+
+ def test_the_end_of_the_output_inside_a_tag_is_text(self):
+ t = DsmlTranslator()
+ got = t.feed("ok <|DSML| inv") + t.flush()
+ self.assertEqual(got, "ok <|DSML| inv")
+
+ def test_a_value_that_looks_like_a_tag(self):
+ raw = call(("write", [("text", "<b>bold</b> and < 3", True)]))
+ self.assertIn("<parameter=text>\n<b>bold</b> and < 3\n</parameter>", self.all_at_once(raw))
+
+ def test_a_missing_closing_tag_is_closed(self):
+ raw = '<|DSML| calls>\n<|DSML| invoke name="f">\n<|DSML| parameter name="a" string="true">x</|DSML| invoke>'
+ self.assertEqual(self.all_at_once(raw),
+ "<tool_call>\n<function=f>\n<parameter=a>\nx\n</parameter>\n</function>\n</tool_call>\n")
+
+
+class Parser(unittest.TestCase):
+ TOOLS = [WEATHER["function"], {"name": "lookup", "description": "", "parameters": {
+ "type": "object", "properties": {"id": {}}}}]
+
+ def run_parser(self, text, thinking=True, step=7, cls=DsmlOutputParser):
+ p = cls(thinking=thinking, tools=self.TOOLS, stream_tools=True)
+ evs = []
+ for i in range(0, len(text), step):
+ evs += p.feed(text[i:i + step])
+ evs += p.finish("stop")
+ return evs
+
+ def test_reasoning_then_two_calls(self):
+ raw = call(("get_weather", [("city", "Zagreb", True), ("days", "3", False)]), ("lookup", [("id", "123", True)]))
+ for step in (1, 4, 1000):
+ evs = self.run_parser("Think < ok.</think>" + raw, step=step)
+ self.assertEqual("".join(e.text for e in evs if e.kind == "reasoning"), "Think < ok.")
+ calls = [e.call for e in evs if e.kind == "tool_call"]
+ self.assertEqual([(c.name, c.arguments) for c in calls],
+ [("get_weather", {"city": "Zagreb", "days": 3}), ("lookup", {"id": "123"})])
+ self.assertEqual("".join(e.text for e in evs if e.kind == "tool_args"),
+ '{"city":"Zagreb","days":3}{"id":"123"}')
+
+ def test_a_string_value_stays_a_string_even_when_the_schema_says_nothing(self):
+ evs = self.run_parser(call(("lookup", [("id", "007", True)])), thinking=False)
+ self.assertEqual([e.call.arguments for e in evs if e.kind == "tool_call"], [{"id": "007"}])
+ evs = self.run_parser(call(("lookup", [("id", "7", False)])), thinking=False)
+ self.assertEqual([e.call.arguments for e in evs if e.kind == "tool_call"], [{"id": 7}])
+
+ def test_a_non_string_value_is_json(self):
+ evs = self.run_parser(call(("lookup", [("id", '{"a": [1, 2]}', False)])), thinking=False)
+ self.assertEqual([e.call.arguments for e in evs if e.kind == "tool_call"], [{"id": {"a": [1, 2]}}])
+
+ def test_content_before_the_call_and_none_after(self):
+ evs = self.run_parser("Checking.\n\n" + call(("lookup", [("id", "x", True)]))[2:], thinking=False)
+ self.assertEqual("".join(e.text for e in evs if e.kind == "content").strip(), "Checking.")
+ self.assertEqual(sum(e.kind == "tool_call" for e in evs), 1)
+
+ def test_a_call_cut_off_by_the_length_limit_is_not_a_call(self):
+ raw = call(("get_weather", [("city", "Zagr", True)]))
+ raw = raw[:raw.index("Zagr") + 4]
+ p = DsmlOutputParser(thinking=False, tools=self.TOOLS, stream_tools=True)
+ evs = p.feed(raw) + p.finish("length")
+ self.assertEqual(sum(e.kind == "tool_call" for e in evs), 0)
+
+ def test_the_plain_parser_is_unchanged(self):
+ p = OutputParser(thinking=False, tools=self.TOOLS)
+ evs = p.feed("<tool_call>\n<function=lookup>\n<parameter=id>\n5\n</parameter>\n</function>\n</tool_call>")
+ self.assertEqual([e.call.arguments for e in evs if e.kind == "tool_call"], [{"id": 5}])
+ self.assertTrue(p.clean)
+
+ def test_clean_counts_a_held_tag(self):
+ p = DsmlOutputParser(thinking=False, tools=self.TOOLS)
+ p.feed("hello <|DS")
+ self.assertFalse(p.clean)
+ p.feed("ML| calls>")
+ self.assertTrue(p.clean)
+
+ def test_forced_opening(self):
+ self.assertEqual(to_dsml_force("<tool_call>\n<function=get_weather>\n"),
+ '<|DSML| calls>\n<|DSML| invoke name="get_weather">\n')
+ self.assertEqual(to_dsml_force("<tool_call>\n<function="), '<|DSML| calls>\n<|DSML| invoke name="')
+ self.assertIsNone(to_dsml_force(None))
+ # the opening and the model's continuation are one call
+ p = DsmlOutputParser(thinking=False, tools=self.TOOLS, stream_tools=True)
+ evs = p.feed(to_dsml_force("<tool_call>\n<function=get_weather>\n"))
+ evs += p.feed('<|DSML| parameter name="city" string="true">Rome</|DSML| parameter>\n</|DSML| invoke>\n</|DSML| calls>')
+ evs += p.finish("stop")
+ self.assertEqual([(e.call.name, e.call.arguments) for e in evs if e.kind == "tool_call"],
+ [("get_weather", {"city": "Rome"})])
+
+
+class Api(unittest.TestCase):
+ """Both APIs against a fake engine that replays scripted DeepSeek text."""
+
+ @classmethod
+ def setUpClass(cls):
+ cls.tok = mini_tokenizer()
+
+ def start(self, script):
+ tok = self.tok
+ engine = DsEngine(tok, script)
+ svc = Service(engine, tok, ChatTemplate(TEMPLATE), model_name=default_model_name({"arch": "deepseek41"}))
+ svc.model_arch = "deepseek41"
+ svc.set_family("deepseek41")
+ httpd = serve(svc, port=0)
+ self.addCleanup(httpd.server_close)
+ self.addCleanup(httpd.shutdown)
+ self.addCleanup(frontend.use_family, None)
+ self.engine, self.svc = engine, svc
+ self.base = f"http://127.0.0.1:{httpd.server_address[1]}"
+ return engine
+
+ def post(self, path, body, raw=False):
+ req = urllib.request.Request(self.base + path, data=json.dumps(body).encode(),
+ headers={"Content-Type": "application/json"})
+ try:
+ with urllib.request.urlopen(req, timeout=30) as r:
+ data = r.read().decode()
+ return r.status, (data if raw else json.loads(data))
+ except urllib.error.HTTPError as e:
+ return e.code, json.loads(e.read().decode())
+
+ def prompt(self):
+ return self.tok.decode(self.engine.last_prompt)
+
+ def chat(self, **extra):
+ body = {"messages": [{"role": "user", "content": "hi"}], "max_tokens": 400, **extra}
+ return self.post("/v1/chat/completions", body)
+
+ # -- thinking --------------------------------------------------------------------------------
+ def test_thinking_by_default(self):
+ self.start("The user greets me.</think>\n\nHello!")
+ status, r = self.chat()
+ self.assertEqual(status, 200, r)
+ self.assertTrue(self.prompt().endswith("<|User|>hi<|Assistant|><think>"))
+ msg = r["choices"][0]["message"]
+ self.assertEqual(msg["reasoning_content"], "The user greets me.")
+ self.assertEqual(msg["content"].strip(), "Hello!")
+ self.assertEqual(r["choices"][0]["finish_reason"], "stop")
+
+ def test_thinking_off(self):
+ self.start("Hello!")
+ for extra in ({"chat_template_kwargs": {"enable_thinking": False}}, {"reasoning_effort": "none"}):
+ status, r = self.chat(**extra)
+ self.assertEqual(status, 200, r)
+ self.assertTrue(self.prompt().endswith("<|User|>hi<|Assistant|></think>"), extra)
+ msg = r["choices"][0]["message"]
+ self.assertEqual(msg["content"].strip(), "Hello!")
+ self.assertFalse(msg.get("reasoning_content"))
+
+ def test_the_end_token_is_deepseeks_not_qwens(self):
+ self.start("one<|im_end|>two") # (not a token here: just text, like any other)
+ status, r = self.chat(chat_template_kwargs={"enable_thinking": False})
+ self.assertEqual(r["choices"][0]["message"]["content"], "one<|im_end|>two")
+
+ def test_model_name_and_card_sampling(self):
+ self.start("ok")
+ self.assertEqual(FAMILY_MODEL_NAMES["deepseek41"], "deepseek-v4.1-flash")
+ self.assertEqual(default_model_name({"arch": "deepseek41"}), "deepseek-v4.1-flash")
+ self.assertEqual(model_card_sampling("deepseek41", True), {"temperature": 1.0, "top_p": 1.0})
+ self.chat()
+ self.assertEqual(self.engine.last_sampling.get("temperature"), 1.0)
+
+ # -- effort ------------------------------------------------------------------------------------
+ def test_effort_words(self):
+ self.start("ok")
+ for given, line in (("low", "Reasoning Effort: 50 "), ("medium", "Reasoning Effort: 62 "),
+ ("high", "Reasoning Effort: 75 "), ("max", "Reasoning Effort: 100 "),
+ ("xhigh", "Reasoning Effort: 100 "), (40, "Reasoning Effort: 40 ")):
+ status, r = self.chat(reasoning_effort=given)
+ self.assertEqual(status, 200, (given, r))
+ self.assertIn(line, self.prompt(), given)
+ status, r = self.chat()
+ self.assertIn("Reasoning Effort: 75 ", self.prompt())
+ status, r = self.chat(reasoning_effort="extreme")
+ self.assertEqual(status, 400)
+
+ def test_anthropic_budget_picks_the_effort(self):
+ self.start("ok")
+ status, r = self.post("/v1/messages", {"model": "x", "max_tokens": 300, "thinking": {"type": "enabled", "budget_tokens": 1000},
+ "messages": [{"role": "user", "content": "hi"}]})
+ self.assertEqual(status, 200, r)
+ self.assertIn("Reasoning Effort: 50 ", self.prompt())
+
+ # -- tool calls -------------------------------------------------------------------------------
+ def test_openai_tool_call(self):
+ self.start("Need the weather.</think>" + call(("get_weather", [("city", "Zagreb", True), ("days", "2", False)])))
+ status, r = self.chat(tools=[WEATHER])
+ self.assertEqual(status, 200, r)
+ self.assertIn('"name": "get_weather"', self.prompt())
+ ch = r["choices"][0]
+ self.assertEqual(ch["finish_reason"], "tool_calls")
+ tc = ch["message"]["tool_calls"][0]
+ self.assertEqual(tc["function"]["name"], "get_weather")
+ self.assertEqual(json.loads(tc["function"]["arguments"]), {"city": "Zagreb", "days": 2})
+ self.assertEqual(ch["message"]["reasoning_content"], "Need the weather.")
+
+ def test_two_calls_in_one_turn(self):
+ self.start("</think>" + call(("get_weather", [("city", "A", True)]), ("get_weather", [("city", "B", True)])))
+ status, r = self.chat(tools=[WEATHER])
+ names = [json.loads(c["function"]["arguments"])["city"] for c in r["choices"][0]["message"]["tool_calls"]]
+ self.assertEqual(names, ["A", "B"])
+
+ def test_streamed_tool_call(self):
+ self.start("</think>" + call(("get_weather", [("city", "Zagreb", True)])))
+ status, text = self.post("/v1/chat/completions", {"messages": [{"role": "user", "content": "w?"}], "tools": [WEATHER],
+ "stream": True, "max_tokens": 300}, raw=True)
+ self.assertEqual(status, 200)
+ chunks = [json.loads(x[6:]) for x in text.split("\n\n") if x.startswith("data: ") and x != "data: [DONE]"]
+ calls = [tc for c in chunks for tc in c["choices"][0]["delta"].get("tool_calls", [])]
+ self.assertEqual(calls[0]["function"]["name"], "get_weather")
+ self.assertEqual(json.loads("".join(tc["function"].get("arguments", "") for tc in calls)), {"city": "Zagreb"})
+ self.assertEqual([c["choices"][0]["finish_reason"] for c in chunks if c["choices"][0].get("finish_reason")],
+ ["tool_calls"])
+
+ def test_a_tool_result_turn_renders_for_the_model(self):
+ self.start("The weather is sunny.")
+ msgs = [{"role": "user", "content": "weather in Zagreb?"},
+ {"role": "assistant", "content": None, "tool_calls": [{"id": "c1", "type": "function", "function": {
+ "name": "get_weather", "arguments": "{\"city\": \"Zagreb\"}"}}]},
+ {"role": "tool", "tool_call_id": "c1", "content": "sunny, 21C"}]
+ status, r = self.post("/v1/chat/completions", {"messages": msgs, "tools": [WEATHER], "max_tokens": 200,
+ "chat_template_kwargs": {"enable_thinking": False}})
+ self.assertEqual(status, 200, r)
+ p = self.prompt()
+ self.assertIn('<|DSML| parameter name="city" string="true">Zagreb</|DSML| parameter>', p)
+ self.assertIn("<tool_result>sunny, 21C</tool_result>", p)
+ self.assertEqual(r["choices"][0]["message"]["content"].strip(), "The weather is sunny.")
+
+ def test_forced_tool_choice(self):
+ self.start('<|DSML| parameter name="city" string="true">Rome</|DSML| parameter>\n</|DSML| invoke>\n</|DSML| calls>')
+ status, r = self.chat(tools=[WEATHER], tool_choice={"type": "function", "function": {"name": "get_weather"}},
+ chat_template_kwargs={"enable_thinking": False})
+ self.assertEqual(status, 200, r)
+ self.assertTrue(self.prompt().endswith('<|Assistant|></think><|DSML| calls>\n<|DSML| invoke name="get_weather">\n'), self.prompt()[-120:])
+ tc = r["choices"][0]["message"]["tool_calls"][0]
+ self.assertEqual((tc["function"]["name"], json.loads(tc["function"]["arguments"])), ("get_weather", {"city": "Rome"}))
+
+ # -- anthropic ---------------------------------------------------------------------------------
+ def test_anthropic_tool_use(self):
+ self.start("Weather needed.</think>" + call(("get_weather", [("city", "Oslo", True), ("days", "1", False)])))
+ status, r = self.post("/v1/messages", {"model": "x", "max_tokens": 400,
+ "tools": [{"name": "get_weather", "description": "w", "input_schema":
+ WEATHER["function"]["parameters"]}],
+ "messages": [{"role": "user", "content": "Oslo?"}]})
+ self.assertEqual(status, 200, r)
+ kinds = [b["type"] for b in r["content"]]
+ self.assertEqual(kinds, ["thinking", "tool_use"])
+ self.assertEqual(r["content"][1]["input"], {"city": "Oslo", "days": 1})
+ self.assertEqual(r["stop_reason"], "tool_use")
+
+ def test_anthropic_text_without_thinking(self):
+ self.start("Hello there.")
+ status, r = self.post("/v1/messages", {"model": "x", "max_tokens": 100, "thinking": {"type": "disabled"},
+ "messages": [{"role": "user", "content": "hi"}]})
+ self.assertEqual(status, 200, r)
+ self.assertTrue(self.prompt().endswith("<|Assistant|></think>"))
+ self.assertEqual([b["type"] for b in r["content"]], ["text"])
+ self.assertEqual(r["content"][0]["text"], "Hello there.")
+ self.assertEqual(r["stop_reason"], "end_turn")
+
+ # -- metrics -----------------------------------------------------------------------------------
+ def test_metrics_use_the_one_answer_speed(self):
+ self.start("</think>Hello, how are you today?")
+ self.chat()
+ status = urllib.request.urlopen(self.base + "/metrics", timeout=10)
+ m = json.loads(status.read().decode())
+ last = m["requests"][-1]
+ # produced tokens over the engine's decode time (10 ms a token in the fake), never over the prompt's time
+ self.assertEqual(last["decode_tok_s"], 100.0)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/serve/test_family_name.py b/serve/test_family_name.py
index cd4024ff3..44deb8415 100644
--- a/serve/test_family_name.py
+++ b/serve/test_family_name.py
@@ -19,11 +19,12 @@ class FamilyName(unittest.TestCase):
def test_per_family(self):
self.assertEqual(default_model_name({"arch": "qwen35moe"}), "qwen3.6-35b-a3b")
self.assertEqual(default_model_name({"arch": "qwen4exp"}), "qwen3.8-flash-next")
+ self.assertEqual(default_model_name({"arch": "deepseek41"}), "deepseek-v4.1-flash")
def test_old_configs(self):
self.assertEqual(default_model_name({}), "qwen3.8-flash-next")
self.assertEqual(default_model_name({"arch": "something-else"}), "qwen3.8-flash-next")
- self.assertEqual(set(FAMILY_MODEL_NAMES), {"qwen4exp", "qwen35moe"})
+ self.assertEqual(set(FAMILY_MODEL_NAMES), {"qwen4exp", "qwen35moe", "deepseek41"})
if __name__ == "__main__":
diff --git a/tools/strata_tokenizer.py b/tools/strata_tokenizer.py
index 42a27fed3..33452af86 100644
--- a/tools/strata_tokenizer.py
+++ b/tools/strata_tokenizer.py
@@ -66,6 +66,37 @@ QWEN35_PATTERN = (
)
+# The `deepseek-v3` pre-tokenizer (DeepSeek-V3 / V4.1; llama.cpp LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM, the same regex list as
+# JoyAI and Hunyuan-dense). UNLIKE the single-pattern families it is a SEQUENCE: each pattern splits the pieces the
+# one before it left (a span a pattern does not match stays a piece of its own), so digits (runs of 1..3) and CJK
+# runs are cut out before the general word / punctuation / whitespace pattern sees the rest.
+DEEPSEEK3_PATTERNS = (
+ r"\p{N}{1,3}",
+ "[一-龥぀-ゟ゠-ヿ]+",
+ r"[!\"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~][A-Za-z]+|[^\r\n\p{L}\p{P}\p{S}]?[\p{L}\p{M}]+"
+ r"| ?[\p{P}\p{S}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+",
+)
+
+
+def _sequence_findall(patterns, text: str) -> list[str]:
+ """llama.cpp's unicode_regex_split: the pieces of `text` after each pattern in turn splits every piece into its
+ matches and the unmatched spans between them."""
+ pieces = [text]
+ for pat in patterns:
+ nxt = []
+ for piece in pieces:
+ pos = 0
+ for m in pat.finditer(piece):
+ if m.start() > pos:
+ nxt.append(piece[pos:m.start()])
+ nxt.append(m.group())
+ pos = m.end()
+ if pos < len(piece):
+ nxt.append(piece[pos:])
+ pieces = nxt
+ return pieces
+
+
class Tokenizer:
def __init__(self, tokens: list[str], merges: list[str], token_types: list[int] | None = None,
pre: str = "qwen35", special_ids: dict[str, int] | None = None):
@@ -88,7 +119,12 @@ class Tokenizer:
if parts[0] not in self.ids or parts[1] not in self.ids:
raise ValueError("merge %d names a token outside the vocabulary: %r" % (i, m))
self.ranks[(parts[0], parts[1])] = i
- self._re = regex.compile(QWEN35_PATTERN)
+ if str(pre).startswith("deepseek"):
+ seq = [regex.compile(x) for x in DEEPSEEK3_PATTERNS]
+ self._split = lambda text: _sequence_findall(seq, text)
+ else:
+ self._re = regex.compile(QWEN35_PATTERN)
+ self._split = self._re.findall
# The literals matched directly instead of being run through BPE. GGUF token types: 3 = CONTROL,
# 4 = USER_DEFINED. The two classes behave DIFFERENTLY and llama.cpp's own tokenizer settled which:
@@ -223,7 +259,7 @@ class Tokenizer:
# A piece's ids depend on the piece alone, so repeated pieces (most of a resent conversation) are looked up
# instead of merged again. The ids are the ones _bpe gives: this only skips the work.
cache = self._piece_ids
- for piece in self._re.findall(text):
+ for piece in self._split(text):
got = cache.get(piece)
if got is None:
mapped = "".join(BYTE_TO_UNICODE[b] for b in piece.encode("utf-8"))
@@ -304,6 +340,11 @@ class Tokenizer:
if b is None:
if i < 0 or i >= len(self.tokens):
raise IndexError("token id %d is outside the vocabulary (%d)" % (i, len(self.tokens)))
+ if self.token_types and i < len(self.token_types) and self.token_types[i] in (3, 4):
+ # a special token's string is the literal text (DeepSeek's `<|User|>`, `|DSML|` hold characters outside
+ # the byte alphabet); for Qwen's ASCII specials it is the same bytes the alphabet would give
+ b = cache[i] = self.tokens[i].encode("utf-8")
+ return b
raw = bytearray()
for ch in self.tokens[i]:
v = UNICODE_TO_BYTE.get(ch)