Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
serve: DeepSeek-V4.1 through the server (deepseek-v3 pre-tokenizer, DSML tool calls, effort words, end token, card sampling)
Niko12215a52007005a0
diff --git a/serve/dsml.py b/serve/dsml.pynew file mode 100644index 000000000..c4a1166a5--- /dev/null+++ b/serve/dsml.py@@ -0,0 +1,212 @@+"""serve/dsml.py - DeepSeek-V4.1's tool-call text (DSML) for the server.++DeepSeek-V4.1 writes a tool call as (the model's own chat template, the same bytes as DeepSeek's encoding.py)++ <|DSML| calls>+ <|DSML| invoke name="get_weather">+ <|DSML| parameter name="city" string="true">Zagreb</|DSML| parameter>+ <|DSML| parameter name="days" string="false">3</|DSML| parameter>+ </|DSML| invoke>+ </|DSML| calls>++`|DSML|` is one token (id 128825, a user-defined one, so it is also text when a request is tokenized). String+values are written as they are; every other type is JSON (string="false").++The server's OutputParser (serve/frontend.py) reads the Qwen form `<tool_call><function=NAME><parameter=P>...`, with+its streaming of tool arguments, its calls inside the thinking and its recovery rules. DsmlTranslator rewrites the+DSML text into that form as it streams (holding back only a tag that is still arriving), and DsmlOutputParser is the+OutputParser behind it. So the OpenAI and Anthropic paths, tool_choice, streaming arguments and the thinking/answer+split behave for DeepSeek as they do for the other families, and there is one place that knows the DSML spelling.+"""+from __future__ import annotations++import re++from serve.frontend import OutputParser++DSML = "|DSML|"+_OPEN = "<" + DSML+_CLOSE = "</" + DSML+_TAG_MAX = 600 # a tag (a tool name, a parameter name) longer than this is text, not held any further++_CALLS = re.compile(r"<|DSML|\s*(?:tool_|function_)?calls\s*>")+_CALLS_END = re.compile(r"</|DSML|\s*(?:tool_|function_)?calls\s*>")+_INVOKE = re.compile(r'<|DSML|\s*invoke\s+name\s*=\s*"([^"]*)"\s*>')+_INVOKE_END = re.compile(r"</|DSML|\s*invoke\s*>")+_PARAM = re.compile(r'<|DSML|\s*parameter\s+name\s*=\s*"([^"]*)"(?:\s+string\s*=\s*"(true|false)")?\s*>')+_PARAM_END = re.compile(r"</|DSML|\s*parameter\s*>")+# a tag cut short is held while it can still become one of those: "<", "<|DS", '<|DSML| invoke name="get_we'+_PARTIAL = re.compile(r"^</?(?:|(?:D(?:S(?:M(?:L(?:|[^<>]*)?)?)?)?)?)?$")+++def _could_still_be_a_tag(s: str) -> bool:+ """`s` starts with '<' and has no '>' yet: it may be the start of a DSML tag."""+ return len(s) <= _TAG_MAX and _PARTIAL.match(s) is not None+++class DsmlTranslator:+ """DSML text -> the Qwen tool-call text, a delta at a time. Text outside a call passes through unchanged.++ on_param(tool, name, is_string) is called when a parameter tag is read, before its text is returned, so the parser+ behind it can treat a string="true" value as the string it is."""++ def __init__(self, on_param=None):+ self.buf = ""+ self.state = "text" # text | calls (inside the block) | invoke | param (inside a value)+ self.tool = None+ self.on_param = on_param++ @property+ def held(self) -> str:+ """Text held back because it may be the start of a tag."""+ return self.buf++ def feed(self, delta: str) -> str:+ self.buf += delta+ return self._run(final=False)++ def flush(self) -> str:+ """The end of the output: whatever is held is text."""+ return self._run(final=True)++ # ------------------------------------------------------------------------------------------------+ def _run(self, final: bool) -> str:+ out: list[str] = []+ while self.buf:+ lt = self.buf.find("<")+ if lt < 0:+ self._text(self.buf, out)+ self.buf = ""+ break+ if lt:+ self._text(self.buf[:lt], out)+ self.buf = self.buf[lt:]+ tag = self._tag(out)+ if tag == "wait":+ if final:+ self._text(self.buf, out)+ self.buf = ""+ break+ if tag is None: # an ordinary '<'+ self._text("<", out)+ self.buf = self.buf[1:]+ return "".join(out)++ def _text(self, text: str, out: list[str]) -> None:+ if self.state in ("calls", "invoke"):+ if text.strip() == "":+ return # the layout between tags+ # stray text inside the block: it is shown as text, and the block goes on+ out.append(text)++ def _tag(self, outl: list[str]):+ """Handle the tag at the start of self.buf. Returns True (a tag was consumed), "wait" (it may still become+ one) or None (it is not one)."""+ s = self.buf+ gt = s.find(">")+ if gt < 0:+ return "wait" if _could_still_be_a_tag(s) else None+ head = s[:gt + 1]+ if not (head.startswith(_OPEN) or head.startswith(_CLOSE)):+ return None+ out = outl.append+ m = _PARAM_END.fullmatch(head)+ if m:+ if self.state == "param":+ self.buf = s[gt + 1:]+ self.state = "invoke"+ out("\n</parameter>\n")+ return True+ return None+ m = _INVOKE_END.fullmatch(head) or _CALLS_END.fullmatch(head)+ if m:+ closing_invoke = bool(_INVOKE_END.fullmatch(head))+ if self.state == "param": # the value's closing tag was left out+ out("\n</parameter>\n")+ self.state = "invoke"+ if closing_invoke and self.state == "invoke":+ self.buf = s[gt + 1:]+ self.state = "calls"+ out("</function>\n</tool_call>\n")+ return True+ if not closing_invoke and self.state in ("calls", "invoke"):+ if self.state == "invoke":+ out("</function>\n</tool_call>\n")+ self.buf = s[gt + 1:]+ self.state = "text"+ return True+ return None+ if self.state == "param":+ return None # inside a value only a closing tag means anything+ m = _CALLS.fullmatch(head)+ if m:+ self.buf = s[gt + 1:]+ self.state = "calls"+ return True+ m = _INVOKE.fullmatch(head)+ if m:+ if self.state == "invoke": # the previous call's closing tag was left out+ out("</function>\n</tool_call>\n")+ self.buf = s[gt + 1:]+ self.state, self.tool = "invoke", m.group(1)+ out(f"<tool_call>\n<function={m.group(1)}>\n")+ return True+ m = _PARAM.fullmatch(head)+ if m and self.state in ("calls", "invoke"):+ self.buf = s[gt + 1:]+ self.state = "param"+ if self.on_param is not None:+ self.on_param(self.tool, m.group(1), m.group(2) != "false")+ out(f"<parameter={m.group(1)}>\n")+ return True+ return None+++class DsmlOutputParser(OutputParser):+ """OutputParser for DeepSeek-V4.1: the model's DSML calls are read as the template-form calls they are."""++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ self.dsml = DsmlTranslator(self._on_param)++ def _on_param(self, tool, name, is_string) -> None:+ """string="true": the value is a string whatever it looks like ("123", "true"), as the template wrote it."""+ if not is_string:+ return+ spec = self.schemas.get(tool)+ if spec is None:+ return+ params = dict(spec.get("parameters") or {})+ props = dict(params.get("properties") or {})+ if (props.get(name) or {}).get("type") != "string":+ props[name] = {**(props.get(name) or {}), "type": "string"}+ params["properties"] = props+ self.schemas[tool] = {**spec, "parameters": params}+ if self.scall is not None and self.scall.name == tool:+ self.sdeclared[name] = "string"++ def feed(self, delta: str):+ return super().feed(self.dsml.feed(delta))++ def finish(self, reason: str | None = None):+ rest = self.dsml.flush()+ out = super().feed(rest) if rest else []+ return out + super().finish(reason)++ @property+ def clean(self) -> bool:+ """No tag is held back anywhere: a clean point to stop the thinking at."""+ return not self.buf and not self.dsml.held+++def to_dsml_force(force: str | None) -> str | None:+ """forced_call's opening of a call in the Qwen form -> the same opening in DSML (the model's own spelling)."""+ if not force:+ return force+ head = _OPEN + " calls>\n"+ marker = "<function="+ at = force.find(marker)+ name = force[at + len(marker):] if at >= 0 else ""+ if name.endswith(">\n"):+ return head + f'{_OPEN} invoke name="{name[:-2]}">\n'+ return head + f'{_OPEN} invoke name="{name}'diff --git a/serve/fixtures/deepseek41/chat_template.jinja b/serve/fixtures/deepseek41/chat_template.jinjanew file mode 100644index 000000000..ecbc24b07--- /dev/null+++ b/serve/fixtures/deepseek41/chat_template.jinja@@ -0,0 +1,291 @@+{#- Unsloth port of DS41 #}+{%- set IMG = "<|deepseek_image|>" -%}+{%- set DSML = "|DSML|" -%}++{%- if thinking_mode is not defined or thinking_mode is none -%}+ {%- if enable_thinking is defined and enable_thinking is not none -%}+ {%- set thinking_mode = "thinking" if enable_thinking else "chat" -%}+ {%- elif thinking is defined and thinking is not none and thinking is not string -%}+ {%- set thinking_mode = "thinking" if thinking else "chat" -%}+ {%- else -%}+ {%- set thinking_mode = "thinking" -%}+ {%- endif -%}+{%- endif -%}+{%- if thinking_mode not in ["chat", "thinking"] -%}+ {{- raise_exception("Invalid thinking_mode `" ~ thinking_mode ~ "`, should be \"chat\" or \"thinking\"") -}}+{%- endif -%}+{%- if drop_thinking is not defined or drop_thinking is none -%}{%- set drop_thinking = true -%}{%- endif -%}+{%- if add_generation_prompt is not defined -%}{%- set add_generation_prompt = false -%}{%- endif -%}+{%- if tools is not defined -%}{%- set tools = none -%}{%- endif -%}+{%- if response_format is not defined -%}{%- set response_format = none -%}{%- endif -%}++{%- if reasoning_effort is not defined or reasoning_effort is none -%}{%- set reasoning_effort = "high" -%}{%- endif -%}+{%- set effort_map = {"low": 50, "medium": 62, "high": 75, "max": 100} -%}+{%- if reasoning_effort is string and reasoning_effort in effort_map -%}+ {%- set budget = effort_map[reasoning_effort] -%}+{%- elif reasoning_effort is integer and reasoning_effort >= 1 and reasoning_effort <= 100 -%}+ {%- set budget = reasoning_effort -%}+{%- else -%}+ {{- raise_exception("Invalid reasoning effort: " ~ reasoning_effort ~ ", should be int within [1,100] or one of \"low\", \"medium\", \"high\", \"max\"") -}}+{%- endif -%}++{#- Splits/validates "ns::name" against an explicit namespace and returns the qualified name. -#}+{%- macro qualified_name(name, ns) -%}+ {%- if ns is mapping -%}{%- set ns = ns.name -%}{%- endif -%}+ {%- set parts = name.split("::", 1) -%}+ {%- if parts | length == 2 -%}+ {%- if ns is not none and ns != parts[0] -%}{{- raise_exception("Conflicting tool namespaces: " ~ ns ~ " != " ~ parts[0]) -}}{%- endif -%}+ {%- set ns = parts[0] -%}+ {%- set name = parts[1] -%}+ {%- endif -%}+ {%- if "::" in name -%}{{- raise_exception("Tool name must not contain '::': " ~ name) -}}{%- endif -%}+ {%- if ns is not none and "::" in ns -%}{{- raise_exception("Tool namespace must not contain '::': " ~ ns) -}}{%- endif -%}+ {{- name if ns is none else ns ~ "::" ~ name -}}+{%- endmacro -%}++{%- macro render_tools(tool_list) -%}+ {%- set ts = namespace(schemas=[]) -%}+ {%- for tool in tool_list -%}+ {%- set fn = tool.function if tool.function is defined else tool -%}+ {%- set ns = tool.namespace if tool.namespace is defined and tool.namespace is not none else fn.get("namespace") -%}+ {#- JSON assembled by hand (key order kept, name/description replaced in place): llama.cpp has no ** unpacking. -#}+ {%- set qname = qualified_name(fn.name, ns) -%}+ {%- set sch = namespace(parts=[]) -%}+ {%- for k, v in fn.items() if k != "namespace" -%}+ {%- if k == "name" -%}{%- set v = qname -%}+ {%- elif k == "description" and ns is mapping and ns.description -%}{%- set v = ns.description ~ "\n" ~ (fn.description or "") -%}+ {%- endif -%}+ {%- set sch.parts = sch.parts + [(k | tojson) ~ ": " ~ (v | tojson)] -%}+ {%- endfor -%}+ {%- if "description" not in fn and ns is mapping and ns.description -%}+ {%- set sch.parts = sch.parts + ['"description": ' ~ ((ns.description ~ "\n") | tojson)] -%}+ {%- endif -%}+ {%- set ts.schemas = ts.schemas + ["{" ~ sch.parts | join(", ") ~ "}"] -%}+ {%- endfor -%}+ {{- "## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<|DSML| calls>\" block like the following:\n\n<|DSML| calls>\n<|DSML| invoke name=\"$TOOL_NAME\">\n<|DSML| parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</|DSML| parameter>\n...\n</|DSML| invoke>\n<|DSML| invoke name=\"$TOOL_NAME2\">\n...\n</|DSML| invoke>\n</|DSML| calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n" -}}+ {{- ts.schemas | join("\n") -}}+ {{- "\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n" -}}+{%- endmacro -%}++{#- 1. Merge tool results (and consecutive user turns) into user messages made of content blocks. -#}+{%- set mns = namespace(msgs=[]) -%}+{%- for msg in messages -%}+ {%- set content = msg.get("content") -%}+ {%- set is_blocks = content is iterable and content is not string and content is not mapping -%}+ {%- if content is string and IMG in content -%}+ {{- raise_exception("Message content contains image special token " ~ IMG ~ ". Images should be provided as image content blocks.") -}}+ {%- endif -%}+ {%- set rc = msg.reasoning_content if msg.get("reasoning_content") is not none else msg.get("thinking") -%}+ {%- if rc is string and IMG in rc -%}+ {{- raise_exception("reasoning_content contains image special token " ~ IMG) -}}+ {%- endif -%}+ {%- set prev = mns.msgs[-1] if mns.msgs else none -%}+ {%- if msg.role == "tool" -%}+ {%- set tblock = {"type": "tool_result", "tool_use_id": msg.get("tool_call_id", ""), "content": content if content is not none else ""} -%}+ {%- if prev is not none and prev.role == "user" and prev.content_blocks is defined -%}+ {%- set mns.msgs = mns.msgs[:-1] + [dict(prev, content_blocks=prev.content_blocks + [tblock])] -%}+ {%- else -%}+ {%- set mns.msgs = mns.msgs + [{"role": "user", "content_blocks": [tblock]}] -%}+ {%- endif -%}+ {%- elif msg.role == "user" -%}+ {%- set blocks = msg.content_blocks if msg.get("content_blocks") is not none else (content if is_blocks else [{"type": "text", "text": content or ""}]) -%}+ {%- if prev is not none and prev.role == "user" and prev.content_blocks is defined and prev.get("task") is none -%}+ {%- set mns.msgs = mns.msgs[:-1] + [dict(prev, content_blocks=prev.content_blocks + blocks)] -%}+ {%- else -%}+ {%- set mns.msgs = mns.msgs + [dict(msg, content_blocks=blocks)] -%}+ {%- endif -%}+ {%- else -%}+ {%- set mns.msgs = mns.msgs + [msg] -%}+ {%- endif -%}+{%- endfor -%}++{#- Top-level tools / response_format attach to a leading system message, or to a new empty one. -#}+{%- if tools or response_format -%}+ {%- if not (mns.msgs and mns.msgs[0].role == "system") -%}+ {%- set mns.msgs = [{"role": "system", "content": ""}] + mns.msgs -%}+ {%- endif -%}+ {%- set first = mns.msgs[0] -%}+ {%- if tools and not first.get("tools") -%}{%- set first = dict(first, tools=tools) -%}{%- endif -%}+ {%- if response_format and not first.get("response_format") -%}{%- set first = dict(first, response_format=response_format) -%}{%- endif -%}+ {%- set mns.msgs = [first] + mns.msgs[1:] -%}+{%- endif -%}++{#- 2. Reorder tool results to match the tool_calls order of the preceding assistant message. -#}+{%- set sns = namespace(msgs=[], ids=[]) -%}+{%- for m in mns.msgs -%}+ {%- if m.role == "assistant" and m.get("tool_calls") -%}+ {%- set sns.ids = [] -%}+ {%- for tc in m.tool_calls -%}+ {%- set sns.ids = sns.ids + [tc.get("id") or (tc.function.get("id", "") if tc.function is defined else "")] -%}+ {%- endfor -%}+ {%- set sns.msgs = sns.msgs + [m] -%}+ {%- elif m.role == "user" and m.content_blocks -%}+ {%- set tb = namespace(blocks=[]) -%}+ {%- for b in m.content_blocks if b.type == "tool_result" -%}+ {%- set order = namespace(idx=0) -%}+ {%- for tc_id in sns.ids -%}+ {%- if tc_id and tc_id == b.get("tool_use_id", "") -%}{%- set order.idx = loop.index0 -%}{%- endif -%}+ {%- endfor -%}+ {%- set tb.blocks = tb.blocks + [dict(b, _order=order.idx)] -%}+ {%- endfor -%}+ {%- if tb.blocks | length > 1 and sns.ids -%}+ {%- set sorted_blocks = tb.blocks | sort(attribute="_order") -%}+ {%- set rb = namespace(blocks=[], i=0) -%}+ {%- for b in m.content_blocks -%}+ {%- if b.type == "tool_result" -%}+ {%- set rb.blocks = rb.blocks + [sorted_blocks[rb.i]] -%}+ {%- set rb.i = rb.i + 1 -%}+ {%- else -%}+ {%- set rb.blocks = rb.blocks + [b] -%}+ {%- endif -%}+ {%- endfor -%}+ {%- set sns.msgs = sns.msgs + [dict(m, content_blocks=rb.blocks)] -%}+ {%- else -%}+ {%- set sns.msgs = sns.msgs + [m] -%}+ {%- endif -%}+ {%- else -%}+ {%- set sns.msgs = sns.msgs + [m] -%}+ {%- endif -%}+{%- endfor -%}+{%- set msgs = sns.msgs -%}++{#- Last user turn: users and mid-conversation systems count, except user turns made only of tool results.+ Tools anywhere disable drop_thinking. -#}+{%- set state = namespace(last_user=-1, has_tools=false) -%}+{%- for m in msgs -%}+ {%- if m.role == "user" or (m.role == "system" and loop.index0 > 0) -%}+ {%- set real = namespace(v=true) -%}+ {%- if m.get("content_blocks") -%}+ {%- set real.v = false -%}+ {%- for b in m.content_blocks if b.type in ["text", "image", "image_url"] -%}{%- set real.v = true -%}{%- endfor -%}+ {%- endif -%}+ {%- if real.v -%}{%- set state.last_user = loop.index0 -%}{%- endif -%}+ {%- endif -%}+ {%- if m.get("tools") -%}{%- set state.has_tools = true -%}{%- endif -%}+{%- endfor -%}+{%- set effective_drop = drop_thinking and not state.has_tools -%}+{%- set is_thinking = thinking_mode == "thinking" -%}+{#- Like encoding.py, thinking mode with drop_thinking silently drops messages of other roles before the last user turn. -#}+{%- if is_thinking and effective_drop -%}+ {%- set kept = namespace(msgs=[]) -%}+ {%- for m in msgs -%}+ {%- if m.role in ["user", "system", "tool", "latest_reminder", "direct_search_results", "assistant"] or loop.index0 >= state.last_user -%}+ {%- set kept.msgs = kept.msgs + [m] -%}+ {%- endif -%}+ {%- endfor -%}+ {%- if kept.msgs | length != msgs | length -%}+ {%- set msgs = kept.msgs -%}+ {%- set state.last_user = -1 -%}+ {%- for m in msgs -%}+ {%- if m.role == "user" or (m.role == "system" and loop.index0 > 0) -%}+ {%- set real = namespace(v=true) -%}+ {%- if m.get("content_blocks") -%}+ {%- set real.v = false -%}+ {%- for b in m.content_blocks if b.type in ["text", "image", "image_url"] -%}{%- set real.v = true -%}{%- endfor -%}+ {%- endif -%}+ {%- if real.v -%}{%- set state.last_user = loop.index0 -%}{%- endif -%}+ {%- endif -%}+ {%- endfor -%}+ {%- endif -%}+{%- endif -%}++{{- "<|begin▁of▁sentence|>" -}}+{#- 3. Render each message. -#}+{%- for m in msgs -%}+ {%- set idx = loop.index0 -%}+ {%- set content = m.get("content") -%}+ {%- if content is iterable and content is not string and content is not mapping -%}+ {%- set sp = namespace(parts=[]) -%}+ {%- for b in content -%}+ {%- if b.type == "text" -%}{%- set sp.parts = sp.parts + [b.text or ""] -%}+ {%- elif b.type in ["image", "image_url"] -%}{%- set sp.parts = sp.parts + [IMG] -%}+ {%- endif -%}+ {%- endfor -%}+ {%- set content = sp.parts | join("\n\n") -%}+ {%- endif -%}++ {%- if idx == 0 -%}+ {%- if is_thinking or m.role == "system" -%}{{- "<|System|>" -}}{%- endif -%}+ {%- if is_thinking -%}{{- "Reasoning Effort: " ~ budget ~ " (range 1-100, the higher the value, the more thorough the reasoning)\n\n" -}}{%- endif -%}+ {%- endif -%}++ {%- if m.role == "system" -%}+ {%- if idx > 0 -%}{{- "<|System|>" -}}{%- endif -%}+ {{- content or "" -}}+ {%- if m.get("tools") -%}{{- "\n\n" ~ render_tools(m.tools) -}}{%- endif -%}+ {%- if m.get("response_format") -%}+ {{- "\n\n## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n" ~ (m.response_format | tojson) -}}+ {%- endif -%}++ {%- elif m.role == "user" -%}+ {{- "<|User|>" -}}+ {%- set up = namespace(parts=[]) -%}+ {%- for b in m.content_blocks -%}+ {%- if b.type == "text" -%}+ {%- if IMG in (b.text or "") -%}{{- raise_exception("Text block contains image placeholder " ~ IMG ~ ". Images should be separate content blocks.") -}}{%- endif -%}+ {%- set up.parts = up.parts + [b.text or ""] -%}+ {%- elif b.type in ["image", "image_url"] -%}+ {%- set up.parts = up.parts + [IMG] -%}+ {%- elif b.type == "tool_result" -%}+ {%- set tc = b.get("content", "") -%}+ {%- if tc is iterable and tc is not string and tc is not mapping -%}+ {%- set sub = namespace(parts=[]) -%}+ {%- for s in tc -%}+ {%- if s.type == "text" -%}+ {%- if IMG in (s.text or "") -%}{{- raise_exception("Text block contains image placeholder " ~ IMG ~ ". Images should be separate content blocks.") -}}{%- endif -%}+ {%- set sub.parts = sub.parts + [s.get("text", "")] -%}+ {%- elif s.type in ["image", "image_url"] -%}{%- set sub.parts = sub.parts + [IMG] -%}+ {%- else -%}{%- set sub.parts = sub.parts + ["[Unsupported " ~ s.type ~ "]"] -%}+ {%- endif -%}+ {%- endfor -%}+ {%- set tc = sub.parts | join("\n\n") -%}+ {%- endif -%}+ {%- set up.parts = up.parts + ["<tool_result>" ~ tc ~ "</tool_result>"] -%}+ {%- else -%}+ {%- set up.parts = up.parts + ["[Unsupported " ~ b.type ~ "]"] -%}+ {%- endif -%}+ {%- endfor -%}+ {{- up.parts | join("\n\n") -}}++ {%- elif m.role == "latest_reminder" -%}+ {{- "<|latest_reminder|>" ~ (content or "") -}}++ {%- elif m.role == "assistant" -%}+ {%- set prev_has_task = idx > 0 and msgs[idx - 1].get("task") is not none -%}+ {%- if is_thinking and not prev_has_task and (not effective_drop or idx > state.last_user) -%}+ {{- ((m.reasoning_content if m.get("reasoning_content") is not none else m.get("thinking")) or "") ~ "</think>" -}}+ {%- endif -%}+ {{- content or "" -}}+ {%- if m.get("tool_calls") -%}+ {%- set tcs = namespace(lines=[]) -%}+ {%- for tc in m.tool_calls -%}+ {%- set fn = tc.function if tc.function is defined else tc -%}+ {%- set params = namespace(lines=[]) -%}+ {%- set args = fn.arguments if fn.arguments is mapping else {"arguments": fn.arguments} -%}+ {%- for k, v in args.items() -%}+ {%- set params.lines = params.lines + ["<" ~ DSML ~ " parameter name=\"" ~ k ~ "\" string=\"" ~ ("true" if v is string else "false") ~ "\">" ~ (v if v is string else v | tojson) ~ "</" ~ DSML ~ " parameter>"] -%}+ {%- endfor -%}+ {%- set name = qualified_name(fn.name, tc.get("namespace") or fn.get("namespace")) -%}+ {%- set tcs.lines = tcs.lines + ["<" ~ DSML ~ " invoke name=\"" ~ name ~ "\">\n" ~ params.lines | join("\n") ~ "\n</" ~ DSML ~ " invoke>"] -%}+ {%- endfor -%}+ {{- "\n\n<" ~ DSML ~ " calls>\n" ~ tcs.lines | join("\n") ~ "\n</" ~ DSML ~ " calls>" -}}+ {%- endif -%}+ {%- if not m.get("wo_eos", false) -%}{{- "<|end▁of▁sentence|>" -}}{%- endif -%}++ {%- else -%}+ {{- raise_exception("Unknown role: " ~ m.role) -}}+ {%- endif -%}++ {#- Transition tokens, only when the next message is an assistant/latest_reminder turn or this is the last one. -#}+ {%- if loop.last or msgs[idx + 1].role in ["assistant", "latest_reminder"] -%}+ {%- set task = m.get("task") -%}+ {%- if task is not none -%}+ {%- set task_tokens = {"action": "<|action|>", "query": "<|query|>", "authority": "<|authority|>", "domain": "<|domain|>", "title": "<|title|>", "read_url": "<|read_url|>"} -%}+ {%- if task not in task_tokens -%}{{- raise_exception("Invalid task: '" ~ task ~ "'") -}}{%- endif -%}+ {%- if task == "action" -%}{{- "<|Assistant|>" ~ ("<think>" if is_thinking else "</think>") -}}{%- endif -%}+ {{- task_tokens[task] -}}+ {%- elif (m.role == "user" or (m.role == "system" and idx > 0)) and (add_generation_prompt or not loop.last) -%}+ {{- "<|Assistant|>" ~ ("<think>" if is_thinking and (not effective_drop or idx >= state.last_user) else "</think>") -}}+ {%- endif -%}+ {%- endif -%}+{%- endfor -%}diff --git a/serve/fixtures/deepseek41/golden.json b/serve/fixtures/deepseek41/golden.jsonnew file mode 100644index 000000000..01276d80f--- /dev/null+++ b/serve/fixtures/deepseek41/golden.json@@ -0,0 +1,965 @@+[+ {+ "id": "sanity-I1",+ "mode": "chat",+ "request": {+ "messages": [+ {+ "role": "user",+ "content": "[case sanity-I1]\nReturn only a JSON object with ok set to true and count set to the integer 3. No other keys."+ }+ ],+ "chat_template_kwargs": {+ "thinking": false,+ "enable_thinking": false+ }+ },+ "thinking": false,+ "text": "<|begin▁of▁sentence|><|User|>[case sanity-I1]\nReturn only a JSON object with ok set to true and count set to the integer 3. No other keys.<|Assistant|></think>",+ "ids": [+ 0,+ 128803,+ 61,+ 11675,+ 102630,+ 20448,+ 19,+ 2296,+ 25529,+ 1353,+ 260,+ 26639,+ 2873,+ 418,+ 9109,+ 1341,+ 304,+ 3292,+ 305,+ 1895,+ 1341,+ 304,+ 270,+ 9132,+ 223,+ 21,+ 16,+ 3011,+ 915,+ 18222,+ 16,+ 128804,+ 128822+ ],+ "expect_text": "{\"ok\": true, \"count\": 3}<|end▁of▁sentence|>",+ "finish": "stop"+ },+ {+ "id": "sanity-I1",+ "mode": "thinking",+ "request": {+ "messages": [+ {+ "role": "user",+ "content": "[case sanity-I1]\nReturn only a JSON object with ok set to true and count set to the integer 3. No other keys."+ }+ ]+ },+ "thinking": true,+ "text": "<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>[case sanity-I1]\nReturn only a JSON object with ok set to true and count set to the integer 3. No other keys.<|Assistant|><think>",+ "ids": [+ 0,+ 128799,+ 77666,+ 288,+ 9504,+ 482,+ 28,+ 223,+ 2402,+ 343,+ 16801,+ 223,+ 19,+ 15,+ 1457,+ 14,+ 270,+ 4077,+ 270,+ 1990,+ 14,+ 270,+ 850,+ 15146,+ 270,+ 22805,+ 868,+ 128803,+ 61,+ 11675,+ 102630,+ 20448,+ 19,+ 2296,+ 25529,+ 1353,+ 260,+ 26639,+ 2873,+ 418,+ 9109,+ 1341,+ 304,+ 3292,+ 305,+ 1895,+ 1341,+ 304,+ 270,+ 9132,+ 223,+ 21,+ 16,+ 3011,+ 915,+ 18222,+ 16,+ 128804,+ 128821+ ],+ "expect_text": "We need answer only JSON object with ok true and count integer 3. No other keys. Need comply.</think>{\"ok\":true,\"count\":3}<|end▁of▁sentence|>",+ "finish": "stop"+ },+ {+ "id": "sanity-T1",+ "mode": "chat",+ "request": {+ "messages": [+ {+ "role": "user",+ "content": "[case sanity-T1]\nUse lookup_record to look up record A-17."+ }+ ],+ "tools": [+ {+ "type": "function",+ "function": {+ "name": "lookup_record",+ "description": "Synthetic benchmark lookup; no external action is performed.",+ "parameters": {+ "type": "object",+ "properties": {+ "id": {+ "type": "string"+ }+ },+ "required": [+ "id"+ ],+ "additionalProperties": false+ }+ }+ }+ ],+ "chat_template_kwargs": {+ "thinking": false,+ "enable_thinking": false+ }+ },+ "thinking": false,+ "text": "<|begin▁of▁sentence|><|System|>\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<|DSML| calls>\" block like the following:\n\n<|DSML| calls>\n<|DSML| invoke name=\"$TOOL_NAME\">\n<|DSML| parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</|DSML| parameter>\n...\n</|DSML| invoke>\n<|DSML| invoke name=\"$TOOL_NAME2\">\n...\n</|DSML| invoke>\n</|DSML| calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"name\": \"lookup_record\", \"description\": \"Synthetic benchmark lookup; no external action is performed.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"id\": {\"type\": \"string\"}}, \"required\": [\"id\"], \"additionalProperties\": false}}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<|User|>[case sanity-T1]\nUse lookup_record to look up record A-17.<|Assistant|></think>",+ "ids": [+ 0,+ 128799,+ 271,+ 372,+ 27193,+ 271,+ 3476,+ 611,+ 3278,+ 304,+ 260,+ 1341,+ 294,+ 6704,+ 304,+ 1694,+ 3287,+ 270,+ 3967,+ 734,+ 3417,+ 16,+ 2042,+ 588,+ 34756,+ 6704,+ 513,+ 4985,+ 260,+ 38322,+ 128825,+ 10699,+ 48902,+ 5603,+ 1277,+ 270,+ 2502,+ 979,+ 30,+ 128825,+ 10699,+ 1018,+ 30,+ 128825,+ 34756,+ 2329,+ 84476,+ 6792,+ 5896,+ 35044,+ 3816,+ 30,+ 128825,+ 10767,+ 2329,+ 84476,+ 46630,+ 4130,+ 61729,+ 35044,+ 4,+ 3418,+ 1281,+ 11476,+ 94,+ 19836,+ 3320,+ 6,+ 46630,+ 4130,+ 61729,+ 56909,+ 1718,+ 128825,+ 10767,+ 1018,+ 7835,+ 1718,+ 128825,+ 34756,+ 1018,+ 30,+ 128825,+ 34756,+ 2329,+ 84476,+ 6792,+ 5896,+ 35044,+ 20,+ 3816,+ 7835,+ 1718,+ 128825,+ 34756,+ 1018,+ 1718,+ 128825,+ 10699,+ 4697,+ 3524,+ 8252,+ 1531,+ 366,+ 12038,+ 412,+ 344,+ 305,+ 1341,+ 3608,+ 4463,+ 1281,+ 11476,+ 4,+ 37419,+ 1884,+ 710,+ 915,+ 4815,+ 343,+ 62896,+ 14,+ 2631,+ 2065,+ 634,+ 14,+ 31939,+ 14,+ 8435,+ 754,+ 2281,+ 270,+ 1990,+ 295,+ 26639,+ 8786,+ 305,+ 1341,+ 3608,+ 4463,+ 1281,+ 19836,+ 4,+ 108526,+ 3575,+ 6892,+ 71144,+ 344,+ 22104,+ 343,+ 89944,+ 284,+ 513,+ 223,+ 128821,+ 754,+ 440,+ 74366,+ 5238,+ 782,+ 5553,+ 22805,+ 6352,+ 223,+ 128821,+ 1613,+ 128822,+ 119907,+ 1117,+ 4105,+ 10699,+ 469,+ 4087,+ 4256,+ 339,+ 13079,+ 6922,+ 14,+ 5238,+ 6578,+ 1561,+ 223,+ 128822,+ 418,+ 4105,+ 10699,+ 469,+ 4087,+ 4256,+ 339,+ 795,+ 17829,+ 28249,+ 13178,+ 8380,+ 271,+ 24313,+ 2852,+ 3362,+ 582,+ 12533,+ 1425,+ 119326,+ 1760,+ 582,+ 20855,+ 3362,+ 582,+ 53,+ 48819,+ 35312,+ 64018,+ 29,+ 1119,+ 9390,+ 4271,+ 344,+ 7451,+ 66910,+ 582,+ 46172,+ 3362,+ 28612,+ 4611,+ 3362,+ 582,+ 10325,+ 1760,+ 582,+ 68838,+ 3362,+ 28612,+ 329,+ 3362,+ 28612,+ 4611,+ 3362,+ 582,+ 4463,+ 4,+ 55695,+ 582,+ 24486,+ 3362,+ 20584,+ 329,+ 45390,+ 582,+ 102970,+ 34520,+ 3362,+ 5062,+ 33236,+ 3476,+ 74366,+ 29851,+ 1605,+ 270,+ 3554,+ 6428,+ 4105,+ 2329,+ 305,+ 10767,+ 5815,+ 8380,+ 304,+ 34756,+ 4105,+ 10699,+ 603,+ 128803,+ 61,+ 11675,+ 102630,+ 8139,+ 19,+ 2296,+ 14885,+ 64018,+ 119326,+ 304,+ 1747,+ 890,+ 3676,+ 334,+ 15,+ 1002,+ 16,+ 128804,+ 128822+ ],+ "expect_text": "\n\n<|DSML| calls>\n<|DSML| invoke name=\"lookup_record\">\n<|DSML| parameter name=\"id\" string=\"true\">A-17</|DSML| parameter>\n</|DSML| invoke>\n</|DSML| calls><|end▁of▁sentence|>",+ "finish": "tool_calls"+ },+ {+ "id": "sanity-T2",+ "mode": "thinking",+ "request": {+ "messages": [+ {+ "role": "user",+ "content": "[case sanity-T2]\nUse get_weather for Zagreb in celsius."+ }+ ],+ "tools": [+ {+ "type": "function",+ "function": {+ "name": "get_weather",+ "description": "Synthetic benchmark lookup; no external action is performed.",+ "parameters": {+ "type": "object",+ "properties": {+ "city": {+ "type": "string"+ },+ "unit": {+ "type": "string"+ }+ },+ "required": [+ "city",+ "unit"+ ],+ "additionalProperties": false+ }+ }+ }+ ]+ },+ "thinking": true,+ "text": "<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<|DSML| calls>\" block like the following:\n\n<|DSML| calls>\n<|DSML| invoke name=\"$TOOL_NAME\">\n<|DSML| parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</|DSML| parameter>\n...\n</|DSML| invoke>\n<|DSML| invoke name=\"$TOOL_NAME2\">\n...\n</|DSML| invoke>\n</|DSML| calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"name\": \"get_weather\", \"description\": \"Synthetic benchmark lookup; no external action is performed.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}, \"unit\": {\"type\": \"string\"}}, \"required\": [\"city\", \"unit\"], \"additionalProperties\": false}}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<|User|>[case sanity-T2]\nUse get_weather for Zagreb in celsius.<|Assistant|><think>",+ "ids": [+ 0,+ 128799,+ 77666,+ 288,+ 9504,+ 482,+ 28,+ 223,+ 2402,+ 343,+ 16801,+ 223,+ 19,+ 15,+ 1457,+ 14,+ 270,+ 4077,+ 270,+ 1990,+ 14,+ 270,+ 850,+ 15146,+ 270,+ 22805,+ 86262,+ 372,+ 27193,+ 271,+ 3476,+ 611,+ 3278,+ 304,+ 260,+ 1341,+ 294,+ 6704,+ 304,+ 1694,+ 3287,+ 270,+ 3967,+ 734,+ 3417,+ 16,+ 2042,+ 588,+ 34756,+ 6704,+ 513,+ 4985,+ 260,+ 38322,+ 128825,+ 10699,+ 48902,+ 5603,+ 1277,+ 270,+ 2502,+ 979,+ 30,+ 128825,+ 10699,+ 1018,+ 30,+ 128825,+ 34756,+ 2329,+ 84476,+ 6792,+ 5896,+ 35044,+ 3816,+ 30,+ 128825,+ 10767,+ 2329,+ 84476,+ 46630,+ 4130,+ 61729,+ 35044,+ 4,+ 3418,+ 1281,+ 11476,+ 94,+ 19836,+ 3320,+ 6,+ 46630,+ 4130,+ 61729,+ 56909,+ 1718,+ 128825,+ 10767,+ 1018,+ 7835,+ 1718,+ 128825,+ 34756,+ 1018,+ 30,+ 128825,+ 34756,+ 2329,+ 84476,+ 6792,+ 5896,+ 35044,+ 20,+ 3816,+ 7835,+ 1718,+ 128825,+ 34756,+ 1018,+ 1718,+ 128825,+ 10699,+ 4697,+ 3524,+ 8252,+ 1531,+ 366,+ 12038,+ 412,+ 344,+ 305,+ 1341,+ 3608,+ 4463,+ 1281,+ 11476,+ 4,+ 37419,+ 1884,+ 710,+ 915,+ 4815,+ 343,+ 62896,+ 14,+ 2631,+ 2065,+ 634,+ 14,+ 31939,+ 14,+ 8435,+ 754,+ 2281,+ 270,+ 1990,+ 295,+ 26639,+ 8786,+ 305,+ 1341,+ 3608,+ 4463,+ 1281,+ 19836,+ 4,+ 108526,+ 3575,+ 6892,+ 71144,+ 344,+ 22104,+ 343,+ 89944,+ 284,+ 513,+ 223,+ 128821,+ 754,+ 440,+ 74366,+ 5238,+ 782,+ 5553,+ 22805,+ 6352,+ 223,+ 128821,+ 1613,+ 128822,+ 119907,+ 1117,+ 4105,+ 10699,+ 469,+ 4087,+ 4256,+ 339,+ 13079,+ 6922,+ 14,+ 5238,+ 6578,+ 1561,+ 223,+ 128822,+ 418,+ 4105,+ 10699,+ 469,+ 4087,+ 4256,+ 339,+ 795,+ 17829,+ 28249,+ 13178,+ 8380,+ 271,+ 24313,+ 2852,+ 3362,+ 582,+ 1133,+ 65,+ 50219,+ 1760,+ 582,+ 20855,+ 3362,+ 582,+ 53,+ 48819,+ 35312,+ 64018,+ 29,+ 1119,+ 9390,+ 4271,+ 344,+ 7451,+ 66910,+ 582,+ 46172,+ 3362,+ 28612,+ 4611,+ 3362,+ 582,+ 10325,+ 1760,+ 582,+ 68838,+ 3362,+ 28612,+ 37399,+ 3362,+ 28612,+ 4611,+ 3362,+ 582,+ 4463,+ 87928,+ 582,+ 15165,+ 3362,+ 28612,+ 4611,+ 3362,+ 582,+ 4463,+ 4,+ 55695,+ 582,+ 24486,+ 3362,+ 20584,+ 37399,+ 1760,+ 582,+ 15165,+ 45390,+ 582,+ 102970,+ 34520,+ 3362,+ 5062,+ 33236,+ 3476,+ 74366,+ 29851,+ 1605,+ 270,+ 3554,+ 6428,+ 4105,+ 2329,+ 305,+ 10767,+ 5815,+ 8380,+ 304,+ 34756,+ 4105,+ 10699,+ 603,+ 128803,+ 61,+ 11675,+ 102630,+ 8139,+ 20,+ 2296,+ 14885,+ 1178,+ 65,+ 50219,+ 362,+ 121648,+ 295,+ 124281,+ 16,+ 128804,+ 128821+ ],+ "expect_text": "The user wants weather for Zagreb in celsius. Call the tool.</think>\n\n<|DSML| calls>\n<|DSML| invoke name=\"get_weather\">\n<|DSML| parameter name=\"city\" string=\"true\">Zagreb</|DSML| parameter>\n<|DSML| parameter name=\"unit\" string=\"true\">celsius</|DSML| parameter>\n</|DSML| invoke>\n</|DSML| calls><|end▁of▁sentence|>",+ "finish": "tool_calls"+ },+ {+ "id": "sanity-I3",+ "mode": "thinking",+ "request": {+ "messages": [+ {+ "role": "user",+ "content": "[case sanity-I3]\nProject: Orion. Build: 17. Return only a JSON object with keys project (string) and build (integer)."+ }+ ]+ },+ "thinking": true,+ "text": "<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>[case sanity-I3]\nProject: Orion. Build: 17. Return only a JSON object with keys project (string) and build (integer).<|Assistant|><think>",+ "ids": [+ 0,+ 128799,+ 77666,+ 288,+ 9504,+ 482,+ 28,+ 223,+ 2402,+ 343,+ 16801,+ 223,+ 19,+ 15,+ 1457,+ 14,+ 270,+ 4077,+ 270,+ 1990,+ 14,+ 270,+ 850,+ 15146,+ 270,+ 22805,+ 868,+ 128803,+ 61,+ 11675,+ 102630,+ 20448,+ 21,+ 2296,+ 20811,+ 28,+ 84706,+ 16,+ 23718,+ 28,+ 223,+ 1002,+ 16,+ 20870,+ 1353,+ 260,+ 26639,+ 2873,+ 418,+ 18222,+ 2775,+ 343,+ 4463,+ 11,+ 305,+ 2906,+ 343,+ 39648,+ 797,+ 128804,+ 128821+ ],+ "expect_text": "We need answer only JSON object keys project string build integer. Need infer from prompt: Project: Orion. Build: 17. So {\"project\":\"Orion\",\"build\":17}. Ensure only JSON.</think>{\"project\":\"Orion\",\"build\":17}<|end▁of▁sentence|>",+ "finish": "stop"+ },+ {+ "id": "performance-short-r1",+ "mode": "chat",+ "request": {+ "messages": [+ {+ "role": "user",+ "content": "[case performance-short-r1]\nExplain how a hash table finds a value, including collisions and one small example."+ }+ ],+ "chat_template_kwargs": {+ "thinking": false,+ "enable_thinking": false+ }+ },+ "thinking": false,+ "text": "<|begin▁of▁sentence|><|User|>[case performance-short-r1]\nExplain how a hash table finds a value, including collisions and one small example.<|Assistant|></think>",+ "ids": [+ 0,+ 128803,+ 61,+ 11675,+ 4197,+ 10565,+ 482,+ 9954,+ 19,+ 2296,+ 65106,+ 1192,+ 260,+ 19657,+ 4184,+ 15249,+ 260,+ 1990,+ 14,+ 2622,+ 54415,+ 305,+ 834,+ 2395,+ 2511,+ 16,+ 128804,+ 128822+ ],+ "expect_text": "# How a Hash Table Finds a Value\n\nA hash table stores key–value pairs and uses a **hash function** to map each key to an index in an underlying array. Lookup is fast because it jumps almost directly to the right slot instead of scanning.\n\n## The lookup process\n\n1. **Hash the key.** Compute `h = hash(key)`, then reduce it to a valid array index, e.g. `index = h % capacity`.\n2. **Go to that bucket.** The array slot at `index` holds the entry (or a chain/list of entries).\n3. **Compare keys.** If the bucket holds the exact key you asked for, return its value.\n4. **Handle collisions.** If the bucket holds a *different* key that happened to hash to the same index, keep searching within that bucket (see below).\n\n## Collisions\n\nA **collision** happens when two different keys map to the same index. Two common resolutions:\n\n- **Separate chaining:** each bucket is a small list (or tree). On lookup, walk the list comparing keys until you find a match or reach the end (key absent).\n- **Open addressing:** if the slot is occupied by a different key, probe other slots (linear probing:",+ "finish": "length"+ }+]\ No newline at end of filediff --git a/serve/frontend.py b/serve/frontend.pyindex 3fda9c56f..33f4bbd6e 100644--- a/serve/frontend.py+++ b/serve/frontend.py@@ -135,15 +135,36 @@ EFFORT = {"none": None, "off": None, "minimal": None, "disabled": None, "false":"low": "low", "medium": "medium", "high": "xhigh", "xhigh": "xhigh", "max": "xhigh", "maximum": "xhigh"}+# DeepSeek-V4.1's template takes low, medium, high (its default) and max, or a whole number 1..100 (the "Reasoning+# Effort: N" line of its prompt). use_family("deepseek41") switches the table; the other families keep the one above.+EFFORT_DEEPSEEK = {"none": None, "off": None, "minimal": None, "disabled": None, "false": None,+ "low": "low", "medium": "medium", "high": "high", "xhigh": "max", "max": "max", "maximum": "max"}+_EFFORT_QWEN = dict(EFFORT)+_NUMERIC_EFFORT = False+++def use_family(family: str | None) -> None:+ """The reasoning-effort words of the model family this server runs (module state, like ENV_TO_USER)."""+ global _NUMERIC_EFFORT+ EFFORT.clear()+ EFFORT.update(EFFORT_DEEPSEEK if family == "deepseek41" else _EFFORT_QWEN)+ _NUMERIC_EFFORT = family == "deepseek41"++def effort_kwargs(value) -> dict:"""A reasoning effort as given by a client -> the template's kwargs. Unknown values are a 400, not a crash."""if value is None or value == "":return {}if value is False:return {"enable_thinking": False}+ if _NUMERIC_EFFORT and isinstance(value, int) and not isinstance(value, bool):+ if not 1 <= value <= 100:+ raise ValueError(f"unknown reasoning effort {value!r}: use none, low, medium, high, max or 1..100")+ return {"reasoning_effort": value}key = str(value).strip().lower()if key not in EFFORT:- raise ValueError(f"unknown reasoning effort {value!r}: use none, low, medium or high")+ raise ValueError(f"unknown reasoning effort {value!r}: use none, low, medium or high"+ + (", max or 1..100" if _NUMERIC_EFFORT else ""))level = EFFORT[key]return {"enable_thinking": False} if level is None else {"reasoning_effort": level}@@ -154,7 +175,7 @@ def budget_effort(tokens) -> dict:n = int(tokens)except (TypeError, ValueError):return {}- return {"reasoning_effort": "low" if n < 2048 else "medium" if n < 8192 else "xhigh"}+ return {"reasoning_effort": "low" if n < 2048 else "medium" if n < 8192 else EFFORT["high"]}def _has_image(content) -> bool:@@ -905,6 +926,12 @@ class OutputParser:self.batch = False # the call being finished is followed by another in the same wrapperself._reset_scan()+ @property+ def clean(self) -> bool:+ """Nothing is held back: a clean point to stop the thinking at (a parser that rewrites the model's text, as+ serve/dsml.py's does, also counts what it holds)."""+ return not self.buf+def _ok_at(self, p: int) -> bool:"""self.buf[p] would open a call: at the start of a line, outside a code fence and inline code."""snap = (self.fence, self.line, self.ticks)diff --git a/serve/server.py b/serve/server.pyindex 0635a715f..f41d96dfc 100644--- a/serve/server.py+++ b/serve/server.py@@ -3583,6 +3583,8 @@ class Service:self.restore_after_load = None # a session file an image job saved before it unloadedself.stop_ids = set(tokenizer.encode(IM_END, parse_special=True) +tokenizer.encode("<|endoftext|>", parse_special=True))+ self.family = None # the model family ("deepseek41"): set_family() below+ self.parser_cls = OutputParser # reads the model's output (reasoning, content, tool calls)# #567: a prompt re-encodes only what follows the last special token it shares with a recent prompt (the# same ids as a full encode: tools/strata_tokenizer.py PromptEncoder). Tokenizers without resume points# encode in full.@@ -3595,6 +3597,25 @@ class Service:self.prompts = PromptEncoder(tokenizer, keep=slots) if slots is not None and slots > 0 \else PromptEncoder(tokenizer)+ def set_family(self, family: str | None) -> None:+ """A family whose text differs from the Qwen form this server was written for (DeepSeek-V4.1: its own end+ token, DSML tool calls, reasoning-effort words). The others keep the defaults."""+ self.family = family+ if family == "deepseek41":+ from serve import dsml+ eos = (getattr(self.tok, "special_ids", None) or {}).get("eos", 1)+ self.stop_ids = {int(eos)}+ self.parser_cls = dsml.DsmlOutputParser+ self._force_text = dsml.to_dsml_force+ import serve.frontend as _frontend+ _frontend.use_family(family)++ _force_text = None++ def force_text(self, force):+ """forced_call's opening of the call the reply must make, in this model's spelling."""+ return self._force_text(force) if self._force_text is not None else force+def loaded(self) -> bool:return not hasattr(self.engine, "alive") or self.engine.alive()@@ -4718,7 +4739,7 @@ class Service:print(f"[strata] thinking capped at {max_new - reserve} of max_tokens {max_new} to leave room for "f"the answer (budget {budget})", flush=True)budget = max(1, max_new - reserve)- parser = OutputParser(thinking=thinking, tools=tools, stream_tools=True, recover=self.tool_call_recovery)+ parser = self.parser_cls(thinking=thinking, tools=tools, stream_tools=True, recover=self.tool_call_recovery)detok, n, finish = Detokenizer(self.tok), 0, "length"run_tok, run_len, repeated = None, 0, False # #606: the current run of one repeated tokenthinking_n = 0 # tokens written while thinking (Responses' reasoning_tokens)@@ -4943,13 +4964,13 @@ class Service:if budget and parser.state == "reasoning":thought += 1# at a clean point: no tag held back, no character split across tokens- if thought >= budget and not parser.buf and not detok.pending():+ if thought >= budget and parser.clean and not detok.pending():wrap = Truebreakif force:if any(ev.kind in ("tool_start", "tool_call") for ev in evs):force = None # the model is writing a call of its own- elif parser.state == "content" and not parser.buf and not detok.pending():+ elif parser.state == "content" and parser.clean and not detok.pending():opens = Truebreakexcept EngineNonFinite:@@ -6731,7 +6752,7 @@ def make_handler(svc: Service):self._no_local_images(messages)if tool_choice_of(req.get("tool_choice"))[0] == "none": # as the Responses route: no tools are offeredtools = None- force = forced_call(req.get("tool_choice"), tools) # a bad value is a 400 before anything is sent+ force = svc.force_text(forced_call(req.get("tool_choice"), tools)) # a bad value is a 400 before anything is sentmessages, validator = prepare_format(req.get("response_format"), messages)if validator is not None and (tools or req.get("strata_mcp")):raise ValueError("structured response_format with tools/MCP is not supported")@@ -6983,7 +7004,7 @@ def make_handler(svc: Service):self._no_local_images(messages)if tool_choice_of(req.get("tool_choice"))[0] == "none": # Anthropic's {"type": "none"}: no tools offeredtools = None- force = forced_call(req.get("tool_choice"), tools) # "any" / {"type": "tool", "name": N}+ force = svc.force_text(forced_call(req.get("tool_choice"), tools)) # "any" / {"type": "tool", "name": N}max_new = int(req.get("max_tokens") or 0) # 0/-1: the rest of the contextsvc.reasoning_budget(req) # a bad value is a 400 before anything is sentstop_strings(req) # ... and so is a bad stop / stop_sequences@@ -7636,7 +7657,8 @@ def check_request_sampling(req) -> None:# the API's model name per model family (the GGUF architecture, "arch" in the config) when the config names none-FAMILY_MODEL_NAMES = {"qwen4exp": "qwen3.8-flash-next", "qwen35moe": "qwen3.6-35b-a3b"}+FAMILY_MODEL_NAMES = {"qwen4exp": "qwen3.8-flash-next", "qwen35moe": "qwen3.6-35b-a3b",+ "deepseek41": "deepseek-v4.1-flash"}# The sampling each model card recommends ("Best Practices"), used when neither the request, the Chat settings nor the@@ -7649,6 +7671,8 @@ MODEL_CARD_SAMPLING = {# Qwen3.6-35B-A3B: the general-task sets (the card's precise-coding set is temperature 0.6, presence_penalty 0)"qwen35moe": {"thinking": {"temperature": 1.0, "top_p": 0.95, "top_k": 20, "presence_penalty": 1.5},"instruct": {"temperature": 0.7, "top_p": 0.8, "top_k": 20, "presence_penalty": 1.5}},+ # DeepSeek-V4.1-Flash: temperature 1.0, top_p 1.0 for local use (thinking or not), no penalties+ "deepseek41": {"thinking": {"temperature": 1.0, "top_p": 1.0}, "instruct": {"temperature": 1.0, "top_p": 1.0}},}@@ -7909,7 +7933,12 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> intokens[i] = tmerges = (tpath / "merges.txt").read_text(encoding="utf-8").split("\n")types = json.loads((tpath / "token_type.json").read_text())- tok = ST.Tokenizer(tokens, merges, types)+ tcfg = {}+ try: # the pack's pre-tokenizer name and special ids (DeepSeek's differ)+ tcfg = json.loads((tpath / "tokenizer.json").read_text(encoding="utf-8"))+ except (OSError, ValueError):+ pass+ tok = ST.Tokenizer(tokens, merges, types, tcfg.get("pre") or "qwen35", tcfg.get("special_ids") or None)hub = hub_from_config(cfg, a.mcp_config) # before the minutes of loading: a bad entry stops hereif a.offline and (hub is not None or cfg.get("before_load") or a.before_load):ap.error("--offline requires MCP servers and before_load commands to be disabled in this profile")@@ -8015,6 +8044,7 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> insvc.queue_timeout_s = float(queue_timeout) if queue_timeout is not None else Noneif a.engine == "strata": # the model card's sampling when nothing else is said (see MODEL_CARD_SAMPLING)svc.model_arch = cfg.get("arch") or "qwen4exp"+ svc.set_family(svc.model_arch if svc.model_arch == "deepseek41" else None)requested = cfg.get("parallel")if requested is None:args = cfg.get("args", [])diff --git a/serve/test_dsv41.py b/serve/test_dsv41.pynew file mode 100644index 000000000..71c6a12fe--- /dev/null+++ b/serve/test_dsv41.py@@ -0,0 +1,465 @@+"""DeepSeek-V4.1 through the server: its tokenizer, its chat template, the DSML tool calls, the effort words, the end+token and the model-card defaults, over both APIs against a fake engine (no GPU, no pack, no model).++ python -m unittest serve.test_dsv41 -v++STRATA_DSV41_PACK=<dir> (the pack's tokenizer folder: vocab.json, merges.txt, token_type.json, tokenizer.json) adds+the check of the real tokenizer against the ids llama.cpp (PR 30817 b72050d0) made of the same rendered prompts.+"""+from __future__ import annotations++import json+import os+import sys+import unittest+import urllib.error+import urllib.request+from pathlib import Path++ROOT = Path(__file__).resolve().parents[1]+sys.path.insert(0, str(ROOT))+sys.path.insert(0, str(ROOT / "tools"))+import strata_tokenizer as ST # noqa: E402+from serve import frontend # noqa: E402+from serve.dsml import DsmlOutputParser, DsmlTranslator, to_dsml_force # noqa: E402+from serve.frontend import ChatTemplate, OutputParser, openai_to_messages # noqa: E402+from serve.server import (FAMILY_MODEL_NAMES, MockEngine, Service, default_model_name, model_card_sampling, # noqa: E402+ serve)++FIX = ROOT / "serve" / "fixtures" / "deepseek41"+TEMPLATE = FIX / "chat_template.jinja"+GOLDEN = json.loads((FIX / "golden.json").read_text(encoding="utf-8"))++BOS, EOS_TEXT = "<|begin▁of▁sentence|>", "<|end▁of▁sentence|>"+SPECIALS = [(BOS, 3), (EOS_TEXT, 3), ("<|User|>", 3), ("<|Assistant|>", 3), ("<|System|>", 3),+ ("<|latest_reminder|>", 3), ("<think>", 4), ("</think>", 4), ("|DSML|", 4), ("<|deepseek_image|>", 3)]+EOS = 1+++def mini_tokenizer() -> ST.Tokenizer:+ """The real Tokenizer class over a tiny vocabulary: DeepSeek's special tokens, then the 256 byte tokens, no merges."""+ b2u = ST.bytes_to_unicode()+ tokens = [s for s, _ in SPECIALS] + [b2u[b] for b in range(256)]+ types = [t for _, t in SPECIALS] + [1] * 256+ return ST.Tokenizer(tokens, [], types, "deepseek3", {"bos": 0, "eos": EOS})+++class DsEngine(MockEngine):+ """Replays scripted completions as ids, then DeepSeek's end token; keeps what it was asked."""++ def __init__(self, tok, script, max_context=8192):+ super().__init__(tok, script, max_context=max_context)+ self.scripts = [tok.encode(x, parse_special=True) + [EOS] for x in ([script] if isinstance(script, str) else script)]+ self.script = self.scripts[0]+ self.last_sampling = None+ self.last = {}++ def generate(self, ids, max_new, sampling, cancel, embeddings=None):+ self.last_sampling = dict(sampling or {})+ got = 0+ try:+ for t in super().generate(ids, max_new, sampling, cancel, embeddings):+ got += 1+ yield t+ finally: # (the server stops reading at the end token: the engine's DONE comes while it is closed)+ self.last = {"generated": got, "prompt_tokens": len(ids), "prompt_ms": 50.0, "decode_ms": got * 10.0,+ "finish": "stop"}+++def call(*invokes, string="true") -> str:+ """The text DeepSeek writes for calls: [(name, [(param, value, is_string)...])...]."""+ out = ["\n\n<|DSML| calls>"]+ for name, params in invokes:+ out.append(f'<|DSML| invoke name="{name}">')+ for p, v, is_str in params:+ out.append(f'<|DSML| parameter name="{p}" string="{"true" if is_str else "false"}">{v}</|DSML| parameter>')+ out.append("</|DSML| invoke>")+ out.append("</|DSML| calls>")+ return "\n".join(out)+++WEATHER = {"type": "function", "function": {"name": "get_weather", "description": "weather",+ "parameters": {"type": "object", "properties": {+ "city": {"type": "string"}, "days": {"type": "integer"}},+ "required": ["city"]}}}+++class Pre(unittest.TestCase):+ """The pre-tokenizer is llama.cpp's DEEPSEEK3_LLM list: a sequence of splits, not one pattern."""++ def split(self, text):+ return ST._sequence_findall([ST.regex.compile(p) for p in ST.DEEPSEEK3_PATTERNS], text)++ def test_digits_in_runs_of_three(self):+ self.assertEqual(self.split("a12345678b"), ["a", "123", "456", "78", "b"])++ def test_cjk_runs(self):+ self.assertEqual(self.split("你好, world"), ["你好", ",", " world"])++ def test_punctuation_then_letters_and_whitespace(self):+ self.assertEqual(self.split("x_y.z \n\nq"), ["x", "_y", ".z", " \n\n", "q"])++ def test_every_character_is_kept(self):+ text = "Hello, 世界! 123\t\r\n é \U0001f600 |DSML|"+ self.assertEqual("".join(self.split(text)), text)+++class Tokenizer(unittest.TestCase):+ def test_a_special_token_decodes_to_its_text(self):+ tok = mini_tokenizer()+ ids = tok.encode("<|User|>hi|DSML|", parse_special=True)+ self.assertEqual(ids[0], 2)+ self.assertEqual(tok.decode(ids), "<|User|>hi|DSML|")+ self.assertEqual(tok.token_bytes(8), "|DSML|".encode())++ def test_the_dsml_token_is_matched_without_parse_special_and_a_control_token_is_not(self):+ tok = mini_tokenizer()+ self.assertEqual(tok.encode("|DSML|").__len__(), 1)+ self.assertGreater(len(tok.encode("<|User|>")), 1)++ def test_qwen_tokens_are_unchanged(self):+ b2u = ST.bytes_to_unicode()+ tok = ST.Tokenizer([b2u[b] for b in range(256)] + ["<|im_end|>"], [], [1] * 256 + [3])+ self.assertEqual(tok.token_bytes(256), b"<|im_end|>")+ self.assertEqual(tok.encode("a<|im_end|>", parse_special=True), [97, 256])++ @unittest.skipUnless(os.environ.get("STRATA_DSV41_PACK"), "STRATA_DSV41_PACK (the pack's tokenizer folder) not set")+ def test_the_real_tokenizer_makes_the_ids_llama_cpp_made(self):+ d = Path(os.environ["STRATA_DSV41_PACK"])+ vocab = json.loads((d / "vocab.json").read_text(encoding="utf-8"))+ tokens = [None] * len(vocab)+ for t, i in vocab.items():+ tokens[i] = t+ cfg = json.loads((d / "tokenizer.json").read_text())+ tok = ST.Tokenizer(tokens, (d / "merges.txt").read_text(encoding="utf-8").split("\n"),+ json.loads((d / "token_type.json").read_text()), cfg["pre"], cfg["special_ids"])+ for case in GOLDEN:+ self.assertEqual(tok.encode(case["text"], parse_special=True), case["ids"], case["id"] + " " + case["mode"])+++class Template(unittest.TestCase):+ """The GGUF's own chat template (the same bytes as DeepSeek's encoding.py on the cases checked), through the+ server's request reader and renderer: the text llama.cpp tokenized for the reference run."""++ @classmethod+ def setUpClass(cls):+ cls.tpl = ChatTemplate(TEMPLATE)++ def test_the_golden_prompts(self):+ for case in GOLDEN:+ messages, tools, kw = openai_to_messages(case["request"])+ self.assertEqual(self.tpl.render(messages, tools=tools, **kw), case["text"], case["id"] + " " + case["mode"])++ def test_thinking_is_the_default(self):+ text = self.tpl.render([{"role": "user", "content": "hi"}])+ self.assertTrue(text.endswith("<|Assistant|><think>"))+ text = self.tpl.render([{"role": "user", "content": "hi"}], enable_thinking=False)+ self.assertTrue(text.endswith("<|Assistant|></think>"))++ def test_a_tool_call_and_its_result_render_in_dsml(self):+ msgs = [{"role": "user", "content": "weather?"},+ {"role": "assistant", "content": "", "reasoning_content": "call it",+ "tool_calls": [{"function": {"name": "get_weather", "arguments": {"city": "Zagreb", "days": 3}}}]},+ {"role": "tool", "content": "sunny"}]+ text = self.tpl.render(msgs, tools=[WEATHER["function"]], enable_thinking=False)+ self.assertIn('<|DSML| invoke name="get_weather">', text)+ self.assertIn('<|DSML| parameter name="city" string="true">Zagreb</|DSML| parameter>', text)+ self.assertIn('<|DSML| parameter name="days" string="false">3</|DSML| parameter>', text)+ self.assertIn("<tool_result>sunny</tool_result>", text)+++class Translator(unittest.TestCase):+ RAW = call(("get_weather", [("city", "Zagreb", True), ("days", "3", False)]), ("ping", []))++ def all_at_once(self, text):+ t = DsmlTranslator()+ return t.feed(text) + t.flush()++ def test_a_call_becomes_the_qwen_form(self):+ got = self.all_at_once("Sure." + self.RAW)+ self.assertEqual(got, "Sure.\n\n<tool_call>\n<function=get_weather>\n<parameter=city>\nZagreb\n</parameter>\n"+ "<parameter=days>\n3\n</parameter>\n</function>\n</tool_call>\n"+ "<tool_call>\n<function=ping>\n</function>\n</tool_call>\n")++ def test_the_split_of_the_stream_changes_nothing(self):+ want = self.all_at_once("a < b and 1<2" + self.RAW + "tail")+ for step in (1, 2, 3, 5, 8, 13):+ t = DsmlTranslator()+ text = "a < b and 1<2" + self.RAW + "tail"+ got = "".join(t.feed(text[i:i + step]) for i in range(0, len(text), step)) + t.flush()+ self.assertEqual(got, want, step)++ def test_text_with_angle_brackets_passes(self):+ text = "x <div> y </|DSML| nope> <|User|> <|DSML| unknown>"+ self.assertEqual(self.all_at_once(text), text)++ def test_the_end_of_the_output_inside_a_tag_is_text(self):+ t = DsmlTranslator()+ got = t.feed("ok <|DSML| inv") + t.flush()+ self.assertEqual(got, "ok <|DSML| inv")++ def test_a_value_that_looks_like_a_tag(self):+ raw = call(("write", [("text", "<b>bold</b> and < 3", True)]))+ self.assertIn("<parameter=text>\n<b>bold</b> and < 3\n</parameter>", self.all_at_once(raw))++ def test_a_missing_closing_tag_is_closed(self):+ raw = '<|DSML| calls>\n<|DSML| invoke name="f">\n<|DSML| parameter name="a" string="true">x</|DSML| invoke>'+ self.assertEqual(self.all_at_once(raw),+ "<tool_call>\n<function=f>\n<parameter=a>\nx\n</parameter>\n</function>\n</tool_call>\n")+++class Parser(unittest.TestCase):+ TOOLS = [WEATHER["function"], {"name": "lookup", "description": "", "parameters": {+ "type": "object", "properties": {"id": {}}}}]++ def run_parser(self, text, thinking=True, step=7, cls=DsmlOutputParser):+ p = cls(thinking=thinking, tools=self.TOOLS, stream_tools=True)+ evs = []+ for i in range(0, len(text), step):+ evs += p.feed(text[i:i + step])+ evs += p.finish("stop")+ return evs++ def test_reasoning_then_two_calls(self):+ raw = call(("get_weather", [("city", "Zagreb", True), ("days", "3", False)]), ("lookup", [("id", "123", True)]))+ for step in (1, 4, 1000):+ evs = self.run_parser("Think < ok.</think>" + raw, step=step)+ self.assertEqual("".join(e.text for e in evs if e.kind == "reasoning"), "Think < ok.")+ calls = [e.call for e in evs if e.kind == "tool_call"]+ self.assertEqual([(c.name, c.arguments) for c in calls],+ [("get_weather", {"city": "Zagreb", "days": 3}), ("lookup", {"id": "123"})])+ self.assertEqual("".join(e.text for e in evs if e.kind == "tool_args"),+ '{"city":"Zagreb","days":3}{"id":"123"}')++ def test_a_string_value_stays_a_string_even_when_the_schema_says_nothing(self):+ evs = self.run_parser(call(("lookup", [("id", "007", True)])), thinking=False)+ self.assertEqual([e.call.arguments for e in evs if e.kind == "tool_call"], [{"id": "007"}])+ evs = self.run_parser(call(("lookup", [("id", "7", False)])), thinking=False)+ self.assertEqual([e.call.arguments for e in evs if e.kind == "tool_call"], [{"id": 7}])++ def test_a_non_string_value_is_json(self):+ evs = self.run_parser(call(("lookup", [("id", '{"a": [1, 2]}', False)])), thinking=False)+ self.assertEqual([e.call.arguments for e in evs if e.kind == "tool_call"], [{"id": {"a": [1, 2]}}])++ def test_content_before_the_call_and_none_after(self):+ evs = self.run_parser("Checking.\n\n" + call(("lookup", [("id", "x", True)]))[2:], thinking=False)+ self.assertEqual("".join(e.text for e in evs if e.kind == "content").strip(), "Checking.")+ self.assertEqual(sum(e.kind == "tool_call" for e in evs), 1)++ def test_a_call_cut_off_by_the_length_limit_is_not_a_call(self):+ raw = call(("get_weather", [("city", "Zagr", True)]))+ raw = raw[:raw.index("Zagr") + 4]+ p = DsmlOutputParser(thinking=False, tools=self.TOOLS, stream_tools=True)+ evs = p.feed(raw) + p.finish("length")+ self.assertEqual(sum(e.kind == "tool_call" for e in evs), 0)++ def test_the_plain_parser_is_unchanged(self):+ p = OutputParser(thinking=False, tools=self.TOOLS)+ evs = p.feed("<tool_call>\n<function=lookup>\n<parameter=id>\n5\n</parameter>\n</function>\n</tool_call>")+ self.assertEqual([e.call.arguments for e in evs if e.kind == "tool_call"], [{"id": 5}])+ self.assertTrue(p.clean)++ def test_clean_counts_a_held_tag(self):+ p = DsmlOutputParser(thinking=False, tools=self.TOOLS)+ p.feed("hello <|DS")+ self.assertFalse(p.clean)+ p.feed("ML| calls>")+ self.assertTrue(p.clean)++ def test_forced_opening(self):+ self.assertEqual(to_dsml_force("<tool_call>\n<function=get_weather>\n"),+ '<|DSML| calls>\n<|DSML| invoke name="get_weather">\n')+ self.assertEqual(to_dsml_force("<tool_call>\n<function="), '<|DSML| calls>\n<|DSML| invoke name="')+ self.assertIsNone(to_dsml_force(None))+ # the opening and the model's continuation are one call+ p = DsmlOutputParser(thinking=False, tools=self.TOOLS, stream_tools=True)+ evs = p.feed(to_dsml_force("<tool_call>\n<function=get_weather>\n"))+ evs += p.feed('<|DSML| parameter name="city" string="true">Rome</|DSML| parameter>\n</|DSML| invoke>\n</|DSML| calls>')+ evs += p.finish("stop")+ self.assertEqual([(e.call.name, e.call.arguments) for e in evs if e.kind == "tool_call"],+ [("get_weather", {"city": "Rome"})])+++class Api(unittest.TestCase):+ """Both APIs against a fake engine that replays scripted DeepSeek text."""++ @classmethod+ def setUpClass(cls):+ cls.tok = mini_tokenizer()++ def start(self, script):+ tok = self.tok+ engine = DsEngine(tok, script)+ svc = Service(engine, tok, ChatTemplate(TEMPLATE), model_name=default_model_name({"arch": "deepseek41"}))+ svc.model_arch = "deepseek41"+ svc.set_family("deepseek41")+ httpd = serve(svc, port=0)+ self.addCleanup(httpd.server_close)+ self.addCleanup(httpd.shutdown)+ self.addCleanup(frontend.use_family, None)+ self.engine, self.svc = engine, svc+ self.base = f"http://127.0.0.1:{httpd.server_address[1]}"+ return engine++ def post(self, path, body, raw=False):+ req = urllib.request.Request(self.base + path, data=json.dumps(body).encode(),+ headers={"Content-Type": "application/json"})+ try:+ with urllib.request.urlopen(req, timeout=30) as r:+ data = r.read().decode()+ return r.status, (data if raw else json.loads(data))+ except urllib.error.HTTPError as e:+ return e.code, json.loads(e.read().decode())++ def prompt(self):+ return self.tok.decode(self.engine.last_prompt)++ def chat(self, **extra):+ body = {"messages": [{"role": "user", "content": "hi"}], "max_tokens": 400, **extra}+ return self.post("/v1/chat/completions", body)++ # -- thinking --------------------------------------------------------------------------------+ def test_thinking_by_default(self):+ self.start("The user greets me.</think>\n\nHello!")+ status, r = self.chat()+ self.assertEqual(status, 200, r)+ self.assertTrue(self.prompt().endswith("<|User|>hi<|Assistant|><think>"))+ msg = r["choices"][0]["message"]+ self.assertEqual(msg["reasoning_content"], "The user greets me.")+ self.assertEqual(msg["content"].strip(), "Hello!")+ self.assertEqual(r["choices"][0]["finish_reason"], "stop")++ def test_thinking_off(self):+ self.start("Hello!")+ for extra in ({"chat_template_kwargs": {"enable_thinking": False}}, {"reasoning_effort": "none"}):+ status, r = self.chat(**extra)+ self.assertEqual(status, 200, r)+ self.assertTrue(self.prompt().endswith("<|User|>hi<|Assistant|></think>"), extra)+ msg = r["choices"][0]["message"]+ self.assertEqual(msg["content"].strip(), "Hello!")+ self.assertFalse(msg.get("reasoning_content"))++ def test_the_end_token_is_deepseeks_not_qwens(self):+ self.start("one<|im_end|>two") # (not a token here: just text, like any other)+ status, r = self.chat(chat_template_kwargs={"enable_thinking": False})+ self.assertEqual(r["choices"][0]["message"]["content"], "one<|im_end|>two")++ def test_model_name_and_card_sampling(self):+ self.start("ok")+ self.assertEqual(FAMILY_MODEL_NAMES["deepseek41"], "deepseek-v4.1-flash")+ self.assertEqual(default_model_name({"arch": "deepseek41"}), "deepseek-v4.1-flash")+ self.assertEqual(model_card_sampling("deepseek41", True), {"temperature": 1.0, "top_p": 1.0})+ self.chat()+ self.assertEqual(self.engine.last_sampling.get("temperature"), 1.0)++ # -- effort ------------------------------------------------------------------------------------+ def test_effort_words(self):+ self.start("ok")+ for given, line in (("low", "Reasoning Effort: 50 "), ("medium", "Reasoning Effort: 62 "),+ ("high", "Reasoning Effort: 75 "), ("max", "Reasoning Effort: 100 "),+ ("xhigh", "Reasoning Effort: 100 "), (40, "Reasoning Effort: 40 ")):+ status, r = self.chat(reasoning_effort=given)+ self.assertEqual(status, 200, (given, r))+ self.assertIn(line, self.prompt(), given)+ status, r = self.chat()+ self.assertIn("Reasoning Effort: 75 ", self.prompt())+ status, r = self.chat(reasoning_effort="extreme")+ self.assertEqual(status, 400)++ def test_anthropic_budget_picks_the_effort(self):+ self.start("ok")+ status, r = self.post("/v1/messages", {"model": "x", "max_tokens": 300, "thinking": {"type": "enabled", "budget_tokens": 1000},+ "messages": [{"role": "user", "content": "hi"}]})+ self.assertEqual(status, 200, r)+ self.assertIn("Reasoning Effort: 50 ", self.prompt())++ # -- tool calls -------------------------------------------------------------------------------+ def test_openai_tool_call(self):+ self.start("Need the weather.</think>" + call(("get_weather", [("city", "Zagreb", True), ("days", "2", False)])))+ status, r = self.chat(tools=[WEATHER])+ self.assertEqual(status, 200, r)+ self.assertIn('"name": "get_weather"', self.prompt())+ ch = r["choices"][0]+ self.assertEqual(ch["finish_reason"], "tool_calls")+ tc = ch["message"]["tool_calls"][0]+ self.assertEqual(tc["function"]["name"], "get_weather")+ self.assertEqual(json.loads(tc["function"]["arguments"]), {"city": "Zagreb", "days": 2})+ self.assertEqual(ch["message"]["reasoning_content"], "Need the weather.")++ def test_two_calls_in_one_turn(self):+ self.start("</think>" + call(("get_weather", [("city", "A", True)]), ("get_weather", [("city", "B", True)])))+ status, r = self.chat(tools=[WEATHER])+ names = [json.loads(c["function"]["arguments"])["city"] for c in r["choices"][0]["message"]["tool_calls"]]+ self.assertEqual(names, ["A", "B"])++ def test_streamed_tool_call(self):+ self.start("</think>" + call(("get_weather", [("city", "Zagreb", True)])))+ status, text = self.post("/v1/chat/completions", {"messages": [{"role": "user", "content": "w?"}], "tools": [WEATHER],+ "stream": True, "max_tokens": 300}, raw=True)+ self.assertEqual(status, 200)+ chunks = [json.loads(x[6:]) for x in text.split("\n\n") if x.startswith("data: ") and x != "data: [DONE]"]+ calls = [tc for c in chunks for tc in c["choices"][0]["delta"].get("tool_calls", [])]+ self.assertEqual(calls[0]["function"]["name"], "get_weather")+ self.assertEqual(json.loads("".join(tc["function"].get("arguments", "") for tc in calls)), {"city": "Zagreb"})+ self.assertEqual([c["choices"][0]["finish_reason"] for c in chunks if c["choices"][0].get("finish_reason")],+ ["tool_calls"])++ def test_a_tool_result_turn_renders_for_the_model(self):+ self.start("The weather is sunny.")+ msgs = [{"role": "user", "content": "weather in Zagreb?"},+ {"role": "assistant", "content": None, "tool_calls": [{"id": "c1", "type": "function", "function": {+ "name": "get_weather", "arguments": "{\"city\": \"Zagreb\"}"}}]},+ {"role": "tool", "tool_call_id": "c1", "content": "sunny, 21C"}]+ status, r = self.post("/v1/chat/completions", {"messages": msgs, "tools": [WEATHER], "max_tokens": 200,+ "chat_template_kwargs": {"enable_thinking": False}})+ self.assertEqual(status, 200, r)+ p = self.prompt()+ self.assertIn('<|DSML| parameter name="city" string="true">Zagreb</|DSML| parameter>', p)+ self.assertIn("<tool_result>sunny, 21C</tool_result>", p)+ self.assertEqual(r["choices"][0]["message"]["content"].strip(), "The weather is sunny.")++ def test_forced_tool_choice(self):+ self.start('<|DSML| parameter name="city" string="true">Rome</|DSML| parameter>\n</|DSML| invoke>\n</|DSML| calls>')+ status, r = self.chat(tools=[WEATHER], tool_choice={"type": "function", "function": {"name": "get_weather"}},+ chat_template_kwargs={"enable_thinking": False})+ self.assertEqual(status, 200, r)+ self.assertTrue(self.prompt().endswith('<|Assistant|></think><|DSML| calls>\n<|DSML| invoke name="get_weather">\n'), self.prompt()[-120:])+ tc = r["choices"][0]["message"]["tool_calls"][0]+ self.assertEqual((tc["function"]["name"], json.loads(tc["function"]["arguments"])), ("get_weather", {"city": "Rome"}))++ # -- anthropic ---------------------------------------------------------------------------------+ def test_anthropic_tool_use(self):+ self.start("Weather needed.</think>" + call(("get_weather", [("city", "Oslo", True), ("days", "1", False)])))+ status, r = self.post("/v1/messages", {"model": "x", "max_tokens": 400,+ "tools": [{"name": "get_weather", "description": "w", "input_schema":+ WEATHER["function"]["parameters"]}],+ "messages": [{"role": "user", "content": "Oslo?"}]})+ self.assertEqual(status, 200, r)+ kinds = [b["type"] for b in r["content"]]+ self.assertEqual(kinds, ["thinking", "tool_use"])+ self.assertEqual(r["content"][1]["input"], {"city": "Oslo", "days": 1})+ self.assertEqual(r["stop_reason"], "tool_use")++ def test_anthropic_text_without_thinking(self):+ self.start("Hello there.")+ status, r = self.post("/v1/messages", {"model": "x", "max_tokens": 100, "thinking": {"type": "disabled"},+ "messages": [{"role": "user", "content": "hi"}]})+ self.assertEqual(status, 200, r)+ self.assertTrue(self.prompt().endswith("<|Assistant|></think>"))+ self.assertEqual([b["type"] for b in r["content"]], ["text"])+ self.assertEqual(r["content"][0]["text"], "Hello there.")+ self.assertEqual(r["stop_reason"], "end_turn")++ # -- metrics -----------------------------------------------------------------------------------+ def test_metrics_use_the_one_answer_speed(self):+ self.start("</think>Hello, how are you today?")+ self.chat()+ status = urllib.request.urlopen(self.base + "/metrics", timeout=10)+ m = json.loads(status.read().decode())+ last = m["requests"][-1]+ # produced tokens over the engine's decode time (10 ms a token in the fake), never over the prompt's time+ self.assertEqual(last["decode_tok_s"], 100.0)+++if __name__ == "__main__":+ unittest.main()diff --git a/serve/test_family_name.py b/serve/test_family_name.pyindex cd4024ff3..44deb8415 100644--- a/serve/test_family_name.py+++ b/serve/test_family_name.py@@ -19,11 +19,12 @@ class FamilyName(unittest.TestCase):def test_per_family(self):self.assertEqual(default_model_name({"arch": "qwen35moe"}), "qwen3.6-35b-a3b")self.assertEqual(default_model_name({"arch": "qwen4exp"}), "qwen3.8-flash-next")+ self.assertEqual(default_model_name({"arch": "deepseek41"}), "deepseek-v4.1-flash")def test_old_configs(self):self.assertEqual(default_model_name({}), "qwen3.8-flash-next")self.assertEqual(default_model_name({"arch": "something-else"}), "qwen3.8-flash-next")- self.assertEqual(set(FAMILY_MODEL_NAMES), {"qwen4exp", "qwen35moe"})+ self.assertEqual(set(FAMILY_MODEL_NAMES), {"qwen4exp", "qwen35moe", "deepseek41"})if __name__ == "__main__":diff --git a/tools/strata_tokenizer.py b/tools/strata_tokenizer.pyindex 42a27fed3..33452af86 100644--- a/tools/strata_tokenizer.py+++ b/tools/strata_tokenizer.py@@ -66,6 +66,37 @@ QWEN35_PATTERN = ()+# The `deepseek-v3` pre-tokenizer (DeepSeek-V3 / V4.1; llama.cpp LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM, the same regex list as+# JoyAI and Hunyuan-dense). UNLIKE the single-pattern families it is a SEQUENCE: each pattern splits the pieces the+# one before it left (a span a pattern does not match stays a piece of its own), so digits (runs of 1..3) and CJK+# runs are cut out before the general word / punctuation / whitespace pattern sees the rest.+DEEPSEEK3_PATTERNS = (+ r"\p{N}{1,3}",+ "[一-龥-ゟ゠-ヿ]+",+ r"[!\"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~][A-Za-z]+|[^\r\n\p{L}\p{P}\p{S}]?[\p{L}\p{M}]+"+ r"| ?[\p{P}\p{S}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+",+)+++def _sequence_findall(patterns, text: str) -> list[str]:+ """llama.cpp's unicode_regex_split: the pieces of `text` after each pattern in turn splits every piece into its+ matches and the unmatched spans between them."""+ pieces = [text]+ for pat in patterns:+ nxt = []+ for piece in pieces:+ pos = 0+ for m in pat.finditer(piece):+ if m.start() > pos:+ nxt.append(piece[pos:m.start()])+ nxt.append(m.group())+ pos = m.end()+ if pos < len(piece):+ nxt.append(piece[pos:])+ pieces = nxt+ return pieces++class Tokenizer:def __init__(self, tokens: list[str], merges: list[str], token_types: list[int] | None = None,pre: str = "qwen35", special_ids: dict[str, int] | None = None):@@ -88,7 +119,12 @@ class Tokenizer:if parts[0] not in self.ids or parts[1] not in self.ids:raise ValueError("merge %d names a token outside the vocabulary: %r" % (i, m))self.ranks[(parts[0], parts[1])] = i- self._re = regex.compile(QWEN35_PATTERN)+ if str(pre).startswith("deepseek"):+ seq = [regex.compile(x) for x in DEEPSEEK3_PATTERNS]+ self._split = lambda text: _sequence_findall(seq, text)+ else:+ self._re = regex.compile(QWEN35_PATTERN)+ self._split = self._re.findall# The literals matched directly instead of being run through BPE. GGUF token types: 3 = CONTROL,# 4 = USER_DEFINED. The two classes behave DIFFERENTLY and llama.cpp's own tokenizer settled which:@@ -223,7 +259,7 @@ class Tokenizer:# A piece's ids depend on the piece alone, so repeated pieces (most of a resent conversation) are looked up# instead of merged again. The ids are the ones _bpe gives: this only skips the work.cache = self._piece_ids- for piece in self._re.findall(text):+ for piece in self._split(text):got = cache.get(piece)if got is None:mapped = "".join(BYTE_TO_UNICODE[b] for b in piece.encode("utf-8"))@@ -304,6 +340,11 @@ class Tokenizer:if b is None:if i < 0 or i >= len(self.tokens):raise IndexError("token id %d is outside the vocabulary (%d)" % (i, len(self.tokens)))+ if self.token_types and i < len(self.token_types) and self.token_types[i] in (3, 4):+ # a special token's string is the literal text (DeepSeek's `<|User|>`, `|DSML|` hold characters outside+ # the byte alphabet); for Qwen's ASCII specials it is the same bytes the alphabet would give+ b = cache[i] = self.tokens[i].encode("utf-8")+ return braw = bytearray()for ch in self.tokens[i]:v = UNICODE_TO_BYTE.get(ch)