Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

serve: damaged tool-call arguments, a stray closing think tag and a missing </function> no longer cost the call (#4427 #4077)

Niko1221d71132738e61
diff --git a/serve/frontend.py b/serve/frontend.py
index cb8e1cda9..395f00677 100644
--- a/serve/frontend.py
+++ b/serve/frontend.py
@@ -754,7 +754,8 @@ _QUOTED_OPEN = re.compile(r"<function=[^<>\s]+>\s*(?:<parameter=|</function>)")
def param_end(text: str, final: bool = False, name: str | None = None) -> int:
"""Where a parameter value in `text` ends: the first PARAM_END followed (after whitespace) by the next
- PARAM_START or FUNC_END - the same text inside a value (a file that documents the call format, #210) is
+ PARAM_START or FUNC_END, or by the wrapper's CALL_END when the call's `</function>` was left out (#4427) - the
+ same text inside a value (a file that documents the call format, #210) is
part of the value. #4078: a close inside a call the value itself quotes (more _QUOTED_OPEN than FUNC_END before
it), or one followed by the same parameter `name` again, is part of the value too. `text` starts at the value.
-1: none yet; -2: a candidate whose follower has not arrived (streaming; `final` accepts it)."""
@@ -768,9 +769,9 @@ def param_end(text: str, final: bool = False, name: str | None = None) -> int:
return at if final else -2
if name is None or after[len(PARAM_START):gt].strip() != name:
return at
- elif after.startswith(FUNC_END):
+ elif after.startswith((FUNC_END, CALL_END)): # #4427: the model left out its `</function>`
return at
- elif not after or PARAM_START.startswith(after) or FUNC_END.startswith(after):
+ elif not after or PARAM_START.startswith(after) or FUNC_END.startswith(after) or CALL_END.startswith(after):
return at if final else -2
at = text.find(PARAM_END, at + 1)
return -1
@@ -916,7 +917,151 @@ def json_tool_call(body: str, schemas: dict) -> ToolCall | None:
return ToolCall(name=obj["name"], arguments=args) if isinstance(args, dict) else None
-def parse_tool_call(body: str, schema: dict | None = None) -> ToolCall:
+LENIENT_MAX = 1 << 20 # the longest value lenient_json will try (it is a python loop)
+
+
+class _Lenient:
+ """A JSON reader for the slips a model makes inside an array or object argument (#4427): a trailing or doubled comma,
+ single quotes and Python's True/False/None, a bare string between two members of an object (`"status": "pending",
+ "in_progress", "pending": false`), brackets left open at the end, extra closers at the end. It is only asked about
+ text json.loads has refused. It never guesses at a value: a string cut short, an unknown word, a missing colon
+ after a key, anything else raises ValueError."""
+
+ _WORDS = {"true": True, "false": False, "null": None, "True": True, "False": False, "None": None}
+ _ESC = {'"': '"', "'": "'", "\\": "\\", "/": "/", "b": "\b", "f": "\f", "n": "\n", "r": "\r", "t": "\t"}
+ _TOKEN = re.compile(r"[^\s,\]}:]+")
+
+ def __init__(self, text: str):
+ self.s, self.i = text, 0
+
+ def peek(self) -> str:
+ s, n = self.s, len(self.s)
+ while self.i < n and s[self.i] in " \t\r\n":
+ self.i += 1
+ return s[self.i:self.i + 1]
+
+ def string(self) -> str:
+ s, q = self.s, self.s[self.i]
+ self.i += 1
+ out = []
+ while self.i < len(s):
+ c = s[self.i]
+ if c == q:
+ self.i += 1
+ return "".join(out)
+ if c == "\\":
+ self.i += 1
+ e = s[self.i:self.i + 1]
+ if e == "u" and re.fullmatch(r"[0-9a-fA-F]{4}", s[self.i + 1:self.i + 5]):
+ out.append(chr(int(s[self.i + 1:self.i + 5], 16)))
+ self.i += 5
+ continue
+ if e not in self._ESC:
+ raise ValueError("bad escape")
+ out.append(self._ESC[e])
+ else:
+ out.append(c)
+ self.i += 1
+ raise ValueError("string not closed")
+
+ def value(self, depth: int = 0):
+ if depth > 64:
+ raise ValueError("too deep")
+ c = self.peek()
+ if c == "{":
+ return self.obj(depth + 1)
+ if c == "[":
+ return self.arr(depth + 1)
+ if c in ('"', "'"):
+ return self.string()
+ m = self._TOKEN.match(self.s, self.i)
+ if not m:
+ raise ValueError("no value")
+ self.i = m.end()
+ if m.group() in self._WORDS:
+ return self._WORDS[m.group()]
+ return json.loads(m.group()) # a number, or ValueError
+
+ def obj(self, depth: int):
+ self.i += 1
+ out: dict = {}
+ while True:
+ c = self.peek()
+ if not c or c == "}": # (no closer: the brackets were left open at the end)
+ self.i += 1
+ return out
+ if c == ",":
+ self.i += 1
+ continue
+ if c not in ('"', "'"):
+ raise ValueError("object member")
+ key = self.string()
+ if self.peek() == ":":
+ self.i += 1
+ out[key] = self.value(depth)
+ elif self.peek() not in (",", "}", ""): # a bare string where a member belongs is dropped
+ raise ValueError("object member")
+
+ def arr(self, depth: int):
+ self.i += 1
+ out: list = []
+ while True:
+ c = self.peek()
+ if not c or c == "]":
+ self.i += 1
+ return out
+ if c == ",":
+ self.i += 1
+ continue
+ out.append(self.value(depth))
+ if self.peek() not in (",", "]", ""):
+ raise ValueError("array element")
+
+
+def lenient_json(text: str):
+ """The JSON array or object `text` was meant to be, read past the slips of _Lenient; ValueError for anything else
+ (and for text that is not an array or object). Valid JSON is not its business: callers try json.loads first."""
+ t = text.strip()
+ m = _JSON_FENCE.match(t)
+ if m:
+ t = t[m.end():].rstrip()
+ t = t[:-3].rstrip() if t.endswith("```") else t
+ if not t.startswith(("[", "{")) or len(t) > LENIENT_MAX:
+ raise ValueError("not an array or object")
+ r = _Lenient(t)
+ v = r.value()
+ if t[r.i:].strip("]}) \t\r\n"):
+ raise ValueError("text after the value")
+ return v
+
+
+def coerce_value(value: str, declared, notes: list | None = None):
+ """A non-string parameter's text -> its value by the tool's declared type: JSON when it parses; else an array or an
+ object read with lenient_json (a note says so in `notes`); else the text itself, as before. An array or object that
+ arrived as a JSON string of itself is decoded once more."""
+ if declared == "string":
+ return value
+ try:
+ v = json.loads(value)
+ except ValueError:
+ if not value.lstrip().startswith(("[", "{", "`")) or (isinstance(declared, list) and "string" in declared):
+ return value
+ try:
+ v = lenient_json(value)
+ except (ValueError, RecursionError):
+ return value
+ if notes is not None:
+ notes.append("a damaged JSON " + ("array" if isinstance(v, list) else "object") + " argument was read as written")
+ return v
+ if isinstance(v, str) and declared in ("array", "object") and v.lstrip().startswith(("[", "{")):
+ try:
+ return json.loads(v)
+ except ValueError:
+ return coerce_value(v, declared, notes)
+ return v
+
+
+def parse_tool_call(body: str, schema: dict | None = None, notes: list | None = None) -> ToolCall:
"""`<function=NAME>\\n<parameter=P>\\nVALUE\\n</parameter>...</function>` -> ToolCall. Values are JSON-decoded
when the tool's schema says the parameter is not a string (or, without a schema, when they parse as JSON
objects/arrays/numbers/booleans)."""
@@ -938,14 +1083,7 @@ def parse_tool_call(body: str, schema: dict | None = None) -> ToolCall:
value = value[1:]
if value.endswith("\n"):
value = value[:-1]
- declared = (props.get(pname) or {}).get("type")
- if declared == "string":
- args[pname] = value
- else:
- try:
- args[pname] = json.loads(value)
- except ValueError:
- args[pname] = value
+ args[pname] = coerce_value(value, (props.get(pname) or {}).get("type"), notes)
return ToolCall(name=name, arguments=args)
@@ -1000,6 +1138,9 @@ class OutputParser:
# reasoning text before them is tracked (code fence, inline code, current line) to tell an act from a quote.
self.pending: list[list] = []
self.rescued = 0 # calls delivered from the reasoning
+ self.notes: list[str] = [] # #4427: what was repaired in the arguments of the calls (a damaged JSON array or object)
+ self.after_think = False # #4077: the reasoning ended at a bare `</think>` (a second one in the answer is a stray)
+ self.stray_think = 0 # ... and the strays of that kind dropped from the answer
self.refused = 0 # declared calls kept as reasoning text (quoted, or the turn was cut)
self.fence, self.line, self.ticks = "", "", 0
# recover (opt-in, the config's "tool_call_recovery"): a declared tool's call written next to the template's
@@ -1165,6 +1306,18 @@ class OutputParser:
return None
return after[0] not in "\r\n"
+ def _stray_think_at(self) -> int:
+ """#4077: where self.buf has a `</think>` at the start of a line, in the answer of a turn whose reasoning
+ already ended at one; -1 for none. A fence or backtick is no reason to keep it: the reasoning may have left
+ one open, and the answer's own tracking of code is wrong from there on."""
+ k = self.buf.find(THINK_END)
+ while k >= 0:
+ before = self.buf[:k]
+ if not (before.rsplit("\n", 1)[1] if "\n" in before else self.line + before).strip():
+ return k
+ k = self.buf.find(THINK_END, k + 1)
+ return -1
+
def _release(self, deliver: bool) -> list[Event]:
"""Settle the calls waiting in self.pending: events for real calls, or all of it back as reasoning text."""
out = []
@@ -1290,10 +1443,7 @@ class OutputParser:
value = value[1:]
if value.endswith("\n"):
value = value[:-1]
- try:
- v = json.loads(value)
- except ValueError:
- v = value
+ v = coerce_value(value, self.sdeclared.get(self.spname)) # the note comes with the final call (parse_tool_call)
args(json.dumps(v, ensure_ascii=False))
self.sp += end + len(PARAM_END)
self.ss = "between"
@@ -1342,22 +1492,26 @@ class OutputParser:
# as a call) until it ends: a declared name is then a tool call, anything else stays reasoning text.
body = self.buf[len(CALL_START):]
end, think = call_end(body), body.find(THINK_END)
+ drop = len(CALL_END)
+ f = function_end(body) # #4427: as in the answer, a following call opener confirms an omitted closer
+ if f >= 0 and (end < 0 or f < end) and re.match(r"<tool_call>\s*<function=\s*[^<>\s]+\s*>", body[f:].lstrip()):
+ end, drop = f, 0
if end >= 0 and (think < 0 or think >= end):
call = None
try:
name = body[:end].strip()[len("<function="):].split(">", 1)[0].strip()
name = self.aliases.get(name, name)
if name in self.schemas:
- call = parse_tool_call(body[:end], self.schemas.get(name))
+ call = parse_tool_call(body[:end], self.schemas.get(name), self.notes)
call.name = name
except ValueError:
pass
- raw = self.buf[:len(CALL_START) + end + len(CALL_END)]
+ raw = self.buf[:len(CALL_START) + end + drop]
if call is not None or self.pending:
self.pending.append([raw, call]) # settled by what follows it (see __init__)
else:
out.append(Event("reasoning", self._track(raw)))
- self.buf = body[end + len(CALL_END):]
+ self.buf = body[end + drop:]
self.state = "reasoning"
elif think >= 0: # the thinking ended inside it: it never was a call
out += self._release(False)
@@ -1421,7 +1575,7 @@ class OutputParser:
if i:
out.append(Event("reasoning", self._track(self.buf[:i])))
self.buf = self.buf[i + len(THINK_END):]
- self.state, self.lead = "content", True
+ self.state, self.lead, self.after_think = "content", True, True
self.fence, self.line, self.ticks = "", "", 0 # the answer's own text starts here
elif self.state == "content":
if self.lead: # newlines right after </think> or a call
@@ -1431,6 +1585,19 @@ class OutputParser:
return out
self.buf, self.lead = stripped, False
i = self.buf.find(CALL_START)
+ stray = self._stray_think_at() if self.after_think else -1
+ if stray >= 0 and (i < 0 or stray < i):
+ # #4077: the reasoning ended at a `</think>` the model wrote while thinking (quoting the format on a
+ # line of its own), and this is the one that ends it. What came between was reasoning, already sent
+ # as the answer; the tag is dropped and the text after it starts over, so a call that follows is
+ # not taken for a quote by the code fence or backtick the reasoning left open.
+ if stray:
+ out.append(Event("content", self._track(self.buf[:stray])))
+ self.buf = self.buf[stray + len(THINK_END):]
+ self.fence, self.line, self.ticks = "", "", 0
+ self.lead = True
+ self.stray_think += 1
+ continue
b, decided = self._bare_opener() if self.recover and self.schemas else (-1, False)
if b >= 0 and (i < 0 or b < i):
# recover: <function=NAME> alone at the start of a line - the call without its <tool_call>.
@@ -1449,7 +1616,7 @@ class OutputParser:
if i < 0:
# Hold a partial tag AND the newlines before it: if a tool call follows, they are dropped,
# so emitting them early would make streamed and whole outputs differ.
- j = len(self.buf) - self._hold(self.buf, (CALL_START,))
+ j = len(self.buf) - self._hold(self.buf, (CALL_START, THINK_END) if self.after_think else (CALL_START,))
if self.recover and self.schemas: # a line that may be becoming a bare <function=
k = self.buf.rfind("\n") + 1
tail = self.buf[k:]
@@ -1610,6 +1777,21 @@ class OutputParser:
how the turn ended: calls waiting from the reasoning become real calls only on a natural stop (None = stop);
a turn cut by max tokens (or cancelled, or failed) keeps them as reasoning text (#1058)."""
out = []
+ if self.state == "rcall" and reason in (None, "stop"):
+ # #4427: a call in the reasoning whose `</function>` is the end of the output (the `</tool_call>` left out)
+ # is as whole as one in the answer: it waits for the turn's end like the others
+ body = self.buf[len(CALL_START):]
+ f = function_end(body)
+ name = body.lstrip()[len(FUNC_START):].split(">", 1)[0].strip() if f >= 0 else ""
+ name = self.aliases.get(name, name)
+ if f >= 0 and not body[f:].strip() and name in self.schemas:
+ try:
+ call = parse_tool_call(body[:f], self.schemas.get(name), self.notes)
+ call.name = name
+ self.pending.append([self.buf, call])
+ self.buf, self.state = "", "reasoning"
+ except ValueError:
+ pass
if self.pending:
out += self._release(reason in (None, "stop") and self.state == "reasoning" and not self.buf)
if self.state == "call" and self.bare and function_end(self.buf) >= 0:
@@ -1625,7 +1807,13 @@ class OutputParser:
if self.ss == "function_done" and not self.buf[self.sp:].strip() and reason in (None, "stop"):
# A natural end immediately after </function> retains native streaming recovery.
out += self._close_scan()
- out.append(Event("tool_call", call=self.scall))
+ call = self.scall
+ try: # the final event carries the call's arguments, as the one _finish_call sends does
+ call = parse_tool_call(self.buf, self.schemas.get(call.name), self.notes)
+ call.name, call.id = self.scall.name, self.scall.id
+ except ValueError:
+ pass
+ out.append(Event("tool_call", call=call))
elif self.ss == "function_done":
out.append(Event("content", self._track(CALL_START + self.buf)))
self.buf = ""
@@ -1651,7 +1839,7 @@ class OutputParser:
try:
if raw_name != raw_name.strip() and name not in self.schemas:
raise ValueError("whitespace tool name is not declared")
- call = parse_tool_call(body, self.schemas.get(name))
+ call = parse_tool_call(body, self.schemas.get(name), self.notes)
call.name = name
except ValueError:
if not self.recover:
diff --git a/serve/server.py b/serve/server.py
index 8c5213b1f..6af172e44 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -4813,6 +4813,8 @@ class Service:
("strata:drafts_accepted_total", "counter", "Speculative draft tokens accepted", t["drafts_accepted"]),
("strata:tool_calls_unread_total", "counter", "<tool_call> blocks returned as text (an undeclared name or a form the template does not use)",
t.get("tool_calls_unread", 0)),
+ ("strata:tool_args_repaired_total", "counter", "Tool call arguments (a JSON array or object) that were damaged and read as written",
+ t.get("tool_args_repaired", 0)),
("strata:model_loaded", "gauge", "1 while the model is loaded", int(self.loaded())),
]
return "".join(f"# HELP {name} {help_}\n# TYPE {name} {kind}\n{name} {value}\n"
@@ -5759,6 +5761,14 @@ class Service:
self.totals["tool_calls_from_reasoning"] = self.totals.get("tool_calls_from_reasoning", 0) + parser.rescued
print(f"[strata] tool calls inside the thinking: {parser.rescued} read as calls, {parser.refused} kept as "
f"reasoning (quoted, or the turn did not end on a stop)", flush=True)
+ if getattr(parser, "stray_think", 0):
+ # #4077: the model wrote a closing think tag while thinking; the reasoning after it was sent as the answer
+ self.totals["stray_think_closers"] = self.totals.get("stray_think_closers", 0) + parser.stray_think
+ print(f"[strata] {parser.stray_think} extra closing think tag(s) in the answer were dropped (the model wrote one "
+ "while thinking, so the rest of its reasoning went out as the answer; a call after it still counts)", flush=True)
+ if getattr(parser, "notes", None):
+ self.totals["tool_args_repaired"] = self.totals.get("tool_args_repaired", 0) + len(parser.notes)
+ print(f"[strata] {len(parser.notes)} tool call argument(s) repaired: {'; '.join(sorted(set(parser.notes)))}", flush=True)
rejected = []
if parser.unread:
named = getattr(parser, "unread_names", None)
diff --git a/serve/test_tool_corpus.py b/serve/test_tool_corpus.py
new file mode 100644
index 000000000..4fe2ab8a0
--- /dev/null
+++ b/serve/test_tool_corpus.py
@@ -0,0 +1,279 @@
+"""A corpus of model outputs for the tool-call and reasoning parser (#4427, #4078, #4077).
+
+Each specimen is the text a model wrote, the tools the request declared, and what a client must get: the calls (name and
+arguments), the answer text, the reasoning text. The sources are named in each entry: the reporters' own examples in
+the issues, the Qwen3.8-Flash-Next form (`<tool_call><function=NAME><parameter=P>`), the drift shapes of the 1,462-turn
+corpus the recovery switch was written for, and DeepSeek-V4.1's DSML form.
+
+Every specimen is read whole, one byte at a time, in pieces of 3 and 7 bytes, and cut in two at every byte; with
+streaming of the arguments on and off. All of them must give the same events, and the streamed arguments of a call
+must be its final arguments.
+
+ python -m unittest serve.test_tool_corpus -v
+"""
+from __future__ import annotations
+
+import json
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from serve.dsml import DsmlOutputParser # noqa: E402
+from serve.frontend import OutputParser # noqa: E402
+
+
+def tool(name, **props):
+ return {"name": name, "parameters": {"type": "object", "properties": props}}
+
+
+S, I, A, O = {"type": "string"}, {"type": "integer"}, {"type": "array"}, {"type": "object"}
+TOOLS = [tool("todo_write", todos=A), tool("run_command", command=S, timeout=I), tool("write_file", path=S, content=S),
+ tool("read", path=S), tool("bash", command=S), tool("edit", path=S, edits=A, options=O)]
+
+
+def qwen(name, **params):
+ """The call as Qwen3.8-Flash-Next writes it (the chat template's own form)."""
+ body = "".join(f"<parameter={k}>\n{v}\n</parameter>\n" for k, v in params.items())
+ return f"<tool_call>\n<function={name}>\n{body}</function>\n</tool_call>"
+
+
+def dsml(*calls):
+ """DeepSeek-V4.1's DSML block: calls are (name, [(parameter, value, is_string)])."""
+ out = ["<|DSML| calls>\n"]
+ for name, params in calls:
+ out.append(f'<|DSML| invoke name="{name}">\n')
+ for k, v, is_str in params:
+ out.append(f'<|DSML| parameter name="{k}" string="{str(is_str).lower()}">{v}</|DSML| parameter>\n')
+ out.append("</|DSML| invoke>\n")
+ return "".join(out) + "</|DSML| calls>"
+
+
+NESTED = qwen("read", path="nested.txt")
+HEREDOC = f"python3 - <<'PY'\ntext = '''{NESTED}'''\nprint('tail that must not be lost')\nPY"
+REOPEN = "first line\n</parameter>\n<parameter=content>\nlast line"
+TODO_OK = ('[{"content": "Generate lorem ipsum page texture atlas (PIL) + inspect", "status": "in_progress"}, '
+ '{"content": "Blender script: book geometry + materials", "status": "pending"}]')
+TODO_STRAY = ('[{"content": "Init git repo with LFS", "status": "in_progress", "pending": false, "completed": false}, '
+ '{"content": "Write Blender build script", "status": "pending", "in_progress", "pending": false, '
+ '"completed": false}]')
+TODO_STRAY_WANT = [{"content": "Init git repo with LFS", "status": "in_progress", "pending": False, "completed": False},
+ {"content": "Write Blender build script", "status": "pending", "pending": False, "completed": False}]
+STRAY_VALUE = "reasoning sample:\n</think>\nanswer sample"
+
+# name: dict(text, calls, content, reasoning, thinking, recover, finish, tools, unread, notes, source)
+CORPUS = {}
+
+
+def add(name, text, calls=(), content="", reasoning="", thinking=False, recover=False, finish="stop", tools=None,
+ unread=0, notes=0, source="", parser="qwen", stream_only=False, announced=False):
+ assert name not in CORPUS, name
+ CORPUS[name] = dict(text=text, calls=[tuple(c) for c in calls], content=content, reasoning=reasoning,
+ thinking=thinking, recover=recover, finish=finish, tools=tools or TOOLS, unread=unread,
+ notes=notes, source=source, parser=parser, stream_only=stream_only, announced=announced)
+
+
+# ---- the reporters' examples ------------------------------------------------------------------------------------
+add("4427-todos-valid", qwen("todo_write", todos=TODO_OK), [("todo_write", {"todos": json.loads(TODO_OK)})],
+ source="#4427 FraYoshi, first example (the Harness todo_write call)")
+add("4427-todos-stray-items", "Old tree project files are present but the task is fresh.\n\n" + qwen("todo_write", todos=TODO_STRAY),
+ [("todo_write", {"todos": TODO_STRAY_WANT})], content="Old tree project files are present but the task is fresh.",
+ notes=1, source="#4427 FraYoshi, second example: `\"status\": \"pending\", \"in_progress\", \"pending\": false`")
+add("4427-todos-single-quotes", qwen("todo_write", todos="[{'content': 'a', 'status': 'pending'}]"),
+ [("todo_write", {"todos": [{"content": "a", "status": "pending"}]})], notes=1, source="#4427 shape: Python quotes")
+add("4427-todos-trailing-comma-and-open-bracket", qwen("todo_write", todos='[{"content": "a", "status": "pending"},'),
+ [("todo_write", {"todos": [{"content": "a", "status": "pending"}]})], notes=1, source="#4427 shape: a cut array")
+add("4427-todos-json-string-of-an-array", qwen("todo_write", todos=json.dumps(TODO_OK)),
+ [("todo_write", {"todos": json.loads(TODO_OK)})], source="#4427 shape: the array written as a JSON string")
+add("4427-todos-not-json-stays-text", qwen("todo_write", todos="buy milk"), [("todo_write", {"todos": "buy milk"})],
+ source="#4427: text that is no array stays the text it is")
+add("4427-one-line", "Let me run ls and read the images.\n\n<tool_call> <function=run_command> <parameter=command> "
+ "ls -la /home/user/imap-attachments/ 2>&1 | head -30 </parameter> </function> </tool_call>",
+ [("run_command", {"command": " ls -la /home/user/imap-attachments/ 2>&1 | head -30 "})],
+ content="Let me run ls and read the images.", source="#4427 miscdebris: the tags on one line")
+add("4427-one-line-in-thinking", "Let me run ls and read the images.\n\n<tool_call> <function=run_command> <parameter=command> "
+ "ls -la 2>&1 | head -30 </parameter> </function> </tool_call>", [("run_command", {"command": " ls -la 2>&1 | head -30 "})],
+ reasoning="Let me run ls and read the images.\n\n", thinking=True,
+ source="#4427 miscdebris: written inside the thinking, the turn ends on it")
+add("4078-nested-call", qwen("write_file", path="fixture.txt", content="first line\n" + NESTED + "\nlast line"),
+ [("write_file", {"path": "fixture.txt", "content": "first line\n" + NESTED + "\nlast line"})],
+ source="#4078 LispEngineer / ackalker: a value holding a whole call")
+add("4078-reopened-parameter", qwen("write_file", path="fixture.txt", content=REOPEN),
+ [("write_file", {"path": "fixture.txt", "content": REOPEN})], source="#4078: a value that reopens its own parameter")
+add("4078-heredoc", qwen("bash", command=HEREDOC), [("bash", {"command": HEREDOC})],
+ source="#4078 Tsury: a heredoc that prints a quoted call")
+add("4078-call-format-in-a-file", qwen("write_file", path="DOC.md", content="Close a call with </function> and then </tool_call>.\nlike this."),
+ [("write_file", {"path": "DOC.md", "content": "Close a call with </function> and then </tool_call>.\nlike this."})],
+ source="#210: a file that documents the format")
+add("4077-stray-think-line", "The user wants a file with a literal.\n" + STRAY_VALUE + "\nSo I write it.\n</think>\n\n" +
+ qwen("write_file", path="out.txt", content=STRAY_VALUE),
+ [("write_file", {"path": "out.txt", "content": STRAY_VALUE})], thinking=True,
+ reasoning="The user wants a file with a literal.\nreasoning sample:\n",
+ content="answer sample\nSo I write it.", source="#4077 Tsury: the model quoted the closing tag at a line start")
+add("4077-stray-think-in-a-fence", "Content:\n```\n" + STRAY_VALUE + "\n```\nNow call it.\n</think>\n\n" +
+ qwen("write_file", path="out.txt", content=STRAY_VALUE),
+ [("write_file", {"path": "out.txt", "content": STRAY_VALUE})], thinking=True,
+ reasoning="Content:\n```\nreasoning sample:\n", content="answer sample\n```\nNow call it.",
+ source="#4077: the quote sat in a code fence, so the fence closes in the 'answer'")
+add("4077-stray-think-after-a-backtick", "The text is `" + STRAY_VALUE + "`. Call it.\n</think>\n\n" +
+ qwen("write_file", path="out.txt", content=STRAY_VALUE),
+ [("write_file", {"path": "out.txt", "content": STRAY_VALUE})], thinking=True,
+ reasoning="The text is `reasoning sample:\n", content="answer sample`. Call it.", source="#4077: an open backtick")
+add("4077-quoted-think-stays-reasoning", "I will not write `</think>` yet, I keep thinking.\nDone thinking.\n</think>\n\nThe answer.",
+ thinking=True, reasoning="I will not write `</think>` yet, I keep thinking.\nDone thinking.\n", content="The answer.",
+ source="#537: a closing tag after a backtick is still reasoning (control)")
+add("4077-closer-in-the-answer-text-is-kept-mid-line", "Reasoning.\n</think>\n\nWrite </think> to end the thinking.",
+ thinking=True, reasoning="Reasoning.\n", content="Write </think> to end the thinking.",
+ source="a closing tag inside a sentence of the answer is text (control)")
+add("4077-no-second-closer-is-the-normal-case", "Reasoning.\n</think>\n\n" + qwen("read", path="a.txt"),
+ [("read", {"path": "a.txt"})], thinking=True, reasoning="Reasoning.\n", source="control")
+
+# ---- the Qwen3.8-Flash-Next form and its slips ------------------------------------------------------------------
+add("qwen-one-call", "I will read it.\n\n" + qwen("read", path="a.txt"), [("read", {"path": "a.txt"})],
+ content="I will read it.", source="the template's own form")
+add("qwen-int-argument", qwen("run_command", command="make -j8", timeout="30"),
+ [("run_command", {"command": "make -j8", "timeout": 30})], source="a declared integer")
+add("qwen-after-thinking", "Plan: read the file.\n</think>\n\n" + qwen("read", path="a.txt"), [("read", {"path": "a.txt"})],
+ thinking=True, reasoning="Plan: read the file.\n", source="the normal agent turn")
+add("qwen-call-in-the-thinking", "Plan: read the file.\n\n" + qwen("read", path="a.txt"), [("read", {"path": "a.txt"})],
+ thinking=True, reasoning="Plan: read the file.\n\n", source="#804: the turn ends inside the thinking, on the call")
+add("qwen-three-calls", "Reading all three.\n\n" + "\n".join(qwen("read", path=f"{n}.txt") for n in "abc"),
+ [("read", {"path": "a.txt"}), ("read", {"path": "b.txt"}), ("read", {"path": "c.txt"})], content="Reading all three.",
+ source="parallel calls, one wrapper each")
+add("qwen-calls-with-prose-between", qwen("read", path="a.txt") + "\nand now the other one:\n" + qwen("read", path="b.txt"),
+ [("read", {"path": "a.txt"}), ("read", {"path": "b.txt"})], content="and now the other one:", source="prose between calls")
+add("qwen-three-calls-in-the-thinking", "Plan.\n\n" + "\n".join(qwen("read", path=f"{n}.txt") for n in "ab"),
+ [("read", {"path": "a.txt"}), ("read", {"path": "b.txt"})], thinking=True, reasoning="Plan.\n\n\n",
+ source="#804: two calls at the end of the thinking")
+add("qwen-function-close-left-out", "<tool_call>\n<function=write_file>\n<parameter=path>\nout.txt\n</parameter>\n"
+ "<parameter=content>\nhello\n</parameter>\n</tool_call>", [("write_file", {"path": "out.txt", "content": "hello"})],
+ source="#4427 shape: no </function> (PR #3960's Coder-L slip)")
+add("qwen-function-close-left-out-in-thinking", "Writing.\n\n<tool_call>\n<function=write_file>\n<parameter=path>\nout.txt\n"
+ "</parameter>\n<parameter=content>\nhello\n</parameter>\n</tool_call>", [("write_file", {"path": "out.txt", "content": "hello"})],
+ thinking=True, reasoning="Writing.\n\n", source="the same inside the thinking")
+add("qwen-wrapper-close-left-out", "<tool_call>\n<function=read>\n<parameter=path>\na.txt\n</parameter>\n</function>",
+ [("read", {"path": "a.txt"})], stream_only=True, source="the output ends on </function> (the server always streams)")
+add("qwen-wrapper-close-left-out-in-thinking", "Plan.\n\n<tool_call>\n<function=read>\n<parameter=path>\na.txt\n</parameter>\n</function>",
+ [("read", {"path": "a.txt"})], thinking=True, reasoning="Plan.\n\n", source="the same inside the thinking")
+add("qwen-wrapper-close-left-out-two-calls", qwen("read", path="a.txt").replace("\n</tool_call>", "") + "\n" + qwen("read", path="b.txt"),
+ [("read", {"path": "a.txt"}), ("read", {"path": "b.txt"})], source="a following opener confirms the missing closer")
+add("qwen-wrapper-close-left-out-two-calls-in-thinking", "Plan.\n\n" + qwen("read", path="a.txt").replace("\n</tool_call>", "") +
+ "\n" + qwen("read", path="b.txt"), [("read", {"path": "a.txt"}), ("read", {"path": "b.txt"})], thinking=True,
+ reasoning="Plan.\n\n\n", source="the same inside the thinking")
+add("qwen-undeclared-name-is-text", qwen("format_disk", target="/"), content=qwen("format_disk", target="/"), unread=1,
+ source="by design: a call for a tool the request did not declare comes back as text (and is counted)")
+add("qwen-cut-inside-a-value", "<tool_call>\n<function=write_file>\n<parameter=path>\nout.txt\n</parameter>\n<parameter=content>\nhal",
+ content="<tool_call>\n<function=write_file>\n<parameter=path>\nout.txt\n</parameter>\n<parameter=content>\nhal",
+ finish="length", announced=True, source="an unterminated tag: the output stopped at the token limit; no call")
+add("qwen-cut-inside-the-opener", "Now.\n\n<tool_call>\n<function=writ", content="Now.\n\n<tool_call>\n<function=writ",
+ finish="length", source="cut inside the name")
+add("qwen-cut-after-function-close-on-length", "<tool_call>\n<function=read>\n<parameter=path>\na.txt\n</parameter>\n</function>",
+ finish="length", content="<tool_call>\n<function=read>\n<parameter=path>\na.txt\n</parameter>\n</function>",
+ source="a turn cut by the token limit is not a natural stop: no call")
+add("qwen-quoted-in-a-fence", "Example:\n```\n" + qwen("read", path="a.txt") + "\n```", content="Example:\n```\n" + qwen("read", path="a.txt") + "\n```",
+ source="#1058: a call in a code fence is a quote")
+add("qwen-prose-names-the-format", "Calls are written as <tool_call> blocks.", content="Calls are written as <tool_call> blocks.",
+ source="prose that names the tag")
+
+# ---- the drift shapes (tool_call_recovery on; the corpus of 1,462 agent turns) ----------------------------------
+PW = "<parameter=path>\na.md\n</parameter>\n<parameter=content>\nhi\n</parameter>\n</function>"
+WRITE_TOOLS = [tool("write", path=S, content=S)]
+add("drift-parameter-as-opener", f"<tool_call>\n<parameter=write>\n{PW}\n</tool_call>", [("write", {"path": "a.md", "content": "hi"})],
+ recover=True, tools=WRITE_TOOLS, source="corpus 300dfba2 (x11)")
+add("drift-json-in-the-wrapper", '<tool_call>\n{"name": "write", "arguments": {"path": "a.md", "content": "hi"}}\n</tool_call>',
+ [("write", {"path": "a.md", "content": "hi"})], recover=True, tools=WRITE_TOOLS, source="corpus 5e847539 (x6)")
+add("drift-bare-function", f"Writing it now.\n\n<function=write>\n{PW}", [("write", {"path": "a.md", "content": "hi"})],
+ content="Writing it now.", recover=True, tools=WRITE_TOOLS, source="corpus 395efd4c / 6e71eed4 (x5 each)")
+add("drift-batch-in-one-wrapper", f"<tool_call>\n<function=write>\n{PW}\n<function=write>\n{PW.replace('a.md', 'b.md')}\n</tool_call>",
+ [("write", {"path": "a.md", "content": "hi"}), ("write", {"path": "b.md", "content": "hi"})], recover=True,
+ tools=WRITE_TOOLS, source="corpus 0912870b / 51b155c9")
+
+# ---- DeepSeek-V4.1 (DSML) ---------------------------------------------------------------------------------------
+add("dsml-one-call", dsml(("write_file", [("path", "out.txt", True), ("content", "hello\nworld", True)])),
+ [("write_file", {"path": "out.txt", "content": "hello\nworld"})], parser="dsml", source="the model's own template")
+add("dsml-after-thinking", "Plan.</think>\n\n" + dsml(("read", [("path", "a.txt", True)])), [("read", {"path": "a.txt"})],
+ thinking=True, reasoning="Plan.", parser="dsml", source="a thinking turn")
+add("dsml-typed-arguments", dsml(("run_command", [("command", "make", True), ("timeout", "30", False)]),
+ ("todo_write", [("todos", TODO_OK, False)])),
+ [("run_command", {"command": "make", "timeout": 30}), ("todo_write", {"todos": json.loads(TODO_OK)})], parser="dsml",
+ source="string=\"false\" values are JSON")
+add("dsml-damaged-array", dsml(("todo_write", [("todos", TODO_STRAY, False)])), [("todo_write", {"todos": TODO_STRAY_WANT})],
+ notes=1, parser="dsml", source="#4427 shape inside DSML")
+add("dsml-several-calls-and-text", "Reading both.\n\n" + dsml(("read", [("path", "a.txt", True)]), ("read", [("path", "b.txt", True)])),
+ [("read", {"path": "a.txt"}), ("read", {"path": "b.txt"})], content="Reading both.", parser="dsml", source="a batch")
+add("dsml-value-holds-a-qwen-call", dsml(("write_file", [("path", "x.txt", True), ("content", NESTED, True)])),
+ [("write_file", {"path": "x.txt", "content": NESTED})], parser="dsml", source="#4078 shape in DSML")
+add("dsml-stray-think", "Quote:\n</think>\nmore thinking.\n</think>\n\n" + dsml(("read", [("path", "a.txt", True)])),
+ [("read", {"path": "a.txt"})], thinking=True, reasoning="Quote:\n", content="more thinking.", parser="dsml",
+ source="#4077 shape in DSML")
+add("dsml-cut-call", dsml(("read", [("path", "a.txt", True)]))[:60], finish="length", content=None,
+ parser="dsml", source="a DSML call cut by the token limit is no call (its text is the translator's, not checked)")
+
+
+def run(spec, pieces, stream_tools):
+ cls = DsmlOutputParser if spec["parser"] == "dsml" else OutputParser
+ p = cls(thinking=spec["thinking"], tools=spec["tools"], stream_tools=stream_tools, recover=spec["recover"],
+ declared_only=True)
+ evs = []
+ for piece in pieces:
+ evs += p.feed(piece)
+ evs += p.finish(spec["finish"])
+ return evs, p
+
+
+def cut(text, size):
+ return [text[i:i + size] for i in range(0, len(text), size)]
+
+
+class ToolCorpus(unittest.TestCase):
+ def check(self, name, spec, pieces, stream_tools, label):
+ evs, p = run(spec, pieces, stream_tools)
+ calls = [(e.call.name, e.call.arguments) for e in evs if e.kind == "tool_call"]
+ text = lambda kind: "".join(e.text for e in evs if e.kind == kind) # noqa: E731
+ where = f"{name} [{label}, stream_tools={stream_tools}]"
+ self.assertEqual(calls, spec["calls"], where)
+ # a call announced to the client and then cut stays unfinished there (#211): its text is not repeated as content
+ if spec["content"] is not None:
+ self.assertEqual(text("content").strip(), "" if spec["announced"] and stream_tools else spec["content"], where)
+ self.assertEqual(text("reasoning").strip(), spec["reasoning"].strip(), where)
+ self.assertEqual(p.unread, spec["unread"], where)
+ self.assertEqual(len(p.notes), spec["notes"], where)
+ if stream_tools:
+ for e in (x for x in evs if x.kind == "tool_call"):
+ streamed = "".join(x.text for x in evs if x.kind == "tool_args" and x.call is not None and x.call.id == e.call.id)
+ if streamed:
+ self.assertEqual(json.loads(streamed), e.call.arguments, where)
+ return evs
+
+ def test_every_specimen_whole_and_in_pieces(self):
+ for name, spec in CORPUS.items():
+ for stream_tools in (True,) if spec["stream_only"] else (False, True):
+ self.check(name, spec, [spec["text"]], stream_tools, "whole")
+ for size in (1, 3, 7):
+ self.check(name, spec, cut(spec["text"], size), stream_tools, f"pieces of {size}")
+
+ def test_every_specimen_cut_in_two_at_every_byte(self):
+ for name, spec in CORPUS.items():
+ text = spec["text"]
+ for at in range(1, len(text)):
+ self.check(name, spec, [text[:at], text[at:]], True, f"cut at {at}")
+
+ def test_the_same_events_whatever_the_pieces(self):
+ """Text and calls of a specimen do not depend on how the stream was cut (the ids aside)."""
+ for name, spec in CORPUS.items():
+ whole, _ = run(spec, [spec["text"]], True)
+ want = [(e.kind, e.text) for e in whole if e.kind in ("content", "reasoning")]
+ joined = lambda evs, k: "".join(t for kind, t in evs if kind == k) # noqa: E731
+ for size in (1, 5):
+ got, _ = run(spec, cut(spec["text"], size), True)
+ got = [(e.kind, e.text) for e in got if e.kind in ("content", "reasoning")]
+ for kind in ("content", "reasoning"):
+ self.assertEqual(joined(got, kind), joined(want, kind), f"{name} pieces of {size}")
+
+ def test_the_sources_are_named(self):
+ for name, spec in CORPUS.items():
+ self.assertTrue(spec["source"], name)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/serve/test_tool_routes.py b/serve/test_tool_routes.py
new file mode 100644
index 000000000..bdc72d4b3
--- /dev/null
+++ b/serve/test_tool_routes.py
@@ -0,0 +1,161 @@
+"""#4427, #4077, #4078 through the HTTP routes: what a client gets for the damaged calls and the stray closing think tag.
+
+The model's text is replayed by the mock engine; the reply is read from Chat Completions, the Anthropic Messages API and
+the Responses API, each whole and streamed. Every route must hand over the same calls with the same arguments.
+
+ python -m unittest serve.test_tool_routes -v
+"""
+from __future__ import annotations
+
+import contextlib
+import http.client
+import io
+import json
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from serve.frontend import ChatTemplate # noqa: E402
+from serve.server import ByteTokenizer, MockEngine, Service, serve # noqa: E402
+
+ROOT = Path(__file__).resolve().parents[1]
+TEMPLATE = ChatTemplate(ROOT / "serve/chat_template.jinja")
+PROPS = {"todos": {"type": "array"}, "path": {"type": "string"}, "content": {"type": "string"}}
+NAMES = ("todo_write", "write_file")
+
+
+def call(name, **params):
+ body = "".join(f"<parameter={k}>\n{v}\n</parameter>\n" for k, v in params.items())
+ return f"<tool_call>\n<function={name}>\n{body}</function>\n</tool_call>"
+
+
+class Routes(unittest.TestCase):
+ def read(self, script, **service):
+ """{route: (finish, [(name, arguments)], answer text)} for the model's `script`, on every route."""
+ tok = ByteTokenizer()
+ svc = Service(MockEngine(tok, script, max_context=16384), tok, TEMPLATE)
+ for k, v in service.items():
+ setattr(svc, k, v)
+ httpd = serve(svc, port=0)
+ port = httpd.server_address[1]
+ out = {}
+
+ def post(path, body):
+ c = http.client.HTTPConnection("127.0.0.1", port, timeout=60)
+ try:
+ with contextlib.redirect_stdout(io.StringIO()):
+ c.request("POST", path, body=json.dumps(body).encode(),
+ headers={"Content-Type": "application/json", "anthropic-version": "2023-06-01"})
+ return c.getresponse().read().decode()
+ finally:
+ c.close()
+
+ def sse(raw):
+ return [json.loads(line[6:]) for line in raw.splitlines() if line.startswith("data: {")]
+
+ chat_tools = [{"type": "function", "function": {"name": n, "parameters": {"type": "object", "properties": PROPS}}}
+ for n in NAMES]
+ try:
+ for stream in (False, True):
+ raw = post("/v1/chat/completions", {"model": "x", "max_tokens": 900, "stream": stream, "tools": chat_tools,
+ "messages": [{"role": "user", "content": "go"}]})
+ if stream:
+ evs, calls, text, finish = sse(raw), {}, "", None
+ for e in evs:
+ ch = e["choices"][0]
+ text += ch["delta"].get("content") or ""
+ finish = ch.get("finish_reason") or finish
+ for tc in ch["delta"].get("tool_calls") or []:
+ slot = calls.setdefault(tc["index"], {"name": None, "args": ""})
+ slot["name"] = (tc.get("function") or {}).get("name") or slot["name"]
+ slot["args"] += (tc.get("function") or {}).get("arguments") or ""
+ got = [(c["name"], json.loads(c["args"])) for _, c in sorted(calls.items())]
+ else:
+ d = json.loads(raw)["choices"][0]
+ text, finish = d["message"].get("content") or "", d["finish_reason"]
+ got = [(t["function"]["name"], json.loads(t["function"]["arguments"])) for t in d["message"].get("tool_calls") or []]
+ out["chat", stream] = (finish, got, text.strip())
+
+ raw = post("/v1/messages", {"model": "x", "max_tokens": 900, "stream": stream,
+ "tools": [{"name": n, "input_schema": {"type": "object", "properties": PROPS}} for n in NAMES],
+ "messages": [{"role": "user", "content": "go"}]})
+ if stream:
+ evs, blocks, text, finish = sse(raw), {}, "", None
+ for e in evs:
+ if e["type"] == "content_block_start" and e["content_block"]["type"] == "tool_use":
+ blocks[e["index"]] = {"name": e["content_block"]["name"], "json": ""}
+ elif e["type"] == "content_block_delta":
+ if e["delta"]["type"] == "input_json_delta":
+ blocks[e["index"]]["json"] += e["delta"]["partial_json"]
+ elif e["delta"]["type"] == "text_delta":
+ text += e["delta"]["text"]
+ elif e["type"] == "message_delta":
+ finish = e["delta"]["stop_reason"]
+ got = [(b["name"], json.loads(b["json"] or "{}")) for _, b in sorted(blocks.items())]
+ else:
+ d = json.loads(raw)
+ finish = d["stop_reason"]
+ text = "".join(b.get("text", "") for b in d["content"] if b["type"] == "text")
+ got = [(b["name"], b["input"]) for b in d["content"] if b["type"] == "tool_use"]
+ out["anthropic", stream] = (finish, got, text.strip())
+
+ raw = post("/v1/responses", {"model": "x", "input": "go", "stream": stream, "store": False,
+ "tools": [{"type": "function", "name": n, "parameters": {"type": "object", "properties": PROPS}}
+ for n in NAMES]})
+ if stream:
+ items = [e["item"] for e in sse(raw) if e["type"] == "response.output_item.done"]
+ else:
+ items = json.loads(raw)["output"]
+ got = [(i["name"], json.loads(i["arguments"])) for i in items if i["type"] == "function_call"]
+ text = "".join(c["text"] for i in items if i["type"] == "message" for c in i["content"] if c["type"] == "output_text")
+ out["responses", stream] = ("-", got, text.strip())
+ finally:
+ httpd.shutdown()
+ httpd.server_close()
+ return out
+
+ def every_route(self, script, calls, text="", finish=("tool_calls", "tool_use"), **service):
+ got = self.read(script, **service)
+ for (route, stream), (fin, got_calls, got_text) in got.items():
+ with self.subTest(route=route, stream=stream):
+ self.assertEqual(got_calls, calls)
+ self.assertEqual(got_text, text)
+ if route == "chat":
+ self.assertEqual(fin, finish[0])
+ elif route == "anthropic":
+ self.assertEqual(fin, finish[1])
+
+ def test_a_damaged_array_argument_arrives_as_the_array(self):
+ stray = ('[{"content": "Init", "status": "in_progress", "pending": false}, '
+ '{"content": "Write", "status": "pending", "in_progress", "pending": false}]')
+ want = [{"content": "Init", "status": "in_progress", "pending": False},
+ {"content": "Write", "status": "pending", "pending": False}]
+ self.every_route("Plan.\n</think>\n\n" + call("todo_write", todos=stray), [("todo_write", {"todos": want})])
+
+ def test_a_call_without_its_function_closer(self):
+ text = ("Plan.\n</think>\n\n<tool_call>\n<function=write_file>\n<parameter=path>\nout.txt\n</parameter>\n"
+ "<parameter=content>\nhello\n</parameter>\n</tool_call>")
+ self.every_route(text, [("write_file", {"path": "out.txt", "content": "hello"})])
+
+ def test_a_call_without_its_wrapper_closer_at_the_end(self):
+ text = "Plan.\n</think>\n\n" + call("write_file", path="out.txt", content="hello").removesuffix("\n</tool_call>")
+ self.every_route(text, [("write_file", {"path": "out.txt", "content": "hello"})])
+
+ def test_a_value_that_quotes_a_whole_call(self):
+ value = "first line\n" + call("write_file", path="nested.txt") + "\nlast line"
+ self.every_route("Plan.\n</think>\n\n" + call("write_file", path="fixture.txt", content=value),
+ [("write_file", {"path": "fixture.txt", "content": value})])
+
+ def test_a_stray_closing_think_tag_keeps_the_call(self):
+ value = "reasoning sample:\n</think>\nanswer sample"
+ script = ("The content is\n```\n" + value + "\n```\nCall it.\n</think>\n\n" + call("write_file", path="out.txt", content=value))
+ got = self.read(script)
+ for (route, stream), (fin, calls, text) in got.items():
+ with self.subTest(route=route, stream=stream):
+ self.assertEqual(calls, [("write_file", {"path": "out.txt", "content": value})])
+ self.assertNotIn("</think>", text)
+
+
+if __name__ == "__main__":
+ unittest.main()