Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Fix #3441, #3075: a drifted tool-call opener is read as the call; a call for a tool the request did not declare stays text
Niko122170f424072824
diff --git a/docs/DETAILS.md b/docs/DETAILS.mdindex 1432c709d..dbcf4e43f 100644--- a/docs/DETAILS.md+++ b/docs/DETAILS.md@@ -991,6 +991,12 @@ print(r.choices[0].message.content)long prompt the stream sends keep-alives, so agents do not time out; the server window prints progress every15 s, and `GET /status` says what it is doing (`reading the prompt`, `answering`, tokens so far). Closing theconnection or pressing stop in your app really stops the model, so the next request starts at once.+- **Which tool calls come back as calls.** A call counts only when it names a tool the request declared. A call for+ another name, or any call in a Chat Completions or Anthropic request that declared no tools, is returned as the+ text it was written as, so a client never runs a tool it did not offer (the Responses route keeps its own rule+ for Codex's compaction requests). A call whose envelope drifted by a token, `<=NAME>`, `<function:NAME>` or `<NAME>`+ instead of `<function=NAME>`, is read as the call when NAME is a declared tool. A `<tool_call>` block that still+ does not read as a call is counted in the server window.- **Tool calls in other forms (opt-in).** `"tool_call_recovery": true` in `strata-<model>.json` reads a call of atool the request declared also when the model writes it as `<parameter=NAME>` instead of `<function=NAME>`, as JSON(`{"name": ..., "arguments": ...}`) inside `<tool_call>`, as a `<function=NAME>` block at the start of a linediff --git a/serve/frontend.py b/serve/frontend.pyindex 3fda9c56f..4cf0b4fab 100644--- a/serve/frontend.py+++ b/serve/frontend.py@@ -682,6 +682,10 @@ CALL_END = "</tool_call>"PARAM_END = "</parameter>"FUNC_START = "<function="FUNC_END = "</function>"+# #3441: the opener a model writes when it drifts by a token inside the envelope the template documents:+# `<=NAME>` (the word `function` dropped), `<function:NAME>`, `<function =NAME>`, or the bare `<NAME>`+_LOOSE_OPEN = re.compile(r"<\s*(?:function\s*[=:]\s*|=\s*)?([A-Za-z0-9_.\-]+)\s*>")+_LOOSE_PART = re.compile(r"<\s*(?:function[ \t]*[=:]?\s*|=\s*)?[A-Za-z0-9_.\-]*\s*$")def param_end(text: str, final: bool = False) -> int:@@ -880,11 +884,19 @@ class OutputParser:until it is complete, so clients never see `<tool_` or `</thi`."""def __init__(self, thinking: bool = True, tools: list[dict] | None = None, stream_tools: bool = False,- recover: bool = False):+ recover: bool = False, declared_only: bool = False):self.state = "reasoning" if thinking else "content"self.buf = ""self.lead = Falseself.schemas = {t.get("name"): t for t in tools or []}+ # #804/#3075: in the answer too, a call counts only for a tool the request declared. With tools declared a+ # call for another name stays text (the Responses spelling `namespace__tool` of a `namespace.tool` counts as+ # that tool); `declared_only` extends the rule to a request that declared none (Chat Completions, Anthropic:+ # the client has no tool to run). The Responses route leaves it off: a Codex compaction request keeps its+ # conversation's tools in the prompt only, and its calls come back as before.+ self.declared_only = declared_only+ self._alias = {k.replace(".", "__", 1) for k in self.schemas if isinstance(k, str) and "." in k}+ self.unread = 0 # #3441: `<tool_call>` blocks that opened like a call and were returned as text# stream_tools: a tool call is also reported while it is being written - "tool_start" (its name and id) as# soon as the name is known, then "tool_args" pieces of its JSON arguments (string parameters character by# character; other types whole, once complete) - before the final "tool_call". Without it, a client sees@@ -913,6 +925,30 @@ class OutputParser:self.fence, self.line, self.ticks = snapreturn ok+ def _callable(self, name: str) -> bool:+ """#3075: may a call for `name` be a call? A tool the request declared (or its Responses spelling); with+ none declared, only where `declared_only` is off."""+ if not self.schemas:+ return not self.declared_only+ return name in self.schemas or name in self._alias++ def _loose_open(self, after: str):+ """#3441: `after` (text after a `<tool_call>`, whitespace dropped) opens a call in a drifted form that names a+ declared tool -> (name, length of the opener); "wait" while it may still become one; None otherwise."""+ m = _LOOSE_OPEN.match(after)+ if m:+ name = m.group(1)+ return (name, m.end()) if self._callable(name) else None+ if ">" in after or "\n" in after or not after.startswith("<"):+ return None+ if _LOOSE_PART.match(after):+ bare = after[1:].strip().lstrip("=").strip()+ if after[1:].lstrip().startswith(("=", "function")):+ return "wait"+ if any(k.startswith(bare) for k in list(self.schemas) + list(self._alias) if isinstance(k, str)):+ return "wait"+ return None+def _follower(self, after: str) -> str:"""recover: what a `<tool_call>` followed (after whitespace) by `after` opens: "call" (`<function=`), "param"(a declared tool's name written as `<parameter=NAME>`), "json" (`{`), "wait" (not decided yet) or "text"."""@@ -1327,7 +1363,24 @@ class OutputParser:# it is prose that names the format ("I'll use a <tool_call> block") - content, not a malformed call# that ends the request. Until its follower has arrived it is held, like a partial tag.after = self.buf[i + len(CALL_START):].lstrip()+ if self.schemas and not after.startswith(FUNC_START):+ loose = self._loose_open(after)+ if loose == "wait": # may still become one: held like a partial tag+ j = i+ while j > 0 and self.buf[j - 1] == "\n":+ j -= 1+ if j > 0:+ out.append(Event("content", self._track(self.buf[:j])))+ self.buf = self.buf[j:]+ return out+ if loose: # #3441: `<=NAME>` etc. is `<function=NAME>`+ at = len(self.buf) - len(after)+ self.buf = self.buf[:at] + FUNC_START + loose[0] + ">" + after[loose[1]:]+ continuekind = self._follower(after) if self.recover else None+ if (after.startswith("<") and kind is None and not FUNC_START.startswith(after)+ and not after.startswith(FUNC_START)):+ self.unread += 1if kind in ("param", "json"):if i and self.buf[:i].strip():out.append(Event("content", self._track(self.buf[:i].rstrip("\n"))))@@ -1352,6 +1405,22 @@ class OutputParser:out.append(Event("content", self._track(self.buf[:j])))self.buf = self.buf[j:]return out+ if self.schemas or self.declared_only:+ # #3075: a call counts only for a declared tool; the name must be complete to tell+ gt = after.find(">")+ nl = after.find("\n")+ if gt < 0 and nl < 0:+ j = i+ while j > 0 and self.buf[j - 1] == "\n":+ j -= 1+ if j > 0:+ out.append(Event("content", self._track(self.buf[:j])))+ self.buf = self.buf[j:]+ return out+ if gt >= 0 and (nl < 0 or gt < nl) and not self._callable(after[len(FUNC_START):gt].strip()):+ out.append(Event("content", self._track(self.buf[:i + len(CALL_START)])))+ self.buf = self.buf[i + len(CALL_START):]+ continueif i and self.buf[:i].strip():out.append(Event("content", self._track(self.buf[:i].rstrip("\n"))))self.buf = self.buf[i + len(CALL_START):]diff --git a/serve/server.py b/serve/server.pyindex 969651106..31409aecc 100644--- a/serve/server.py+++ b/serve/server.py@@ -4763,18 +4763,20 @@ class Service:f"{el:.0f} s", flush=True)return now- def run(self, ids, thinking, tools, max_new, sampling, cancel, force=None) -> Iterator[tuple[str, object]]:+ def run(self, ids, thinking, tools, max_new, sampling, cancel, force=None,+ declared_only=False) -> Iterator[tuple[str, object]]:"""The request's events (see _run); with "prevent_sleep" in the config, the PC stays awake while it runs or waits."""ka = self.keep_awakeif ka is not None:ka.acquire()try:- yield from self._run(ids, thinking, tools, max_new, sampling, cancel, force)+ yield from self._run(ids, thinking, tools, max_new, sampling, cancel, force, declared_only)finally:if ka is not None:ka.release()- def _run(self, ids, thinking, tools, max_new, sampling, cancel, force=None) -> Iterator[tuple[str, object]]:+ def _run(self, ids, thinking, tools, max_new, sampling, cancel, force=None,+ declared_only=False) -> Iterator[tuple[str, object]]:"""Yields ("event", Event) as text arrives, then ("done", {"finish": .., "completion_tokens": ..}).`force` (forced_call): the opening of the call the reply must make - see prepare()."""# the model card's sampling for this thinking state, then the config's block, then the Chat settings shared with@@ -4802,7 +4804,8 @@ class Service:print(f"[strata] thinking capped at {max_new - reserve} of max_tokens {max_new} to leave room for "f"the answer (budget {budget})", flush=True)budget = max(1, max_new - reserve)- parser = OutputParser(thinking=thinking, tools=tools, stream_tools=True, recover=self.tool_call_recovery)+ parser = OutputParser(thinking=thinking, tools=tools, stream_tools=True, recover=self.tool_call_recovery,+ declared_only=declared_only)detok, n, finish = Detokenizer(self.tok), 0, "length"run_tok, run_len, repeated = None, 0, False # #606: the current run of one repeated tokenthinking_n = 0 # tokens written while thinking (Responses' reasoning_tokens)@@ -5284,6 +5287,10 @@ class Service:self.totals["tool_calls_from_reasoning"] = self.totals.get("tool_calls_from_reasoning", 0) + parser.rescuedprint(f"[strata] tool calls inside the thinking: {parser.rescued} read as calls, {parser.refused} kept as "f"reasoning (quoted, or the turn did not end on a stop)", flush=True)+ if parser.unread:+ print(f"[strata] {parser.unread} <tool_call> block(s) in the answer did not read as a call of a declared "+ "tool and went back as text (a name the request did not declare, or a form the template does "+ "not use)", flush=True)if stops is not None and stops.hit is None and stops.held:yield "event", Event("content", stops.flush()) # the held tail was not a stop string after alldone = {"finish": finish, "completion_tokens": n, "reused": (timings or {}).get("cache_n", 0),@@ -5412,7 +5419,7 @@ def run_with_mcp(svc: Service, hub, messages, tools, kw, ids, thinking, max_new,messages = list(messages)while True:text, reasoning, calls, own_calls = [], [], [], 0- for kind, x in svc.run(ids, thinking, tools, max_new, sampling, cancel):+ for kind, x in svc.run(ids, thinking, tools, max_new, sampling, cancel, declared_only=True):if kind == "done":done = xcontinue@@ -5533,7 +5540,7 @@ def openai_chunks(svc: Service, req: dict, ids, thinking, tools, max_new, cancelcalls = 0streamed = {} # tool call id -> index, for calls sent piece by piecefinished = set() # ... and the ones whose final tool_call came (#211)- for kind, x in run if run is not None else svc.run(ids, thinking, tools, max_new, req, cancel, force=force):+ for kind, x in run if run is not None else svc.run(ids, thinking, tools, max_new, req, cancel, force=force, declared_only=True):if kind == "ping":# a PP line: the prompt is being read and there is nothing to say yet. It used to be only the SSE comment;# now it carries the progress when the client asked, and stays the comment when there is nothing to report.@@ -5659,7 +5666,7 @@ def anthropic_events(svc: Service, req: dict, ids, thinking, tools, max_new, canreturn ("content_block_stop", {"type": "content_block_stop", "index": index})streamed, finished = set(), set() # calls sent piece by piece; those whose final tool_call came (#211)- for kind, x in svc.run(ids, thinking, tools, max_new, req, cancel, force=force):+ for kind, x in svc.run(ids, thinking, tools, max_new, req, cancel, force=force, declared_only=True):if kind == "ping":yield Nonecontinuediff --git a/serve/test_mcp.py b/serve/test_mcp.pyindex c80c5d015..2c414ef04 100644--- a/serve/test_mcp.py+++ b/serve/test_mcp.py@@ -326,7 +326,10 @@ class ToolLoop(unittest.TestCase):self.assertNotIn("fake__echo", self.engine.prompt_text(0))cs = self.chunks(text)self.assertFalse([c for c in cs if "strata_mcp" in c])- self.assertEqual(cs[-1]["choices"][0]["finish_reason"], "tool_calls") # returned to the client as always+ # #3075: a request that declared no tools gets no tool call back; the call is the text it was written as+ self.assertEqual(cs[-1]["choices"][0]["finish_reason"], "stop")+ self.assertFalse([tc for c in cs for tc in c["choices"][0]["delta"].get("tool_calls") or []])+ self.assertIn("<function=fake__echo>", "".join(c["choices"][0]["delta"].get("content") or "" for c in cs))self.assertEqual(len(self.engine.prompts), 1)self.assertNotIn('"call"', Path(self.log.name).read_text()) # nothing ranfor req in ({"strata_mcp": "yes"}, {"strata_mcp": 1}): # only a real true opts indiff --git a/serve/test_server.py b/serve/test_server.pyindex 4249585d4..233f9e36e 100644--- a/serve/test_server.py+++ b/serve/test_server.py@@ -4411,6 +4411,41 @@ class ForcedToolChoice(unittest.TestCase):finish = c["choices"][0]["finish_reason"] or finishreturn finish, [(names[i], json.loads(args[i])) for i in sorted(names)], reasoning+ def says(self, answer):+ """The engine answers `answer` after an empty thought."""+ def generate(ids, max_new, sampling, cancel, embeddings=None):+ self.engine.prompts.append(list(ids))+ yield from self.tok.encode("</think>\n\n" + answer) + self.tok.encode("<|im_end|>", parse_special=True)+ self.engine.generate = generate++ def test_a_call_the_request_never_offered_stays_text(self):+ """#3075: the model goes on writing the format it was shown; a request that offered no such tool gets text."""+ from serve.frontend import CALL_END, CALL_START+ answer = ("I will check that.\n" + CALL_START + "\n<function=lookup>\n<parameter=q>\n2+2\n</parameter>\n"+ "</function>\n" + CALL_END + "\nHere is the report.")+ self.says(answer)+ for label, extra in {"nothing declared": {"tools": []}, "tool_choice none": {"tool_choice": "none"},+ "other tools": {}}.items():+ for stream in (False, True):+ with self.subTest(case=label, stream=stream):+ code, b = self.openai(stream=stream, **extra)+ finish, calls, _ = self.call_of(code, b, stream)+ self.assertEqual((finish, calls), ("stop", []), b)+ if not stream:+ content = b["choices"][0]["message"]["content"] or ""+ self.assertIn("Here is the report.", content)+ self.assertIn("<function=lookup>", content)++ def test_a_call_with_a_dropped_function_keyword_is_still_the_call(self):+ """#3441: `<=search>` for `<function=search>`, as a model under a large tool surface writes it."""+ from serve.frontend import CALL_END, CALL_START+ self.says(CALL_START + "\n<=search>\n<parameter=q>\n2+2\n</parameter>\n</function>\n" + CALL_END)+ for stream in (False, True):+ with self.subTest(stream=stream):+ code, b = self.openai(stream=stream)+ finish, calls, _ = self.call_of(code, b, stream)+ self.assertEqual((finish, calls), ("tool_calls", [("search", {"q": "2+2"})]), b)+def test_required_and_named_after_the_thinking(self):for choice in ("required", self.NAMED):for stream in (False, True):diff --git a/serve/test_tool_call_envelope.py b/serve/test_tool_call_envelope.pynew file mode 100644index 000000000..75ec2ad2b--- /dev/null+++ b/serve/test_tool_call_envelope.py@@ -0,0 +1,121 @@+"""#3441 (a call whose envelope drifted by a token) and #3075 (a call for a tool the request never declared)."""+from __future__ import annotations++import sys+import unittest+from pathlib import Path++sys.path.insert(0, str(Path(__file__).resolve().parent))+import frontend # noqa: E402++TOOLS = [{"name": "functions.exec", "parameters": {"properties": {"input": {"type": "string"}}}},+ {"name": "shell", "parameters": {"properties": {"cmd": {"type": "string"}}}},+ {"name": "ns.web_search", "parameters": {"properties": {"q": {"type": "string"}}}}]+++def body(opener, param="input", value="1+1", name=None):+ return f"<tool_call>\n{opener}\n<parameter={param}>\n{value}\n</parameter>\n</function>\n</tool_call>"+++def run(text, width=0, **kw):+ """-> (events, parser); `width` 0 feeds the text whole, else that many characters at a time."""+ p = frontend.OutputParser(thinking=False, stream_tools=True, **kw)+ evs = []+ if width:+ for k in range(0, len(text), width):+ evs += p.feed(text[k:k + width])+ else:+ evs += p.feed(text)+ evs += p.finish()+ return evs, p+++def calls(evs):+ return [(e.call.name, e.call.arguments) for e in evs if e.kind == "tool_call"]+++def text(evs):+ return "".join(e.text or "" for e in evs if e.kind == "content")+++class DriftedEnvelope(unittest.TestCase):+ def test_a_dropped_function_keyword_is_still_the_call(self):+ for opener in ("<=functions.exec>", "<function:functions.exec>", "<function =functions.exec>",+ "<functions.exec>", "<= functions.exec >"):+ for width in (0, 1, 2, 3, 7):+ with self.subTest(opener=opener, width=width):+ evs, p = run(body(opener), width, tools=TOOLS)+ self.assertEqual(calls(evs), [("functions.exec", {"input": "1+1"})])+ self.assertEqual(text(evs), "")+ self.assertEqual(p.unread, 0)++ def test_the_template_form_is_unchanged(self):+ evs, p = run(body("<function=functions.exec>"), 0, tools=TOOLS)+ self.assertEqual(calls(evs), [("functions.exec", {"input": "1+1"})])++ def test_an_undeclared_name_in_a_drifted_opener_stays_text(self):+ for width in (0, 1, 4):+ with self.subTest(width=width):+ raw = body("<=missing>")+ evs, p = run(raw, width, tools=TOOLS)+ self.assertEqual(calls(evs), [])+ self.assertEqual(text(evs), raw)+ self.assertEqual(p.unread, 1)+ self.assertFalse([e for e in evs if e.kind == "tool_start"])++ def test_prose_about_the_tag_is_not_a_drift(self):+ for raw in ("Use a <tool_call> block for that.", "<tool_call> then the name"):+ evs, p = run(raw, 3, tools=TOOLS)+ self.assertEqual(calls(evs), [])+ self.assertEqual(text(evs), raw)+ self.assertEqual(p.unread, 0)++ def test_streamed_and_whole_agree(self):+ raw = "Let me run it.\n" + body("<=shell>", "cmd", "ls") + "\nDone."+ whole, _ = run(raw, 0, tools=TOOLS)+ for width in (1, 2, 5, 11):+ streamed, _ = run(raw, width, tools=TOOLS)+ self.assertEqual(calls(streamed), calls(whole))+ self.assertEqual(text(streamed), text(whole))+ self.assertEqual(calls(whole), [("shell", {"cmd": "ls"})])+ self.assertEqual(text(whole), "Let me run it.Done.")+++class OnlyDeclaredTools(unittest.TestCase):+ def test_a_call_for_another_tool_stays_text(self):+ raw = body("<function=get_indicators>", "symbol", "600076")+ for width in (0, 1, 3, 9):+ with self.subTest(width=width):+ evs, p = run("I will check.\n" + raw, width, tools=TOOLS)+ self.assertEqual(calls(evs), [])+ self.assertFalse([e for e in evs if e.kind in ("tool_start", "tool_args")])+ self.assertEqual(text(evs), "I will check.\n" + raw)++ def test_a_declared_call_after_an_undeclared_one_is_still_a_call(self):+ raw = body("<function=get_indicators>") + "\n" + body("<function=shell>", "cmd", "ls")+ evs, p = run(raw, 5, tools=TOOLS)+ self.assertEqual(calls(evs), [("shell", {"cmd": "ls"})])+ self.assertIn("get_indicators", text(evs))++ def test_the_responses_spelling_of_a_namespace_tool_counts(self):+ evs, p = run(body("<function=ns__web_search>", "q", "x"), 4, tools=TOOLS)+ self.assertEqual(calls(evs), [("ns__web_search", {"q": "x"})])++ def test_no_tools_declared(self):+ raw = body("<function=search>", "q", "2+2")+ evs, p = run(raw, 3, tools=None, declared_only=True)+ self.assertEqual((calls(evs), text(evs)), ([], raw))+ evs, p = run(raw, 3, tools=[], declared_only=True)+ self.assertEqual((calls(evs), text(evs)), ([], raw))+ # without the flag (the Responses route: a Codex compaction request keeps its tools in the prompt only)+ evs, p = run(raw, 3, tools=None)+ self.assertEqual(calls(evs), [("search", {"q": "2+2"})])++ def test_the_reasoning_state_is_as_before(self):+ p = frontend.OutputParser(thinking=True, tools=TOOLS, stream_tools=True)+ evs = p.feed("think " + body("<function=shell>", "cmd", "ls") + "</think>\n\nok") + p.finish()+ self.assertEqual(calls(evs), [])+++if __name__ == "__main__":+ unittest.main()