Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Fix #3441, #3075: a drifted tool-call opener is read as the call; a call for a tool the request did not declare stays text

Niko122170f424072824
diff --git a/docs/DETAILS.md b/docs/DETAILS.md
index 1432c709d..dbcf4e43f 100644
--- a/docs/DETAILS.md
+++ b/docs/DETAILS.md
@@ -991,6 +991,12 @@ print(r.choices[0].message.content)
long prompt the stream sends keep-alives, so agents do not time out; the server window prints progress every
15 s, and `GET /status` says what it is doing (`reading the prompt`, `answering`, tokens so far). Closing the
connection or pressing stop in your app really stops the model, so the next request starts at once.
+- **Which tool calls come back as calls.** A call counts only when it names a tool the request declared. A call for
+ another name, or any call in a Chat Completions or Anthropic request that declared no tools, is returned as the
+ text it was written as, so a client never runs a tool it did not offer (the Responses route keeps its own rule
+ for Codex's compaction requests). A call whose envelope drifted by a token, `<=NAME>`, `<function:NAME>` or `<NAME>`
+ instead of `<function=NAME>`, is read as the call when NAME is a declared tool. A `<tool_call>` block that still
+ does not read as a call is counted in the server window.
- **Tool calls in other forms (opt-in).** `"tool_call_recovery": true` in `strata-<model>.json` reads a call of a
tool the request declared also when the model writes it as `<parameter=NAME>` instead of `<function=NAME>`, as JSON
(`{"name": ..., "arguments": ...}`) inside `<tool_call>`, as a `<function=NAME>` block at the start of a line
diff --git a/serve/frontend.py b/serve/frontend.py
index 3fda9c56f..4cf0b4fab 100644
--- a/serve/frontend.py
+++ b/serve/frontend.py
@@ -682,6 +682,10 @@ CALL_END = "</tool_call>"
PARAM_END = "</parameter>"
FUNC_START = "<function="
FUNC_END = "</function>"
+# #3441: the opener a model writes when it drifts by a token inside the envelope the template documents:
+# `<=NAME>` (the word `function` dropped), `<function:NAME>`, `<function =NAME>`, or the bare `<NAME>`
+_LOOSE_OPEN = re.compile(r"<\s*(?:function\s*[=:]\s*|=\s*)?([A-Za-z0-9_.\-]+)\s*>")
+_LOOSE_PART = re.compile(r"<\s*(?:function[ \t]*[=:]?\s*|=\s*)?[A-Za-z0-9_.\-]*\s*$")
def param_end(text: str, final: bool = False) -> int:
@@ -880,11 +884,19 @@ class OutputParser:
until it is complete, so clients never see `<tool_` or `</thi`."""
def __init__(self, thinking: bool = True, tools: list[dict] | None = None, stream_tools: bool = False,
- recover: bool = False):
+ recover: bool = False, declared_only: bool = False):
self.state = "reasoning" if thinking else "content"
self.buf = ""
self.lead = False
self.schemas = {t.get("name"): t for t in tools or []}
+ # #804/#3075: in the answer too, a call counts only for a tool the request declared. With tools declared a
+ # call for another name stays text (the Responses spelling `namespace__tool` of a `namespace.tool` counts as
+ # that tool); `declared_only` extends the rule to a request that declared none (Chat Completions, Anthropic:
+ # the client has no tool to run). The Responses route leaves it off: a Codex compaction request keeps its
+ # conversation's tools in the prompt only, and its calls come back as before.
+ self.declared_only = declared_only
+ self._alias = {k.replace(".", "__", 1) for k in self.schemas if isinstance(k, str) and "." in k}
+ self.unread = 0 # #3441: `<tool_call>` blocks that opened like a call and were returned as text
# stream_tools: a tool call is also reported while it is being written - "tool_start" (its name and id) as
# soon as the name is known, then "tool_args" pieces of its JSON arguments (string parameters character by
# character; other types whole, once complete) - before the final "tool_call". Without it, a client sees
@@ -913,6 +925,30 @@ class OutputParser:
self.fence, self.line, self.ticks = snap
return ok
+ def _callable(self, name: str) -> bool:
+ """#3075: may a call for `name` be a call? A tool the request declared (or its Responses spelling); with
+ none declared, only where `declared_only` is off."""
+ if not self.schemas:
+ return not self.declared_only
+ return name in self.schemas or name in self._alias
+
+ def _loose_open(self, after: str):
+ """#3441: `after` (text after a `<tool_call>`, whitespace dropped) opens a call in a drifted form that names a
+ declared tool -> (name, length of the opener); "wait" while it may still become one; None otherwise."""
+ m = _LOOSE_OPEN.match(after)
+ if m:
+ name = m.group(1)
+ return (name, m.end()) if self._callable(name) else None
+ if ">" in after or "\n" in after or not after.startswith("<"):
+ return None
+ if _LOOSE_PART.match(after):
+ bare = after[1:].strip().lstrip("=").strip()
+ if after[1:].lstrip().startswith(("=", "function")):
+ return "wait"
+ if any(k.startswith(bare) for k in list(self.schemas) + list(self._alias) if isinstance(k, str)):
+ return "wait"
+ return None
+
def _follower(self, after: str) -> str:
"""recover: what a `<tool_call>` followed (after whitespace) by `after` opens: "call" (`<function=`), "param"
(a declared tool's name written as `<parameter=NAME>`), "json" (`{`), "wait" (not decided yet) or "text"."""
@@ -1327,7 +1363,24 @@ class OutputParser:
# it is prose that names the format ("I'll use a <tool_call> block") - content, not a malformed call
# that ends the request. Until its follower has arrived it is held, like a partial tag.
after = self.buf[i + len(CALL_START):].lstrip()
+ if self.schemas and not after.startswith(FUNC_START):
+ loose = self._loose_open(after)
+ if loose == "wait": # may still become one: held like a partial tag
+ j = i
+ while j > 0 and self.buf[j - 1] == "\n":
+ j -= 1
+ if j > 0:
+ out.append(Event("content", self._track(self.buf[:j])))
+ self.buf = self.buf[j:]
+ return out
+ if loose: # #3441: `<=NAME>` etc. is `<function=NAME>`
+ at = len(self.buf) - len(after)
+ self.buf = self.buf[:at] + FUNC_START + loose[0] + ">" + after[loose[1]:]
+ continue
kind = self._follower(after) if self.recover else None
+ if (after.startswith("<") and kind is None and not FUNC_START.startswith(after)
+ and not after.startswith(FUNC_START)):
+ self.unread += 1
if kind in ("param", "json"):
if i and self.buf[:i].strip():
out.append(Event("content", self._track(self.buf[:i].rstrip("\n"))))
@@ -1352,6 +1405,22 @@ class OutputParser:
out.append(Event("content", self._track(self.buf[:j])))
self.buf = self.buf[j:]
return out
+ if self.schemas or self.declared_only:
+ # #3075: a call counts only for a declared tool; the name must be complete to tell
+ gt = after.find(">")
+ nl = after.find("\n")
+ if gt < 0 and nl < 0:
+ j = i
+ while j > 0 and self.buf[j - 1] == "\n":
+ j -= 1
+ if j > 0:
+ out.append(Event("content", self._track(self.buf[:j])))
+ self.buf = self.buf[j:]
+ return out
+ if gt >= 0 and (nl < 0 or gt < nl) and not self._callable(after[len(FUNC_START):gt].strip()):
+ out.append(Event("content", self._track(self.buf[:i + len(CALL_START)])))
+ self.buf = self.buf[i + len(CALL_START):]
+ continue
if i and self.buf[:i].strip():
out.append(Event("content", self._track(self.buf[:i].rstrip("\n"))))
self.buf = self.buf[i + len(CALL_START):]
diff --git a/serve/server.py b/serve/server.py
index 969651106..31409aecc 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -4763,18 +4763,20 @@ class Service:
f"{el:.0f} s", flush=True)
return now
- def run(self, ids, thinking, tools, max_new, sampling, cancel, force=None) -> Iterator[tuple[str, object]]:
+ def run(self, ids, thinking, tools, max_new, sampling, cancel, force=None,
+ declared_only=False) -> Iterator[tuple[str, object]]:
"""The request's events (see _run); with "prevent_sleep" in the config, the PC stays awake while it runs or waits."""
ka = self.keep_awake
if ka is not None:
ka.acquire()
try:
- yield from self._run(ids, thinking, tools, max_new, sampling, cancel, force)
+ yield from self._run(ids, thinking, tools, max_new, sampling, cancel, force, declared_only)
finally:
if ka is not None:
ka.release()
- def _run(self, ids, thinking, tools, max_new, sampling, cancel, force=None) -> Iterator[tuple[str, object]]:
+ def _run(self, ids, thinking, tools, max_new, sampling, cancel, force=None,
+ declared_only=False) -> Iterator[tuple[str, object]]:
"""Yields ("event", Event) as text arrives, then ("done", {"finish": .., "completion_tokens": ..}).
`force` (forced_call): the opening of the call the reply must make - see prepare()."""
# the model card's sampling for this thinking state, then the config's block, then the Chat settings shared with
@@ -4802,7 +4804,8 @@ class Service:
print(f"[strata] thinking capped at {max_new - reserve} of max_tokens {max_new} to leave room for "
f"the answer (budget {budget})", flush=True)
budget = max(1, max_new - reserve)
- parser = OutputParser(thinking=thinking, tools=tools, stream_tools=True, recover=self.tool_call_recovery)
+ parser = OutputParser(thinking=thinking, tools=tools, stream_tools=True, recover=self.tool_call_recovery,
+ declared_only=declared_only)
detok, n, finish = Detokenizer(self.tok), 0, "length"
run_tok, run_len, repeated = None, 0, False # #606: the current run of one repeated token
thinking_n = 0 # tokens written while thinking (Responses' reasoning_tokens)
@@ -5284,6 +5287,10 @@ class Service:
self.totals["tool_calls_from_reasoning"] = self.totals.get("tool_calls_from_reasoning", 0) + parser.rescued
print(f"[strata] tool calls inside the thinking: {parser.rescued} read as calls, {parser.refused} kept as "
f"reasoning (quoted, or the turn did not end on a stop)", flush=True)
+ if parser.unread:
+ print(f"[strata] {parser.unread} <tool_call> block(s) in the answer did not read as a call of a declared "
+ "tool and went back as text (a name the request did not declare, or a form the template does "
+ "not use)", flush=True)
if stops is not None and stops.hit is None and stops.held:
yield "event", Event("content", stops.flush()) # the held tail was not a stop string after all
done = {"finish": finish, "completion_tokens": n, "reused": (timings or {}).get("cache_n", 0),
@@ -5412,7 +5419,7 @@ def run_with_mcp(svc: Service, hub, messages, tools, kw, ids, thinking, max_new,
messages = list(messages)
while True:
text, reasoning, calls, own_calls = [], [], [], 0
- for kind, x in svc.run(ids, thinking, tools, max_new, sampling, cancel):
+ for kind, x in svc.run(ids, thinking, tools, max_new, sampling, cancel, declared_only=True):
if kind == "done":
done = x
continue
@@ -5533,7 +5540,7 @@ def openai_chunks(svc: Service, req: dict, ids, thinking, tools, max_new, cancel
calls = 0
streamed = {} # tool call id -> index, for calls sent piece by piece
finished = set() # ... and the ones whose final tool_call came (#211)
- for kind, x in run if run is not None else svc.run(ids, thinking, tools, max_new, req, cancel, force=force):
+ for kind, x in run if run is not None else svc.run(ids, thinking, tools, max_new, req, cancel, force=force, declared_only=True):
if kind == "ping":
# a PP line: the prompt is being read and there is nothing to say yet. It used to be only the SSE comment;
# now it carries the progress when the client asked, and stays the comment when there is nothing to report.
@@ -5659,7 +5666,7 @@ def anthropic_events(svc: Service, req: dict, ids, thinking, tools, max_new, can
return ("content_block_stop", {"type": "content_block_stop", "index": index})
streamed, finished = set(), set() # calls sent piece by piece; those whose final tool_call came (#211)
- for kind, x in svc.run(ids, thinking, tools, max_new, req, cancel, force=force):
+ for kind, x in svc.run(ids, thinking, tools, max_new, req, cancel, force=force, declared_only=True):
if kind == "ping":
yield None
continue
diff --git a/serve/test_mcp.py b/serve/test_mcp.py
index c80c5d015..2c414ef04 100644
--- a/serve/test_mcp.py
+++ b/serve/test_mcp.py
@@ -326,7 +326,10 @@ class ToolLoop(unittest.TestCase):
self.assertNotIn("fake__echo", self.engine.prompt_text(0))
cs = self.chunks(text)
self.assertFalse([c for c in cs if "strata_mcp" in c])
- self.assertEqual(cs[-1]["choices"][0]["finish_reason"], "tool_calls") # returned to the client as always
+ # #3075: a request that declared no tools gets no tool call back; the call is the text it was written as
+ self.assertEqual(cs[-1]["choices"][0]["finish_reason"], "stop")
+ self.assertFalse([tc for c in cs for tc in c["choices"][0]["delta"].get("tool_calls") or []])
+ self.assertIn("<function=fake__echo>", "".join(c["choices"][0]["delta"].get("content") or "" for c in cs))
self.assertEqual(len(self.engine.prompts), 1)
self.assertNotIn('"call"', Path(self.log.name).read_text()) # nothing ran
for req in ({"strata_mcp": "yes"}, {"strata_mcp": 1}): # only a real true opts in
diff --git a/serve/test_server.py b/serve/test_server.py
index 4249585d4..233f9e36e 100644
--- a/serve/test_server.py
+++ b/serve/test_server.py
@@ -4411,6 +4411,41 @@ class ForcedToolChoice(unittest.TestCase):
finish = c["choices"][0]["finish_reason"] or finish
return finish, [(names[i], json.loads(args[i])) for i in sorted(names)], reasoning
+ def says(self, answer):
+ """The engine answers `answer` after an empty thought."""
+ def generate(ids, max_new, sampling, cancel, embeddings=None):
+ self.engine.prompts.append(list(ids))
+ yield from self.tok.encode("</think>\n\n" + answer) + self.tok.encode("<|im_end|>", parse_special=True)
+ self.engine.generate = generate
+
+ def test_a_call_the_request_never_offered_stays_text(self):
+ """#3075: the model goes on writing the format it was shown; a request that offered no such tool gets text."""
+ from serve.frontend import CALL_END, CALL_START
+ answer = ("I will check that.\n" + CALL_START + "\n<function=lookup>\n<parameter=q>\n2+2\n</parameter>\n"
+ "</function>\n" + CALL_END + "\nHere is the report.")
+ self.says(answer)
+ for label, extra in {"nothing declared": {"tools": []}, "tool_choice none": {"tool_choice": "none"},
+ "other tools": {}}.items():
+ for stream in (False, True):
+ with self.subTest(case=label, stream=stream):
+ code, b = self.openai(stream=stream, **extra)
+ finish, calls, _ = self.call_of(code, b, stream)
+ self.assertEqual((finish, calls), ("stop", []), b)
+ if not stream:
+ content = b["choices"][0]["message"]["content"] or ""
+ self.assertIn("Here is the report.", content)
+ self.assertIn("<function=lookup>", content)
+
+ def test_a_call_with_a_dropped_function_keyword_is_still_the_call(self):
+ """#3441: `<=search>` for `<function=search>`, as a model under a large tool surface writes it."""
+ from serve.frontend import CALL_END, CALL_START
+ self.says(CALL_START + "\n<=search>\n<parameter=q>\n2+2\n</parameter>\n</function>\n" + CALL_END)
+ for stream in (False, True):
+ with self.subTest(stream=stream):
+ code, b = self.openai(stream=stream)
+ finish, calls, _ = self.call_of(code, b, stream)
+ self.assertEqual((finish, calls), ("tool_calls", [("search", {"q": "2+2"})]), b)
+
def test_required_and_named_after_the_thinking(self):
for choice in ("required", self.NAMED):
for stream in (False, True):
diff --git a/serve/test_tool_call_envelope.py b/serve/test_tool_call_envelope.py
new file mode 100644
index 000000000..75ec2ad2b
--- /dev/null
+++ b/serve/test_tool_call_envelope.py
@@ -0,0 +1,121 @@
+"""#3441 (a call whose envelope drifted by a token) and #3075 (a call for a tool the request never declared)."""
+from __future__ import annotations
+
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import frontend # noqa: E402
+
+TOOLS = [{"name": "functions.exec", "parameters": {"properties": {"input": {"type": "string"}}}},
+ {"name": "shell", "parameters": {"properties": {"cmd": {"type": "string"}}}},
+ {"name": "ns.web_search", "parameters": {"properties": {"q": {"type": "string"}}}}]
+
+
+def body(opener, param="input", value="1+1", name=None):
+ return f"<tool_call>\n{opener}\n<parameter={param}>\n{value}\n</parameter>\n</function>\n</tool_call>"
+
+
+def run(text, width=0, **kw):
+ """-> (events, parser); `width` 0 feeds the text whole, else that many characters at a time."""
+ p = frontend.OutputParser(thinking=False, stream_tools=True, **kw)
+ evs = []
+ if width:
+ for k in range(0, len(text), width):
+ evs += p.feed(text[k:k + width])
+ else:
+ evs += p.feed(text)
+ evs += p.finish()
+ return evs, p
+
+
+def calls(evs):
+ return [(e.call.name, e.call.arguments) for e in evs if e.kind == "tool_call"]
+
+
+def text(evs):
+ return "".join(e.text or "" for e in evs if e.kind == "content")
+
+
+class DriftedEnvelope(unittest.TestCase):
+ def test_a_dropped_function_keyword_is_still_the_call(self):
+ for opener in ("<=functions.exec>", "<function:functions.exec>", "<function =functions.exec>",
+ "<functions.exec>", "<= functions.exec >"):
+ for width in (0, 1, 2, 3, 7):
+ with self.subTest(opener=opener, width=width):
+ evs, p = run(body(opener), width, tools=TOOLS)
+ self.assertEqual(calls(evs), [("functions.exec", {"input": "1+1"})])
+ self.assertEqual(text(evs), "")
+ self.assertEqual(p.unread, 0)
+
+ def test_the_template_form_is_unchanged(self):
+ evs, p = run(body("<function=functions.exec>"), 0, tools=TOOLS)
+ self.assertEqual(calls(evs), [("functions.exec", {"input": "1+1"})])
+
+ def test_an_undeclared_name_in_a_drifted_opener_stays_text(self):
+ for width in (0, 1, 4):
+ with self.subTest(width=width):
+ raw = body("<=missing>")
+ evs, p = run(raw, width, tools=TOOLS)
+ self.assertEqual(calls(evs), [])
+ self.assertEqual(text(evs), raw)
+ self.assertEqual(p.unread, 1)
+ self.assertFalse([e for e in evs if e.kind == "tool_start"])
+
+ def test_prose_about_the_tag_is_not_a_drift(self):
+ for raw in ("Use a <tool_call> block for that.", "<tool_call> then the name"):
+ evs, p = run(raw, 3, tools=TOOLS)
+ self.assertEqual(calls(evs), [])
+ self.assertEqual(text(evs), raw)
+ self.assertEqual(p.unread, 0)
+
+ def test_streamed_and_whole_agree(self):
+ raw = "Let me run it.\n" + body("<=shell>", "cmd", "ls") + "\nDone."
+ whole, _ = run(raw, 0, tools=TOOLS)
+ for width in (1, 2, 5, 11):
+ streamed, _ = run(raw, width, tools=TOOLS)
+ self.assertEqual(calls(streamed), calls(whole))
+ self.assertEqual(text(streamed), text(whole))
+ self.assertEqual(calls(whole), [("shell", {"cmd": "ls"})])
+ self.assertEqual(text(whole), "Let me run it.Done.")
+
+
+class OnlyDeclaredTools(unittest.TestCase):
+ def test_a_call_for_another_tool_stays_text(self):
+ raw = body("<function=get_indicators>", "symbol", "600076")
+ for width in (0, 1, 3, 9):
+ with self.subTest(width=width):
+ evs, p = run("I will check.\n" + raw, width, tools=TOOLS)
+ self.assertEqual(calls(evs), [])
+ self.assertFalse([e for e in evs if e.kind in ("tool_start", "tool_args")])
+ self.assertEqual(text(evs), "I will check.\n" + raw)
+
+ def test_a_declared_call_after_an_undeclared_one_is_still_a_call(self):
+ raw = body("<function=get_indicators>") + "\n" + body("<function=shell>", "cmd", "ls")
+ evs, p = run(raw, 5, tools=TOOLS)
+ self.assertEqual(calls(evs), [("shell", {"cmd": "ls"})])
+ self.assertIn("get_indicators", text(evs))
+
+ def test_the_responses_spelling_of_a_namespace_tool_counts(self):
+ evs, p = run(body("<function=ns__web_search>", "q", "x"), 4, tools=TOOLS)
+ self.assertEqual(calls(evs), [("ns__web_search", {"q": "x"})])
+
+ def test_no_tools_declared(self):
+ raw = body("<function=search>", "q", "2+2")
+ evs, p = run(raw, 3, tools=None, declared_only=True)
+ self.assertEqual((calls(evs), text(evs)), ([], raw))
+ evs, p = run(raw, 3, tools=[], declared_only=True)
+ self.assertEqual((calls(evs), text(evs)), ([], raw))
+ # without the flag (the Responses route: a Codex compaction request keeps its tools in the prompt only)
+ evs, p = run(raw, 3, tools=None)
+ self.assertEqual(calls(evs), [("search", {"q": "2+2"})])
+
+ def test_the_reasoning_state_is_as_before(self):
+ p = frontend.OutputParser(thinking=True, tools=TOOLS, stream_tools=True)
+ evs = p.feed("think " + body("<function=shell>", "cmd", "ls") + "</think>\n\nok") + p.finish()
+ self.assertEqual(calls(evs), [])
+
+
+if __name__ == "__main__":
+ unittest.main()