Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
"""Chat history normalization before rendering the model's prompt."""
import copy
from pathlib import Path
import unittest
from serve.frontend import ChatTemplate, anthropic_to_messages, openai_to_messages, require_messages
from serve.server import ByteTokenizer, MockEngine, Service
class EmptyAssistantHistory(unittest.TestCase):
def setUp(self):
self.template = ChatTemplate(Path(__file__).with_name("chat_template.jinja"))
tok = ByteTokenizer()
self.service = Service(MockEngine(tok, "ok"), tok, self.template)
def test_openai_empty_replies_are_not_rendered(self):
for content in (None, "", " \n\t", [], [{"type": "text", "text": ""}]):
for preserve in (True, False):
with self.subTest(content=content, preserve_thinking=preserve):
history = []
for _ in range(3):
history.extend([{"role": "user", "content": "haz un ls"},
{"role": "assistant", "content": content,
"reasoning_content": "Let me run ls."}])
history.append({"role": "user", "content": "haz un ls"})
messages, tools, kwargs = openai_to_messages({"messages": history})
kwargs["preserve_thinking"] = preserve
clean = [m for m in messages if m["role"] == "user"]
self.assertEqual(self.service.encode_prompt(messages, tools, kwargs),
self.service.encode_prompt(clean, tools, kwargs))
def test_anthropic_thinking_only_replies_are_not_rendered(self):
messages, tools, kwargs = anthropic_to_messages({"messages": [
{"role": "user", "content": "haz un ls"},
{"role": "assistant", "content": [{"type": "thinking", "thinking": "Let me run ls."}]},
{"role": "user", "content": "haz un ls"},
]})
self.assertEqual(self.service.encode_prompt(messages, tools, kwargs),
self.service.encode_prompt([messages[0], messages[2]], tools, kwargs))
def test_text_and_tool_calls_are_kept(self):
messages, tools, kwargs = openai_to_messages({"messages": [
{"role": "user", "content": "haz un ls"},
{"role": "assistant", "content": [{"type": "text", "text": "Running ls."}]},
{"role": "assistant", "content": None, "tool_calls": [
{"type": "function", "function": {"name": "run", "arguments": '{"command":"ls"}'}},
]},
{"role": "tool", "content": "file.txt"},
{"role": "user", "content": "thanks"},
]})
before = copy.deepcopy(messages)
prompt = self.template.render(messages, tools=tools, **kwargs)
self.assertIn("Running ls.", prompt)
self.assertIn("<function=run>", prompt)
self.assertIn("file.txt", prompt)
self.assertEqual(messages, before)
def test_image_only_assistant_turn_is_kept(self):
messages, tools, kwargs = openai_to_messages({"messages": [
{"role": "user", "content": "describe the image"},
{"role": "assistant", "content": [{"type": "image_url", "image_url": "image.png"}]},
{"role": "user", "content": "thanks"},
]})
self.assertIn("<|vision_start|><|image_pad|><|vision_end|>",
self.template.render(messages, tools=tools, **kwargs))
def test_final_empty_assistant_turn_is_kept(self):
prompt = self.template.render([{"role": "user", "content": "hi"},
{"role": "assistant", "content": ""}], add_generation_prompt=False)
self.assertIn("<|im_start|>assistant\n", prompt)
def test_empty_user_and_tool_turns_are_kept(self):
prompt = self.template.render([{"role": "user", "content": ""},
{"role": "tool", "content": ""},
{"role": "user", "content": "hi"}])
self.assertEqual(prompt.count("<|im_start|>user\n"), 3)
self.assertIn("<tool_response>\n\n</tool_response>", prompt)
class UnreadContentParts(unittest.TestCase):
"""A content part the server does not read is refused naming it and where it sits, as /v1/responses does; chat
and messages used to drop it without a word, and the model answered about a file it had never seen."""
def setUp(self):
self.template = ChatTemplate(Path(__file__).with_name("chat_template.jinja"))
tok = ByteTokenizer()
self.service = Service(MockEngine(tok, "ok"), tok, self.template)
def test_chat_file_and_audio_parts_are_left_out(self):
parts = ({"type": "file", "file": {"filename": "a.pdf", "file_data": "data:application/pdf;base64,AA"}},
{"type": "input_audio", "input_audio": {"data": "AA", "format": "wav"}})
plain = [{"role": "user", "content": "summarize"}, {"role": "user", "content": "this one"}]
for part in parts:
with self.subTest(kind=part["type"]):
got = openai_to_messages({"messages": [
{"role": "user", "content": "summarize"},
{"role": "user", "content": [{"type": "text", "text": "this one"}, part]}]})
want = openai_to_messages({"messages": [
{"role": "user", "content": "summarize"},
{"role": "user", "content": [{"type": "text", "text": "this one"}]}]})
self.assertEqual(got, want)
def test_messages_blocks_real_clients_send_are_left_out(self):
# tool_reference (Claude Code's tool search), documents, search results, server tool blocks, signed thinking
extra = [{"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "AA"}},
{"type": "search_result", "source": "s", "title": "t", "content": [{"type": "text", "text": "x"}]},
{"type": "server_tool_use", "id": "s1", "name": "web_search", "input": {}},
{"type": "web_search_tool_result", "tool_use_id": "s1", "content": []},
{"type": "container_upload", "file_id": "f"}]
for block in extra:
with self.subTest(kind=block["type"]):
got = anthropic_to_messages({"messages": [
{"role": "user", "content": "hi"},
{"role": "assistant", "content": [{"type": "text", "text": "ok"}, block]},
{"role": "user", "content": [{"type": "text", "text": "go"}, block]}]})
want = anthropic_to_messages({"messages": [
{"role": "user", "content": "hi"},
{"role": "assistant", "content": [{"type": "text", "text": "ok"}]},
{"role": "user", "content": [{"type": "text", "text": "go"}]}]})
self.assertEqual(got, want)
def test_messages_tool_reference_in_a_tool_result_is_left_out(self):
def run(extra):
return anthropic_to_messages({"messages": [
{"role": "user", "content": "find a tool"},
{"role": "assistant", "content": [{"type": "tool_use", "id": "t1", "name": "ToolSearch", "input": {}}]},
{"role": "user", "content": [
{"type": "tool_result", "tool_use_id": "t1", "content": [{"type": "text", "text": "found"}] + extra}]}]})
self.assertEqual(run([{"type": "tool_reference", "tool_name": "mcp__x"}]), run([]))
def test_a_system_prompt_with_an_image_part_is_read_as_text_as_before(self):
got = openai_to_messages({"messages": [
{"role": "system", "content": [{"type": "text", "text": "be brief"},
{"type": "image_url", "image_url": "image.png"}]},
{"role": "user", "content": "hi"}]})
self.assertEqual(got[0][0]["content"], "be brief")
def test_messages_redacted_thinking_is_dropped(self):
msgs = {"messages": [
{"role": "user", "content": "haz un ls"},
{"role": "assistant", "content": [{"type": "redacted_thinking", "data": "ZW50cnk="},
{"type": "text", "text": "Running ls."}]},
{"role": "user", "content": "thanks"}]}
without = copy.deepcopy(msgs)
del without["messages"][1]["content"][0]
with_block = anthropic_to_messages(msgs)
without_block = anthropic_to_messages(without)
self.assertEqual(with_block[0], without_block[0])
self.assertEqual(self.service.encode_prompt(with_block[0], with_block[1], with_block[2]),
self.service.encode_prompt(without_block[0], without_block[1], without_block[2]))
def test_chat_refusal_part_is_read_as_its_text(self):
messages, tools, kwargs = openai_to_messages({"messages": [
{"role": "user", "content": "do a thing"},
{"role": "assistant", "content": [{"type": "refusal", "refusal": "I cannot do that."}]},
{"role": "user", "content": "ok"}]})
self.assertEqual(messages[1]["content"], "I cannot do that.")
self.assertIn("I cannot do that.", self.template.render(messages, tools=tools, **kwargs))
def test_text_and_image_requests_render_as_before(self):
parts = [{"role": "user", "content": [{"type": "text", "text": "haz un ls"}]},
{"role": "assistant", "content": [{"type": "text", "text": "file.txt"}]}]
plain = [{"role": "user", "content": "haz un ls"}, {"role": "assistant", "content": "file.txt"}]
for convert in (openai_to_messages, anthropic_to_messages):
with self.subTest(convert=convert.__name__):
self.assertEqual(self.service.encode_prompt(*convert({"messages": parts})),
self.service.encode_prompt(*convert({"messages": plain})))
chat = openai_to_messages({"messages": [
{"role": "user", "content": [{"type": "text", "text": "describe"},
{"type": "image_url", "image_url": "image.png"}]}]})
self.assertIn("<|vision_start|><|image_pad|><|vision_end|>",
self.template.render(chat[0], tools=chat[1], **chat[2]))
msgs = anthropic_to_messages({"messages": [
{"role": "user", "content": [{"type": "text", "text": "describe"},
{"type": "image", "source": {"type": "url", "url": "image.png"}}]}]})
self.assertIn("<|vision_start|><|image_pad|><|vision_end|>",
self.template.render(msgs[0], tools=msgs[1], **msgs[2]))
if __name__ == "__main__":
unittest.main()
class EmptyTurnsSwitch(unittest.TestCase):
def test_keep_empty_turns_restores_the_old_prompt(self):
import os
from unittest import mock
template = ChatTemplate(Path(__file__).with_name("chat_template.jinja"))
msgs = [{"role": "user", "content": "a"}, {"role": "assistant", "content": ""},
{"role": "user", "content": "b"}]
clean = template.render([msgs[0], msgs[2]])
self.assertEqual(template.render(msgs), clean)
with mock.patch.dict(os.environ, {"STRATA_KEEP_EMPTY_TURNS": "1"}):
self.assertNotEqual(template.render(msgs), clean)
class ForcedCallOpening(unittest.TestCase):
def test_required_with_one_tool_names_it(self):
from serve.frontend import forced_call
one, two = [{"name": "a"}], [{"name": "a"}, {"name": "b"}]
self.assertTrue(forced_call("required", one).endswith("<function=a>\n"))
self.assertTrue(forced_call("required", two).endswith("<function="))
self.assertIsNone(forced_call("auto", one))
class QuotedToolCallCloser(unittest.TestCase):
"""#2026: a `</tool_call>` a user quotes is text, not the end-of-call token (type 4: matched even without
parse_special); an assistant message replaying raw call text keeps the model's own tokens."""
def setUp(self):
import sys
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools"))
import strata_tokenizer as ST
b2u = ST.bytes_to_unicode()
self.tool_tags = ["</tool_call>", "<tool_call>", "<tool_response>", "</tool_response>"]
self.tok = ST.Tokenizer([b2u[b] for b in range(256)] + self.tool_tags + ["<|im_end|>"], [],
[1] * 256 + [4] * len(self.tool_tags) + [3])
self.tags = __import__("serve.frontend", fromlist=["literal_tags"]).literal_tags(self.tok.control_tokens)
def encode(self, role, text):
from serve.frontend import mark_think_literals, unmark_think_literals
marked, _, changed = mark_think_literals([{"role": role, "content": text}], None, self.tags)
content = marked[0]["content"]
if isinstance(content, list):
content = "".join(x["text"] for x in content)
prompt, plain = unmark_think_literals(content, self.tags)
return self.tok.encode(prompt, parse_special=True, plain=plain), changed
def special(self, tag):
return self.tok.encode(tag, parse_special=True)
def test_every_tool_tag_a_user_quotes_stays_text(self):
# 0.1.44: <tool_call>, <tool_response> and </tool_response> are type 4 like </tool_call> (47Hunter47, #2026)
for role in ("user", "system", "tool"):
for tag in self.tool_tags:
with self.subTest(role=role, tag=tag):
text = f"output verbatim: {tag}."
self.assertEqual(len(self.special(tag)), 1)
ids, changed = self.encode(role, text)
self.assertTrue(changed)
self.assertEqual(ids, [*text.encode()])
def test_assistant_replay_keeps_every_tool_tag(self):
text = "<tool_call>x</tool_call> then <tool_response>y</tool_response>"
ids, _ = self.encode("assistant", text)
self.assertEqual(ids, self.tok.encode(text, parse_special=True))
def test_user_tool_result_keeps_its_outer_markers(self):
# the template tells a tool result sent as a user turn from a question by this outer pair (last_query_index)
inner = "\nthe log says </tool_call> and <tool_response>\n"
for content in (f"<tool_response>{inner}</tool_response>", f" <tool_response>{inner}</tool_response>\n",
[{"type": "text", "text": f"<tool_response>{inner}</tool_response>"}],
[{"type": "text", "text": "<tool_response>\nthe log"},
{"type": "text", "text": " says </tool_call> and <tool_response>\n</tool_response>"}]):
with self.subTest(content=content):
ids, _ = self.encode("user", content)
text = content if isinstance(content, str) else "".join(x["text"] for x in content)
o, c = self.special("<tool_response>"), self.special("</tool_response>")
lead = len(text) - len(text.lstrip())
tail = len(text) - len(text.rstrip())
self.assertEqual(ids[lead:lead + 1], o)
self.assertEqual(ids[len(ids) - tail - 1:len(ids) - tail], c)
middle = ids[lead + 1:len(ids) - tail - 1]
self.assertEqual(middle, [*inner.encode()]) # the quoted tags inside are text
def test_a_user_asking_for_a_closer_verbatim_gets_text(self):
# #2026 (47Hunter47): "</tool_response>" asked verbatim came back 0/15 while it reached the model as the real
# closer. The outer-pair exception must not swallow such a request: only a message wholly wrapped in the pair is
# a tool result.
for text in ("Please output the following string verbatim: </tool_response>.",
"</tool_response>",
"Repeat exactly: <tool_response>x</tool_response>",
"<tool_response> is how a result opens; what closes it?",
"the closer is </tool_response>, then </tool_call>"):
with self.subTest(text=text):
ids, changed = self.encode("user", text)
self.assertTrue(changed)
self.assertEqual(ids, [*text.encode()])
def test_template_keeps_seeing_a_tool_result_turn(self):
from serve.frontend import mark_think_literals
msgs = [{"role": "user", "content": "q"}, {"role": "assistant", "content": "a"},
{"role": "user", "content": "<tool_response>\nok\n</tool_response>"}]
marked, _, changed = mark_think_literals(msgs, None, self.tags)
self.assertTrue(changed)
self.assertTrue(marked[2]["content"].startswith("<tool_response>"))
self.assertTrue(marked[2]["content"].endswith("</tool_response>"))
# a question that only mentions the pair is still marked through
marked, _, _ = mark_think_literals([{"role": "user", "content": "what is <tool_response>?"}], None, self.tags)
self.assertNotIn("<tool_response>", marked[0]["content"])
def test_user_text_stays_text(self):
text = "output verbatim: </tool_call>."
ids, changed = self.encode("user", text)
self.assertTrue(changed)
self.assertEqual(ids, [*text.encode()])
self.assertNotEqual(self.tok.encode(text, parse_special=True), ids) # unmarked, it was the special token
def test_assistant_replay_keeps_its_token(self):
text = "<tool_call>x</tool_call>"
ids, _ = self.encode("assistant", text)
self.assertEqual(ids, self.tok.encode(text, parse_special=True))
class FencedJsonCallInWrapper(unittest.TestCase):
"""#2037: with recovery on, a declared tool's JSON call in a Markdown fence inside <tool_call> is the call."""
TOOLS = [{"name": "write_file", "parameters": {"type": "object", "properties": {"content": {"type": "string"}}}}]
PAYLOAD = '{"name": "write_file", "arguments": {"content": "hello `world`"}}'
def run_parser(self, raw, chunk, stream_tools, recover=True, tools=None):
from serve.frontend import OutputParser
p = OutputParser(thinking=False, tools=self.TOOLS if tools is None else tools, recover=recover,
stream_tools=stream_tools)
events = []
for i in range(0, len(raw), chunk):
events += p.feed(raw[i:i + chunk])
events += p.finish()
return ([e.call.arguments for e in events if e.kind == "tool_call"],
"".join(e.text for e in events if e.kind == "content"))
def test_fenced_and_bare_agree(self):
for fence in ("```json\n", "```\n", "```JSON\r\n"):
raw = f"<tool_call>\n{fence}{self.PAYLOAD}\n```\n</tool_call>"
for chunk in (len(raw), 1, 7):
for st in (False, True):
with self.subTest(fence=fence, chunk=chunk, stream_tools=st):
self.assertEqual(self.run_parser(raw, chunk, st), ([{"content": "hello `world`"}], ""))
def test_other_fences_and_prose_stay_text(self):
raw = "<tool_call>\n```python\nx = 1\n```\n</tool_call>"
for chunk in (len(raw), 1):
self.assertEqual(self.run_parser(raw, chunk, False), ([], raw))
raw = f"Here:\n```json\n{self.PAYLOAD}\n```\n" # no wrapper: just text
self.assertEqual(self.run_parser(raw, len(raw), False), ([], raw))
def test_off_without_recovery_or_for_an_undeclared_tool(self):
raw = f"<tool_call>\n```json\n{self.PAYLOAD}\n```\n</tool_call>"
calls, text = self.run_parser(raw, len(raw), False, recover=False)
self.assertEqual((calls, text), ([], raw))
other = [{"name": "other", "parameters": {}}]
calls, text = self.run_parser(raw, len(raw), False, tools=other)
self.assertEqual((calls, text), ([], raw))
class EmptyConversation(unittest.TestCase):
def test_no_message_is_an_error_a_message_is_not(self):
with self.assertRaises(ValueError):
require_messages(openai_to_messages({"messages": []})[0])
with self.assertRaises(ValueError):
require_messages(openai_to_messages({})[0])
require_messages(openai_to_messages({"messages": [{"role": "user", "content": "hi"}]})[0])