Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

serve: a DeepSeek-V4.1 pack that ships the V4-Flash chat template (tool_calls, no space) is rendered with the V4.1 template, so tool calls come back as real calls; STRATA_DEBUG prints the raw ids; test

Niko122170efffb5b60a
diff --git a/serve/dsml.py b/serve/dsml.py
index 16b8c2d6b..c885e9703 100644
--- a/serve/dsml.py
+++ b/serve/dsml.py
@@ -21,6 +21,7 @@ split behave for DeepSeek as they do for the other families, and there is one pl
from __future__ import annotations
import re
+from pathlib import Path
from serve.frontend import Event, OutputParser
@@ -219,6 +220,29 @@ class DsmlOutputParser(OutputParser):
return not self.buf and not self.dsml.held
+FIXTURE_TEMPLATE = Path(__file__).resolve().parent / "fixtures" / "deepseek41" / "chat_template.jinja"
+
+
+def v41_template_ok(text: str) -> bool:
+ """The V4.1 model writes its call block as `<|DSML| calls>` / `<|DSML| invoke ...` (a space after the DSML token).
+ Packs made from another GGUF ship the V4-Flash template (`<|DSML|tool_calls>`, no space): the model then sees one
+ spelling in the prompt and writes another, and only 2 of 11 tool-call tries came back as real calls (P2003 install
+ test; 0 of 4 vs 6 of 6 with this template on the same pack)."""
+ return " calls>" in text
+
+
+def pick_template(pack_template: Path | None) -> tuple[Path, bool]:
+ """(the template file to render with, True when it replaced a pack template that spells the call block otherwise)."""
+ if pack_template is not None and pack_template.exists():
+ try:
+ if v41_template_ok(pack_template.read_text(encoding="utf-8")):
+ return pack_template, False
+ except OSError:
+ return pack_template, False
+ return FIXTURE_TEMPLATE, True
+ return FIXTURE_TEMPLATE, False
+
+
def to_dsml_force(force: str | None) -> str | None:
"""forced_call's opening of a call in the Qwen form -> the same opening in DSML (the model's own spelling)."""
if not force:
diff --git a/serve/server.py b/serve/server.py
index cd24086fa..19e03a2be 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -5503,6 +5503,7 @@ class Service:
"strata-<model>.json for every request) leaves room to answer", flush=True)
if os.environ.get("STRATA_DEBUG") and raw_ids:
print(f"[strata] raw: {self.tok.decode(raw_ids)!r}", flush=True)
+ print(f"[strata] raw ids: {list(raw_ids)}", flush=True)
st["busy"] = False
st.pop("tail", None) # #212: the answer's end is not kept once it is done
st.pop("tool", None)
@@ -8377,6 +8378,12 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> in
delay_s=getattr(a, "script_delay", 0.0) or 0.0), None, {}
# the model's own chat template (exported with its tokenizer), else the original model's
tpl = tpath / "chat_template.jinja"
+ if (cfg.get("arch") or "") == "deepseek41":
+ from serve import dsml
+ tpl, swapped = dsml.pick_template(tpl)
+ if swapped:
+ print("[strata] this pack's chat template spells the tool-call block differently from the V4.1 model's; "
+ "using the V4.1 template so tool calls come back as real calls", flush=True)
svc = Service(engine, tok, ChatTemplate(tpl if tpl.exists() else ROOT / "serve/chat_template.jinja"),
model_name=default_model_name(cfg), vision=vision,
sampling_defaults=sampling_defaults,
diff --git a/serve/test_dsv41.py b/serve/test_dsv41.py
index b99cf692f..bc798034c 100644
--- a/serve/test_dsv41.py
+++ b/serve/test_dsv41.py
@@ -527,5 +527,26 @@ class Api(unittest.TestCase):
self.assertEqual(last["decode_tok_s"], 100.0)
+class PickTemplateTests(unittest.TestCase):
+ """A pack made from another GGUF can ship the V4-Flash template (`tool_calls`, no space): the V4.1 model then
+ writes malformed calls (P2003 install test). The server swaps in the V4.1 template for such a pack."""
+
+ def test_v4flash_pack_template_is_replaced(self):
+ import tempfile
+ from serve import dsml
+ with tempfile.TemporaryDirectory() as d:
+ old = Path(d) / "old.jinja"
+ old.write_text("{%- set dsml_token = '|DSML|' -%}{{ '<' + dsml_token + 'tool_calls>' }}", encoding="utf-8")
+ self.assertEqual(dsml.pick_template(old), (dsml.FIXTURE_TEMPLATE, True))
+ same = Path(d) / "same.jinja"
+ same.write_text(dsml.FIXTURE_TEMPLATE.read_text(encoding="utf-8"), encoding="utf-8")
+ self.assertEqual(dsml.pick_template(same), (same, False))
+ self.assertEqual(dsml.pick_template(Path(d) / "none.jinja"), (dsml.FIXTURE_TEMPLATE, False))
+
+ def test_fixture_is_accepted(self):
+ from serve import dsml
+ self.assertTrue(dsml.v41_template_ok(dsml.FIXTURE_TEMPLATE.read_text(encoding="utf-8")))
+
+
if __name__ == "__main__":
unittest.main()