Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(qwen38): continue an open assistant turn

enie41ef1835823
diff --git a/c/openai_server.py b/c/openai_server.py
index 0b544f57..e392729d 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -1467,8 +1467,14 @@ def parse_qwen38_tool_calls(reply, tools=None):
def render_chat_qwen38(messages, enable_thinking=True, reasoning_effort=None, tools=None,
- tool_choice=None):
- """Text-only Qwen3.8 chat-template subset with native reasoning hints."""
+ tool_choice=None, add_generation_prompt=True):
+ """Text-only Qwen3.8 chat-template subset with native reasoning hints.
+
+ add_generation_prompt=False continues a trailing assistant turn. ChatML closes every
+ turn with <|im_end|>, so the open-turn shape is the past-turn render of that last message
+ MINUS its terminator, and no generation cue after it -- the position the model occupies
+ while writing an assistant turn. (GLM has no per-turn terminator, so there suppressing the
+ cue is enough; here the terminator has to be dropped too.)"""
if not isinstance(messages, list) or not messages:
raise APIError(400, "`messages` must be a non-empty array.", "messages")
if tool_choice in ("none",):
@@ -1560,12 +1566,16 @@ def render_chat_qwen38(messages, enable_thinking=True, reasoning_effort=None, to
rendered = f"<think>\n{reasoning.strip()}\n</think>\n\n{text}"
if calls:
rendered += _qwen38_tool_calls(calls, bool(text.strip()), index)
- parts.append(f"<|im_start|>assistant\n{rendered}<|im_end|>\n")
+ # A continued turn is the last message rendered open: no <|im_end|>, no cue.
+ terminator = "" if (not add_generation_prompt and index == len(messages) - 1) \
+ else "<|im_end|>\n"
+ parts.append(f"<|im_start|>assistant\n{rendered}{terminator}")
continue
parts.append(f"<|im_start|>{role}\n{text}<|im_end|>\n")
- parts.append("<|im_start|>assistant\n")
- parts.append("<think>\n" if enable_thinking else "<think>\n\n</think>\n\n")
+ if add_generation_prompt:
+ parts.append("<|im_start|>assistant\n")
+ parts.append("<think>\n" if enable_thinking else "<think>\n\n</think>\n\n")
return "".join(parts)
@@ -2341,7 +2351,7 @@ def render_chat_dsv41(messages, enable_thinking=False, reasoning_effort=None, to
# (the cue is appended, exactly as before this existed) rather than erroring -- continuation is
# on by default, and a family without its open-turn shape yet must not start rejecting requests
# nobody opted into. Each renderer adds itself here in the same commit that derives its shape.
-CONTINUATION_FAMILIES = {"glm53"}
+CONTINUATION_FAMILIES = {"glm53", "qwen38"}
def resolve_generation_prompt(messages, body):
@@ -2429,9 +2439,11 @@ def render_chat_for_arch(messages, enable_thinking=False, reasoning_effort=None,
if ARCH == "glm53":
return render_chat_glm53(messages, enable_thinking, reasoning_effort, tools,
tool_choice, add_generation_prompt)
+ if ARCH == "qwen38":
+ return render_chat_qwen38(messages, enable_thinking, reasoning_effort, tools,
+ tool_choice, add_generation_prompt)
renderer = (render_chat_kimi if ARCH == "kimi" else
render_chat_qwen if ARCH == "qwen36" else
- render_chat_qwen38 if ARCH == "qwen38" else
render_chat_v4 if ARCH == "deepseek_v4" else
render_chat_dsv41 if ARCH == "deepseek_v41" else
render_chat_olmoe if ARCH == "olmoe" else render_chat)
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index ae4f58f7..d11968a3 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -2148,7 +2148,7 @@ class TrailingAssistantTurnTest(unittest.TestCase):
yet must render as before -- append the cue -- not reject a request nobody opted into.
Each family leaves this list in the commit that derives its shape; Kimi K3 (framed
engine-side, so its open turn is a change in kimi_k3.c) is the standing case."""
- for arch in ("glm", "deepseek_v4", "kimi", "qwen38", "olmoe", "inkling", "qwen36"):
+ for arch in ("glm", "deepseek_v4", "kimi", "olmoe", "inkling", "qwen36"):
with self.on(), patch("openai_server.ARCH", arch):
self.assertTrue(resolve_generation_prompt(self.OPEN_TURN, {}))
diff --git a/c/tests/test_qwen38_chat_template.py b/c/tests/test_qwen38_chat_template.py
index 73dd631b..97ebc893 100644
--- a/c/tests/test_qwen38_chat_template.py
+++ b/c/tests/test_qwen38_chat_template.py
@@ -22,7 +22,14 @@ visto: non da' errore, da' chiamate malformate.
Il template vero viene reso con jinja2 e confrontato byte per byte con quello
che produce il gateway. Se manca il template il test si dichiara SALTATO invece
di passare: un test che non ha trovato il suo riferimento non ha verificato
-niente, e dirlo verde sarebbe peggio che non averlo.
+niente, e dirlo verde sarebbe peggio che non averlo -- percio' un salto esce con
+codice 2, distinto dallo 0 di un confronto riuscito.
+
+RIFERIMENTO (scaricato 2026-09-10):
+ repo Qwen/Qwen3.8-Flash-Next-FP8
+ file chat_template.jinja
+ sha256 c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041
+ hf download Qwen/Qwen3.8-Flash-Next-FP8 chat_template.jinja
USO:
python3 tests/test_qwen38_chat_template.py --template PATH/chat_template.jinja
@@ -115,7 +122,8 @@ CASES = {
EFFORTS = ("xhigh", "medium", "low")
-def reference(template_text, *, messages, tools=None, reasoning_effort=None):
+def reference(template_text, *, messages, tools=None, reasoning_effort=None,
+ add_generation_prompt=True):
import jinja2
def raise_exception(message):
@@ -127,7 +135,7 @@ def reference(template_text, *, messages, tools=None, reasoning_effort=None):
lambda value, ensure_ascii=False, **kw: json.dumps(value, ensure_ascii=ensure_ascii))
environment.globals["raise_exception"] = raise_exception
rendered = environment.from_string(template_text)
- arguments = {"messages": messages, "add_generation_prompt": True,
+ arguments = {"messages": messages, "add_generation_prompt": add_generation_prompt,
"enable_thinking": True}
if tools:
arguments["tools"] = tools
@@ -157,12 +165,12 @@ def main() -> int:
if not arguments.template.exists():
print(f"SKIP: manca {arguments.template}; il riferimento non c'e' e "
f"questo test non ha verificato nulla")
- return 0
+ return 2 # salto != successo (vedi docstring)
try:
import jinja2 # noqa: F401
except ImportError:
print("SKIP: jinja2 non installato; senza non c'e' riferimento")
- return 0
+ return 2 # salto != successo (vedi docstring)
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import openai_server
@@ -185,6 +193,49 @@ def main() -> int:
show(name, ours, theirs)
failures += 1
+ # Prosecuzione: l'ultimo turno assistant e' da CONTINUARE, non uno gia' finito.
+ # ChatML chiude ogni turno con <|im_end|>, e questo template non ha un ramo di
+ # continuazione: il suo add_generation_prompt=False toglie solo la cue, il turno resta
+ # chiuso. La forma aperta e' percio' quel rendering MENO il <|im_end|>\n finale -- la
+ # posizione in cui il modello si trova mentre scrive un turno, e da cui prosegue.
+ aperto = [{"role": "user", "content": "capitale della Francia?"},
+ {"role": "assistant", "content": "La capitale e'"}]
+ produced = openai_server.render_chat_for_arch(aperto, enable_thinking=True,
+ reasoning_effort="low",
+ add_generation_prompt=False)
+ closed = reference(template_text, messages=aperto, reasoning_effort="low",
+ add_generation_prompt=False)
+ # Il terminatore da togliere DEVE esserci nel riferimento: se il template cambiasse
+ # convenzione, toglierlo "se c'e'" sarebbe un no-op silenzioso e il confronto perderebbe
+ # senso. Percio' lo esigiamo prima di tagliarlo.
+ TERM = "<|im_end|>\n"
+ if not closed.endswith(TERM):
+ print(f"FAIL prosecuzione: il riferimento non finisce col terminatore {TERM!r} da "
+ f"togliere -- convenzione del template cambiata? coda: {closed[-40:]!r}")
+ failures += 1
+ expected = closed
+ else:
+ expected = closed[:-len(TERM)]
+ if produced == expected:
+ print("ok prosecuzione: turno aperto = template(add_generation_prompt=False) "
+ "senza il <|im_end|> finale")
+ else:
+ show("prosecuzione", produced, expected)
+ failures += 1
+ if not produced.endswith("La capitale e'"):
+ print(f"FAIL prosecuzione: il prompt non finisce sull'apertura del client: "
+ f"{produced[-60:]!r}")
+ failures += 1
+ if produced.rstrip("\n").endswith("<|im_end|>"):
+ print("FAIL prosecuzione: il turno resta chiuso col terminatore")
+ failures += 1
+ # Controllo negativo: col ramo normale lo stesso scambio DEVE finire sulla cue, o il
+ # confronto qui sopra non starebbe distinguendo niente.
+ if not openai_server.render_chat_for_arch(
+ aperto, enable_thinking=True, reasoning_effort="low").endswith("<think>\n"):
+ print("FAIL prosecuzione: il ramo normale non emette piu' il prompt di generazione")
+ failures += 1
+
# Il giro completo: rendere una chiamata e rileggerla deve restituire quello
# che ci era stato dato. E' la meta' che il confronto col template non copre,
# perche' il template sa solo scrivere.