Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(qwen38): continue an open assistant turn
enie41ef1835823
diff --git a/c/openai_server.py b/c/openai_server.pyindex 0b544f57..e392729d 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -1467,8 +1467,14 @@ def parse_qwen38_tool_calls(reply, tools=None):def render_chat_qwen38(messages, enable_thinking=True, reasoning_effort=None, tools=None,- tool_choice=None):- """Text-only Qwen3.8 chat-template subset with native reasoning hints."""+ tool_choice=None, add_generation_prompt=True):+ """Text-only Qwen3.8 chat-template subset with native reasoning hints.++ add_generation_prompt=False continues a trailing assistant turn. ChatML closes every+ turn with <|im_end|>, so the open-turn shape is the past-turn render of that last message+ MINUS its terminator, and no generation cue after it -- the position the model occupies+ while writing an assistant turn. (GLM has no per-turn terminator, so there suppressing the+ cue is enough; here the terminator has to be dropped too.)"""if not isinstance(messages, list) or not messages:raise APIError(400, "`messages` must be a non-empty array.", "messages")if tool_choice in ("none",):@@ -1560,12 +1566,16 @@ def render_chat_qwen38(messages, enable_thinking=True, reasoning_effort=None, torendered = f"<think>\n{reasoning.strip()}\n</think>\n\n{text}"if calls:rendered += _qwen38_tool_calls(calls, bool(text.strip()), index)- parts.append(f"<|im_start|>assistant\n{rendered}<|im_end|>\n")+ # A continued turn is the last message rendered open: no <|im_end|>, no cue.+ terminator = "" if (not add_generation_prompt and index == len(messages) - 1) \+ else "<|im_end|>\n"+ parts.append(f"<|im_start|>assistant\n{rendered}{terminator}")continueparts.append(f"<|im_start|>{role}\n{text}<|im_end|>\n")- parts.append("<|im_start|>assistant\n")- parts.append("<think>\n" if enable_thinking else "<think>\n\n</think>\n\n")+ if add_generation_prompt:+ parts.append("<|im_start|>assistant\n")+ parts.append("<think>\n" if enable_thinking else "<think>\n\n</think>\n\n")return "".join(parts)@@ -2341,7 +2351,7 @@ def render_chat_dsv41(messages, enable_thinking=False, reasoning_effort=None, to# (the cue is appended, exactly as before this existed) rather than erroring -- continuation is# on by default, and a family without its open-turn shape yet must not start rejecting requests# nobody opted into. Each renderer adds itself here in the same commit that derives its shape.-CONTINUATION_FAMILIES = {"glm53"}+CONTINUATION_FAMILIES = {"glm53", "qwen38"}def resolve_generation_prompt(messages, body):@@ -2429,9 +2439,11 @@ def render_chat_for_arch(messages, enable_thinking=False, reasoning_effort=None,if ARCH == "glm53":return render_chat_glm53(messages, enable_thinking, reasoning_effort, tools,tool_choice, add_generation_prompt)+ if ARCH == "qwen38":+ return render_chat_qwen38(messages, enable_thinking, reasoning_effort, tools,+ tool_choice, add_generation_prompt)renderer = (render_chat_kimi if ARCH == "kimi" elserender_chat_qwen if ARCH == "qwen36" else- render_chat_qwen38 if ARCH == "qwen38" elserender_chat_v4 if ARCH == "deepseek_v4" elserender_chat_dsv41 if ARCH == "deepseek_v41" elserender_chat_olmoe if ARCH == "olmoe" else render_chat)diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex ae4f58f7..d11968a3 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -2148,7 +2148,7 @@ class TrailingAssistantTurnTest(unittest.TestCase):yet must render as before -- append the cue -- not reject a request nobody opted into.Each family leaves this list in the commit that derives its shape; Kimi K3 (framedengine-side, so its open turn is a change in kimi_k3.c) is the standing case."""- for arch in ("glm", "deepseek_v4", "kimi", "qwen38", "olmoe", "inkling", "qwen36"):+ for arch in ("glm", "deepseek_v4", "kimi", "olmoe", "inkling", "qwen36"):with self.on(), patch("openai_server.ARCH", arch):self.assertTrue(resolve_generation_prompt(self.OPEN_TURN, {}))diff --git a/c/tests/test_qwen38_chat_template.py b/c/tests/test_qwen38_chat_template.pyindex 73dd631b..97ebc893 100644--- a/c/tests/test_qwen38_chat_template.py+++ b/c/tests/test_qwen38_chat_template.py@@ -22,7 +22,14 @@ visto: non da' errore, da' chiamate malformate.Il template vero viene reso con jinja2 e confrontato byte per byte con quelloche produce il gateway. Se manca il template il test si dichiara SALTATO invecedi passare: un test che non ha trovato il suo riferimento non ha verificato-niente, e dirlo verde sarebbe peggio che non averlo.+niente, e dirlo verde sarebbe peggio che non averlo -- percio' un salto esce con+codice 2, distinto dallo 0 di un confronto riuscito.++RIFERIMENTO (scaricato 2026-09-10):+ repo Qwen/Qwen3.8-Flash-Next-FP8+ file chat_template.jinja+ sha256 c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041+ hf download Qwen/Qwen3.8-Flash-Next-FP8 chat_template.jinjaUSO:python3 tests/test_qwen38_chat_template.py --template PATH/chat_template.jinja@@ -115,7 +122,8 @@ CASES = {EFFORTS = ("xhigh", "medium", "low")-def reference(template_text, *, messages, tools=None, reasoning_effort=None):+def reference(template_text, *, messages, tools=None, reasoning_effort=None,+ add_generation_prompt=True):import jinja2def raise_exception(message):@@ -127,7 +135,7 @@ def reference(template_text, *, messages, tools=None, reasoning_effort=None):lambda value, ensure_ascii=False, **kw: json.dumps(value, ensure_ascii=ensure_ascii))environment.globals["raise_exception"] = raise_exceptionrendered = environment.from_string(template_text)- arguments = {"messages": messages, "add_generation_prompt": True,+ arguments = {"messages": messages, "add_generation_prompt": add_generation_prompt,"enable_thinking": True}if tools:arguments["tools"] = tools@@ -157,12 +165,12 @@ def main() -> int:if not arguments.template.exists():print(f"SKIP: manca {arguments.template}; il riferimento non c'e' e "f"questo test non ha verificato nulla")- return 0+ return 2 # salto != successo (vedi docstring)try:import jinja2 # noqa: F401except ImportError:print("SKIP: jinja2 non installato; senza non c'e' riferimento")- return 0+ return 2 # salto != successo (vedi docstring)sys.path.insert(0, str(Path(__file__).resolve().parents[1]))import openai_server@@ -185,6 +193,49 @@ def main() -> int:show(name, ours, theirs)failures += 1+ # Prosecuzione: l'ultimo turno assistant e' da CONTINUARE, non uno gia' finito.+ # ChatML chiude ogni turno con <|im_end|>, e questo template non ha un ramo di+ # continuazione: il suo add_generation_prompt=False toglie solo la cue, il turno resta+ # chiuso. La forma aperta e' percio' quel rendering MENO il <|im_end|>\n finale -- la+ # posizione in cui il modello si trova mentre scrive un turno, e da cui prosegue.+ aperto = [{"role": "user", "content": "capitale della Francia?"},+ {"role": "assistant", "content": "La capitale e'"}]+ produced = openai_server.render_chat_for_arch(aperto, enable_thinking=True,+ reasoning_effort="low",+ add_generation_prompt=False)+ closed = reference(template_text, messages=aperto, reasoning_effort="low",+ add_generation_prompt=False)+ # Il terminatore da togliere DEVE esserci nel riferimento: se il template cambiasse+ # convenzione, toglierlo "se c'e'" sarebbe un no-op silenzioso e il confronto perderebbe+ # senso. Percio' lo esigiamo prima di tagliarlo.+ TERM = "<|im_end|>\n"+ if not closed.endswith(TERM):+ print(f"FAIL prosecuzione: il riferimento non finisce col terminatore {TERM!r} da "+ f"togliere -- convenzione del template cambiata? coda: {closed[-40:]!r}")+ failures += 1+ expected = closed+ else:+ expected = closed[:-len(TERM)]+ if produced == expected:+ print("ok prosecuzione: turno aperto = template(add_generation_prompt=False) "+ "senza il <|im_end|> finale")+ else:+ show("prosecuzione", produced, expected)+ failures += 1+ if not produced.endswith("La capitale e'"):+ print(f"FAIL prosecuzione: il prompt non finisce sull'apertura del client: "+ f"{produced[-60:]!r}")+ failures += 1+ if produced.rstrip("\n").endswith("<|im_end|>"):+ print("FAIL prosecuzione: il turno resta chiuso col terminatore")+ failures += 1+ # Controllo negativo: col ramo normale lo stesso scambio DEVE finire sulla cue, o il+ # confronto qui sopra non starebbe distinguendo niente.+ if not openai_server.render_chat_for_arch(+ aperto, enable_thinking=True, reasoning_effort="low").endswith("<think>\n"):+ print("FAIL prosecuzione: il ramo normale non emette piu' il prompt di generazione")+ failures += 1+# Il giro completo: rendere una chiamata e rileggerla deve restituire quello# che ci era stato dato. E' la meta' che il confronto col template non copre,# perche' il template sa solo scrivere.