Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(glm53): tool calling in the gateway, pinned to the official template
JustVugg59d76ddae797
diff --git a/c/openai_server.py b/c/openai_server.pyindex 3768cab2..8ac11279 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -1365,6 +1365,73 @@ def render_chat(messages, enable_thinking=False, reasoning_effort=None, tools=Noreturn "".join(prompt)+GLM53_TOOL_PREAMBLE = (+ "<|system|>\n# Tools\n\n"+ "You may call one or more functions to assist with the user query.\n\n"+ "You are provided with function signatures within <tools></tools> XML tags:\n"+ "<tools>\n")+GLM53_TOOL_EPILOGUE = (+ "\n</tools>\n\n"+ "For each function call, output the function name and arguments within the "+ "following XML format:\n"+ "<tool_call>{function-name}<arg_key>{arg-key-1}</arg_key>"+ "<arg_value>{arg-value-1}</arg_value><arg_key>{arg-key-2}</arg_key>"+ "<arg_value>{arg-value-2}</arg_value>...</tool_call>")+++def _glm53_tool_json(tool):+ """Una firma di strumento come la serializza il template di GLM-5.3.++ Le chiavi restano nell'ordine in cui il client le ha mandate, perche' il+ template itera `tool.items()` e non le riordina; `defer_loading` e `strict`+ non entrano nel prompt."""+ if isinstance(tool, dict) and "function" in tool:+ tool = tool["function"]+ if not isinstance(tool, dict):+ raise APIError(400, "each tool must be an object.", "tools")+ parts = [f'"{key}": {json.dumps(value, ensure_ascii=False)}'+ for key, value in tool.items()+ if key not in ("defer_loading", "strict")]+ return "{" + ", ".join(parts) + "}"+++def _glm53_tool_block(tools):+ """Il blocco di dichiarazione, spaziatura compresa.++ Gli a capo non sono decorativi: sono quelli che escono da chat_template.jinja+ e il test li confronta byte a byte contro jinja2, perche' un prompt che+ somiglia a quello dell'addestramento non e' quello dell'addestramento."""+ body = "".join(f"\n{_glm53_tool_json(tool)}\n\n"+ for tool in tools+ if not (isinstance(tool, dict)+ and (tool.get("function", tool) or {}).get("defer_loading")))+ return GLM53_TOOL_PREAMBLE + body + GLM53_TOOL_EPILOGUE+++def _glm53_tool_calls(calls):+ """Le chiamate di un turno assistente passato, nel formato che il modello+ stesso produce: <tool_call>nome<arg_key>k</arg_key><arg_value>v</arg_value>.+ Le stringhe passano cosi' come sono, il resto come JSON."""+ out = []+ for call in calls or []:+ if isinstance(call, dict) and "function" in call:+ call = call["function"]+ name = (call or {}).get("name", "")+ arguments = (call or {}).get("arguments")+ if isinstance(arguments, str):+ try:+ arguments = json.loads(arguments)+ except ValueError:+ arguments = {}+ pieces = [f"<tool_call>{name}"]+ for key, value in (arguments or {}).items():+ rendered = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)+ pieces.append(f"<arg_key>{key}</arg_key><arg_value>{rendered}</arg_value>")+ pieces.append("</tool_call>")+ out.append("".join(pieces))+ return "\n" + "".join(out) + "\n" if out else ""++def render_chat_glm53(messages, enable_thinking=False, reasoning_effort=None, tools=None,tool_choice=None):"""Render the text-only subset of the official GLM-5.3-Flash chat template.@@ -1372,27 +1439,38 @@ def render_chat_glm53(messages, enable_thinking=False, reasoning_effort=None, toNot a variant of the GLM-5.2 renderer above, and the differences are notcosmetic. GLM-5.3 emits the reasoning-effort system line ALWAYS, because itstemplate defaults the effort to Max rather than leaving it unset; it accepts- only low, high and max, not the six-level ladder; and its generation prompt+ only low, high and max, not the six-level ladder; its generation promptOPENS the reasoning block with a bare `<think>` where 5.2 closed it- immediately with `<think></think>`. A past assistant turn still closes it.+ immediately; and it declares tools with its own preamble and spacing.++ What the two share is how a call comes BACK: both models emit+ `<tool_call>name<arg_key>k</arg_key><arg_value>v</arg_value></tool_call>`,+ so the existing parser needs nothing added for this family.- Tools are refused rather than rendered. GLM-5.3 declares them inside a- `<tool_response><tools>` wrapper with its own JSON serialization, which is- not the block 5.2 uses, and emitting 5.2's shape here would put a prompt in- front of the model that it was never trained on -- an answer that looks- fine and is quietly wrong is worse than an error that says what is missing.+ The whole thing is pinned byte for byte against chat_template.jinja rendered+ with jinja2 (tests/test_glm53_chat_template.py). Getting the prompt nearly+ right is the failure mode worth guarding: the model answers either way."""if not isinstance(messages, list) or not messages:raise APIError(400, "`messages` must be a non-empty array.", "messages")- if tools and tool_choice != "none":- raise APIError(400, "Tool calling is not wired for GLM-5.3-Flash yet; "- "its declaration block differs from GLM-5.2's.", "tools")- # low/high passano, tutto il resto e' Max: e' la scala del template, non la- # nostra. `none` non arriva qui, spegne il ragionamento a monte.+ forced = None+ if isinstance(tool_choice, dict):+ forced = ((tool_choice.get("function") or {}).get("name")+ or tool_choice.get("name"))+ if forced:+ tools = [t for t in (tools or [])+ if ((t.get("function", t) if isinstance(t, dict) else {}).get("name") == forced)]+ elif tool_choice == "none":+ tools = None # il client li ha vietati: non si offrono++ # low e high passano, tutto il resto e' Max: e' la scala del template, non+ # la nostra. `none` non arriva qui, spegne il ragionamento a monte.effort = {"minimal": "Low", "low": "Low", "medium": "High","high": "High", "xhigh": "Max"}.get(reasoning_effort, "Max")prompt = ["[gMASK]<sop>", f"<|system|>Reasoning Effort: {effort}"]+ if tools:+ prompt.append(_glm53_tool_block(tools))for message in messages:if not isinstance(message, dict):@@ -1407,16 +1485,16 @@ def render_chat_glm53(messages, enable_thinking=False, reasoning_effort=None, toprompt.append(f"<|user|>{content}")elif role == "system":prompt.append(f"<|system|>{content}")+ elif role == "tool":+ prompt.append(f"<|observation|><tool_response>{content}</tool_response>")elif role == "assistant":reasoning = message.get("reasoning_content")if not isinstance(reasoning, str) and "</think>" in content:reasoning = content.split("</think>")[0].split("<think>")[-1]content = content.split("</think>")[-1]opened = f"<think>{reasoning}</think>" if isinstance(reasoning, str) else "<think></think>"- prompt.append(f"<|assistant|>{opened}{content.strip()}")- elif role == "tool":- raise APIError(400, "Tool results are not wired for GLM-5.3-Flash yet.",- "messages")+ prompt.append(f"<|assistant|>{opened}{content.strip()}"+ f"{_glm53_tool_calls(message.get('tool_calls'))}")else:raise APIError(400, f"unsupported message role {role!r}.", "messages")diff --git a/c/tests/test_glm53_chat_template.py b/c/tests/test_glm53_chat_template.pynew file mode 100644index 00000000..7cd602df--- /dev/null+++ b/c/tests/test_glm53_chat_template.py@@ -0,0 +1,139 @@+#!/usr/bin/env python3+"""Il renderer di GLM-5.3 nel gateway, contro chat_template.jinja ufficiale.++Il gateway rende i prompt a mano invece di far girare jinja a ogni richiesta, e+quella scelta si paga in un modo solo: la copia scritta a mano puo' scostarsi+dall'originale senza che nessuno se ne accorga, perche' il modello risponde+comunque. Un a capo di troppo dentro il blocco degli strumenti non da' errore,+da' risposte peggiori.++Qui il template vero viene reso con jinja2 e confrontato byte per byte con+quello che produce il gateway, sui casi che contano: senza strumenti, con uno,+con due, con una chiamata e il suo risultato, e con i livelli di ragionamento.++Serve chat_template.jinja del checkpoint. Se non c'e' il test si dichiara+saltato invece di passare: un test che non ha trovato il suo riferimento non ha+verificato niente, e dirlo verde sarebbe peggio che non averlo.++USO:+ python3 tests/test_glm53_chat_template.py --template PATH/chat_template.jinja+"""+import argparse+import json+import sys+from pathlib import Path++CASES = {+ "senza strumenti": {+ "messages": [{"role": "user", "content": "ciao"}],+ },+ "un solo strumento": {+ "messages": [{"role": "user", "content": "che tempo fa a Roma?"}],+ "tools": [{"type": "function", "function": {+ "name": "meteo", "description": "Il tempo",+ "parameters": {"type": "object",+ "properties": {"citta": {"type": "string"}},+ "required": ["citta"]}}}],+ },+ "due strumenti": {+ "messages": [{"role": "user", "content": "x"}],+ "tools": [+ {"type": "function", "function": {"name": "meteo", "description": "Il tempo"}},+ {"type": "function", "function": {"name": "ora", "description": "L'ora"}},+ ],+ },+ "chiamata e risultato": {+ "messages": [+ {"role": "user", "content": "che tempo fa a Roma?"},+ {"role": "assistant", "content": "", "tool_calls": [+ {"type": "function", "function": {+ "name": "meteo", "arguments": {"citta": "Roma"}}}]},+ {"role": "tool", "content": "sereno, 24 gradi"},+ ],+ "tools": [{"type": "function", "function": {+ "name": "meteo", "description": "Il tempo"}}],+ },+ "turno precedente con ragionamento": {+ "messages": [+ {"role": "user", "content": "a"},+ {"role": "assistant", "content": "<think>rifletto</think>b"},+ {"role": "user", "content": "c"},+ ],+ },+ "istruzione di sistema": {+ "messages": [+ {"role": "system", "content": "sii breve"},+ {"role": "user", "content": "ciao"},+ ],+ },+}++EFFORTS = {"low": "low", "high": "high", "xhigh": None, None: None}+++def reference(template_text, *, messages, tools=None, reasoning_effort=None):+ import jinja2+ environment = jinja2.Environment(trim_blocks=False, lstrip_blocks=False,+ extensions=["jinja2.ext.loopcontrols"])+ # jinja2 3.x non accetta ensure_ascii sul tojson incorporato; il template+ # lo passa esplicitamente, quindi il filtro va sostituito con uno che lo+ # capisce. Non cambia l'uscita, cambia solo la firma.+ environment.filters["tojson"] = (+ lambda value, ensure_ascii=False, **kw: json.dumps(value, ensure_ascii=ensure_ascii))+ rendered = environment.from_string(template_text)+ arguments = {"messages": messages, "add_generation_prompt": True}+ if tools:+ arguments["tools"] = tools+ if reasoning_effort:+ arguments["reasoning_effort"] = reasoning_effort+ return rendered.render(**arguments)+++def main() -> int:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--template", type=Path, required=True)+ arguments = parser.parse_args()++ if not arguments.template.exists():+ print(f"SKIP: manca {arguments.template}; il riferimento non c'e' e "+ f"questo test non ha verificato nulla")+ return 0+ try:+ import jinja2 # noqa: F401+ except ImportError:+ print("SKIP: jinja2 non installato; senza non c'e' riferimento")+ return 0++ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))+ import openai_server++ openai_server.ARCH = "glm53"+ template_text = arguments.template.read_text(encoding="utf-8")+ checked = 0+ for name, case in CASES.items():+ for effort, passed in EFFORTS.items():+ expected = reference(template_text, messages=case["messages"],+ tools=case.get("tools"), reasoning_effort=passed)+ produced = openai_server.render_chat_for_arch(+ case["messages"], reasoning_effort=effort, tools=case.get("tools"))+ if produced != expected:+ print(f"FAIL {name} (reasoning_effort={effort!r})")+ for position, (left, right) in enumerate(zip(produced, expected)):+ if left != right:+ start = max(0, position - 40)+ print(f" primo scostamento a {position}")+ print(f" gateway: ...{produced[start:position + 40]!r}")+ print(f" template: ...{expected[start:position + 40]!r}")+ break+ else:+ print(f" lunghezze diverse: {len(produced)} contro {len(expected)}")+ return 1+ checked += 1++ print(f"PASS GLM-5.3 chat template: {checked} rese identiche a "+ f"chat_template.jinja, strumenti e livelli di ragionamento compresi")+ return 0+++if __name__ == "__main__":+ sys.exit(main())