Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(glm53): continue an open assistant turn (COLI_CONTINUE_ASSISTANT)

eni4617a3dfb4c1
diff --git a/c/openai_server.py b/c/openai_server.py
index b319b314..2ff47220 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -521,7 +521,7 @@ def _dsv4_tools_block(tools):
def _dsv4_tool_calls(tool_calls):
- """Render OpenAI-format tool_calls into a V4 DSML block (incl. the leading
+ """Render OpenAI-format tool_calls into a V4 DSML block (incl. the leading
)."""
return v4_dsml.render_tool_calls(tool_calls)
@@ -2024,7 +2024,7 @@ def _glm53_tool_calls(calls):
def render_chat_glm53(messages, enable_thinking=False, reasoning_effort=None, tools=None,
- tool_choice=None):
+ tool_choice=None, add_generation_prompt=True):
"""Render the text-only subset of the official GLM-5.3-Flash chat template.
Not a variant of the GLM-5.2 renderer above, and the differences are not
@@ -2128,7 +2128,14 @@ def render_chat_glm53(messages, enable_thinking=False, reasoning_effort=None, to
# su cui il modello e' addestrato" perche' il template lo scrive davanti a un
# TURNO PASSATO senza ragionamento. E' vero per un turno passato e falso per il
# prompt di generazione: la posizione da cui il modello scrive non e' mai quella.
- prompt.append("<|assistant|><think>")
+ #
+ # add_generation_prompt=False non e' una forma nostra: e' l'altro ramo di questo stesso
+ # `if` nel template. Il prompt finisce allora sull'ultimo turno assistant reso come
+ # turno PASSATO -- <think></think> seguito dal contenuto -- e il modello lo prosegue
+ # invece di aprirne uno nuovo. Chi non chiede la prosecuzione non vede differenza:
+ # il ramo True e' invariato, byte per byte, ed e' quello che il test confronta.
+ if add_generation_prompt:
+ prompt.append("<|assistant|><think>")
return "".join(prompt)
@@ -2307,14 +2314,114 @@ def render_chat_dsv41(messages, enable_thinking=False, reasoning_effort=None, to
return "".join(prompt)
+# ---- continuing an unfinished assistant turn (COLI_CONTINUE_ASSISTANT) ----------------
+# A trailing `assistant` message means "continue writing this turn", not "here is a turn I
+# already finished". The official template says exactly that, and says it in one place --
+# {%- if add_generation_prompt -%}<|assistant|>{{- '<think>' -}}{%- endif -%}
+# -- whose False branch every renderer in this file hard-codes to True. With the cue
+# suppressed the prompt ends mid-turn, on the shape the template writes in front of a PAST
+# assistant turn, which is a position the model saw all through training.
+#
+# That distinction is what makes this safe on GLM-5.3 specifically. #1327 measured that a
+# CLOSED, EMPTY <think></think> at the end of a prompt is out of distribution and the model
+# keeps reasoning through it. The position here is a different one: <think></think> followed
+# by real content, i.e. the past-turn shape, which is why a continuation must carry text.
+#
+# llama.cpp needs no switch for this because it runs the checkpoint's jinja at request time,
+# so `add_generation_prompt=False` costs it nothing. This gateway renders by hand, on purpose
+# and for speed (tests/test_glm53_chat_template.py says why), and the bill for that choice is
+# exactly here: one template flag, eight open-turn shapes to derive. That is the honest reason
+# this ships for one family behind a switch instead of for all of them by default.
+
+
+def resolve_generation_prompt(messages, body):
+ """Does this prompt end on a generation cue, or on an assistant turn to continue?
+
+ Returns True for the ordinary case (append the cue) and False for a continuation, which is
+ the template's `add_generation_prompt=False`.
+
+ COLI_CONTINUE_ASSISTANT=1 turns continuation on for the whole server, the way
+ COLI_THINK and COLI_TOOL_SALVAGE turn on their behaviours. It is deliberately NOT a
+ request field: a client would have to know colibri specifically to send one, and the
+ clients that most want this -- anything pointed at an OpenAI- or Anthropic-compatible
+ URL -- send a message list and nothing else. A trailing assistant turn already says
+ "continue me"; a second way to say it in the body would only be reachable by hand.
+
+ Off is the default so that no existing deployment changes under anyone. Read it as a
+ migration switch rather than a mode: what it disables is not a behaviour anybody wants,
+ only the one that is there today.
+
+ While off, both protocol paths keep their existing behaviour so clients sending a
+ trailing assistant turn today do not start getting errors from a release they did not
+ opt into.
+ """
+ continuing = os.environ.get("COLI_CONTINUE_ASSISTANT", "0") == "1"
+ last = messages[-1] if isinstance(messages, list) and messages else None
+ if not (isinstance(last, dict) and last.get("role") == "assistant"):
+ return True
+ where = f"messages.{len(messages) - 1}"
+ if not continuing:
+ return True
+ if ARCH != "glm53":
+ raise APIError(400, f"COLI_CONTINUE_ASSISTANT is on, but continuing an assistant "
+ f"turn is implemented for the glm53 engine only, not {ARCH!r}: every "
+ f"family needs its own open-turn shape derived from its own template.",
+ "messages", "unsupported_parameter")
+ if body.get("tools") or body.get("functions"):
+ raise APIError(400, "A continued assistant turn cannot be combined with `tools`: "
+ "the tool-call parsers read an assistant turn from its start, and a "
+ "continuation can end anywhere -- including inside a <tool_call> "
+ "block.", "tools", "unsupported_parameter")
+ if last.get("tool_calls"):
+ raise APIError(400, "A continued `assistant` message cannot carry `tool_calls`.",
+ f"{where}.tool_calls", "unsupported_value")
+ if len(messages) < 2:
+ raise APIError(400, "A continued `assistant` turn needs a preceding turn to "
+ "continue from.", "messages")
+ raw = last.get("content")
+ if isinstance(raw, list): # multimodal parts: only the text counts
+ text = "".join(part.get("text", "") for part in raw
+ if isinstance(part, dict) and part.get("type") == "text")
+ elif raw is None:
+ text = ""
+ elif isinstance(raw, str):
+ text = raw
+ else:
+ raise APIError(400, "Message content must be a string or an array of blocks.",
+ f"{where}.content")
+ if not text.strip():
+ raise APIError(400, "A continued `assistant` turn needs text to continue. An empty "
+ "one ends the prompt on a closed, empty <think></think> block, which "
+ "is the out-of-distribution position #1327 removed -- the model "
+ "reasons straight through it instead of answering.",
+ f"{where}.content", "invalid_value")
+ if text != text.rstrip():
+ raise APIError(400, "A continued `assistant` turn cannot end with whitespace: the "
+ "template strips it, so the model would resume from different bytes "
+ "than the ones sent. Put the space at the start of what you expect "
+ "back instead.", f"{where}.content", "invalid_value")
+ return False
+
+
def render_chat_for_arch(messages, enable_thinking=False, reasoning_effort=None, tools=None,
- tool_choice=None, audio_out=None):
- """Render a chat request with the active engine's native prompt contract."""
+ tool_choice=None, audio_out=None, add_generation_prompt=True):
+ """Render a chat request with the active engine's native prompt contract.
+
+ `add_generation_prompt=False` (a continued assistant turn) is implemented for glm53 only.
+ resolve_generation_prompt() refuses it for every other family upstream; the assert
+ here is the backstop, because silently appending a cue to a continuation is the exact
+ failure this change exists to remove.
+ """
+ if not add_generation_prompt and ARCH != "glm53":
+ raise APIError(400, f"Continuing an assistant turn is not implemented for {ARCH!r}.",
+ "messages", "unsupported_parameter")
if ARCH == "inkling":
return render_chat_inkling(messages, enable_thinking, reasoning_effort, tools,
tool_choice, audio_out=audio_out)
- renderer = (render_chat_glm53 if ARCH == "glm53" else
- render_chat_kimi if ARCH == "kimi" else
+ if ARCH == "glm53":
+ return render_chat_glm53(messages, enable_thinking, reasoning_effort, tools,
+ tool_choice, add_generation_prompt)
+ renderer = (render_chat_kimi if ARCH == "kimi" else
render_chat_qwen if ARCH == "qwen36" else
render_chat_qwen38 if ARCH == "qwen38" else
render_chat_v4 if ARCH == "deepseek_v4" else
@@ -2332,7 +2439,7 @@ def render_chat_for_arch(messages, enable_thinking=False, reasoning_effort=None,
ANTHROPIC_LOCAL_SIGNATURE = "colibri-local" # opaque compatibility metadata, not a crypto proof
-def starts_in_reasoning(enable_thinking):
+def starts_in_reasoning(enable_thinking, add_generation_prompt=True):
"""Se l'uscita del modello comincia DENTRO al blocco di ragionamento.
Dipende da come il prompt lo ha lasciato, e ogni famiglia lo lascia come
@@ -2345,8 +2452,20 @@ def starts_in_reasoning(enable_thinking):
ha un interruttore, render_chat_glm53 apre <think> SEMPRE, e "thinking
spento" vuol dire solo effort Low. L'uscita comincia dentro al blocco in
ogni caso; partire in modalita' testo perche' il client ha detto False e'
- esattamente il ragionamento incollato davanti alla risposta di #1278."""
- return enable_thinking or ARCH == "glm53"
+ esattamente il ragionamento incollato davanti alla risposta di #1278.
+
+ Il turno proseguito (add_generation_prompt=False) e' il terzo stato, e non
+ lo dice l'interruttore: il prompt finisce sull'ultimo turno assistant reso
+ come turno PASSATO, quindi <think></think> GIA' CHIUSO seguito dal
+ contenuto, col ragionamento acceso o spento che sia. Il modello riprende in
+ modalita' testo; se lo splitter parte in modalita' ragionamento aspetta un
+ </think> che e' gia' passato, e archivia come ragionamento tutta la
+ risposta -- content vuoto, reasoning_content pieno, stop pulito. Misurato
+ su glm53 int4, CPU: 10 e 109 caratteri di ragionamento contro
+ zero di risposta, col prompt corretto sul filo. Vale anche per glm53: la
+ regola di famiglia sopra dice dove comincia un turno NUOVO, e il turno
+ proseguito non ne apre nessuno."""
+ return (enable_thinking or ARCH == "glm53") and add_generation_prompt
class ThinkingStreamSplit:
@@ -2399,10 +2518,12 @@ class ThinkingStreamSplit:
close = finish # interface parity with InklingStreamSplit in the streaming path
-def split_thinking_reply(text, enable_thinking=True):
+def split_thinking_reply(text, enable_thinking=True, add_generation_prompt=True):
"""Return the marker-free (thinking, answer) portions of one GLM reply."""
thinking, answer = [], []
- split = ThinkingStreamSplit(thinking.append, answer.append, initial_thinking=starts_in_reasoning(enable_thinking))
+ split = ThinkingStreamSplit(thinking.append, answer.append,
+ initial_thinking=starts_in_reasoning(enable_thinking,
+ add_generation_prompt))
split.feed(text)
split.finish()
return "".join(thinking), "".join(answer)
@@ -4252,7 +4373,8 @@ class APIHandler(BaseHTTPRequestHandler):
return {"type": "error", "error": {"type": error.error_type, "message": error.message}}
def generation(self, body, prompt, request_id, chat, tools=None, tool_choice=None,
- enable_thinking=False, audio=None, image=None):
+ enable_thinking=False, audio=None, image=None,
+ add_generation_prompt=True):
# COLI_DEBUG tees the engine transaction to stderr: 1 = decoded output stream only,
# 2 = both sides (rendered prompt + output). render_chat already folds prior turns and
# tool results into `prompt`, so level 2 is the full conversation the engine saw.
@@ -4328,7 +4450,8 @@ class APIHandler(BaseHTTPRequestHandler):
# #597 item 4: GLM emits reasoning then </think> then the answer. Route the
# reasoning to reasoning_content instead of dumping it (or the raw </think>)
# into the visible answer / tool-call parser.
- reasoning, text = split_thinking_reply(text, enable_thinking)
+ reasoning, text = split_thinking_reply(text, enable_thinking,
+ add_generation_prompt)
length_finish = "length" if stats["length_limited"] else "stop"
if chat and tools:
content, calls = parse_arch_tool_calls(text, tools, sideband.reply())
@@ -4483,7 +4606,8 @@ class APIHandler(BaseHTTPRequestHandler):
# #597: keep GLM reasoning out of the tool-call buffer β€” a think splitter sends it
# to reasoning_content and passes only the answer text on to feed_content/parser.
think = (ThinkingStreamSplit(emit_reasoning, feed_content,
- initial_thinking=starts_in_reasoning(enable_thinking))
+ initial_thinking=starts_in_reasoning(
+ enable_thinking, add_generation_prompt))
if glm_think else None)
def emit_tools(chunk):
if dbg_echo:
@@ -4518,8 +4642,10 @@ class APIHandler(BaseHTTPRequestHandler):
if splitter is not None: # inkling content/marker splitter
content_split = splitter
elif glm_think: # GLM <think> reasoning β†’ reasoning_content
- content_split = ThinkingStreamSplit(emit_reasoning, emit,
- initial_thinking=starts_in_reasoning(enable_thinking))
+ content_split = ThinkingStreamSplit(
+ emit_reasoning, emit,
+ initial_thinking=starts_in_reasoning(enable_thinking,
+ add_generation_prompt))
else:
content_split = None
def emit_plain(chunk):
@@ -4634,10 +4760,13 @@ class APIHandler(BaseHTTPRequestHandler):
raise APIError(400, "one image per request for now; the engine "
"holds a single pending image.", "messages")
image = images[0] if images else None
+ add_generation_prompt = resolve_generation_prompt(messages, body)
prompt = render_chat_for_arch(messages, enable_thinking, reasoning_effort,
- tools, tool_choice, audio_out=audio_clips)
+ tools, tool_choice, audio_out=audio_clips,
+ add_generation_prompt=add_generation_prompt)
self.generation(body, prompt, request_id, True, tools, tool_choice,
enable_thinking=enable_thinking,
+ add_generation_prompt=add_generation_prompt,
audio=b"".join(audio_clips) if audio_clips else None,
image=image)
@@ -4676,12 +4805,16 @@ class APIHandler(BaseHTTPRequestHandler):
if tool_choice == "none":
tools = None
default_effort = "xhigh" if ARCH == "qwen38" and thinking is None else "high"
+ add_generation_prompt = resolve_generation_prompt(messages, body)
prompt = render_chat_for_arch(messages, enable_thinking,
default_effort if enable_thinking else None,
- tools, tool_choice)
- self.anthropic_generation(translated, prompt, request_id, tools, enable_thinking)
+ tools, tool_choice,
+ add_generation_prompt=add_generation_prompt)
+ self.anthropic_generation(translated, prompt, request_id, tools, enable_thinking,
+ add_generation_prompt)
- def anthropic_generation(self, body, prompt, request_id, tools, enable_thinking):
+ def anthropic_generation(self, body, prompt, request_id, tools, enable_thinking,
+ add_generation_prompt=True):
maximum, temperature, top_p, grammar, _stop_sequences = generation_options(
body, self.server.max_tokens)
# Same policy as /v1/chat/completions: `body` is the translated OpenAI-shaped
@@ -4713,7 +4846,8 @@ class APIHandler(BaseHTTPRequestHandler):
if ARCH == "inkling":
text, reasoning = split_inkling(text)
elif enable_thinking:
- reasoning, text = split_thinking_reply(text)
+ reasoning, text = split_thinking_reply(text, enable_thinking,
+ add_generation_prompt)
if enable_thinking:
content.append({"type": "thinking", "thinking": reasoning,
"signature": ANTHROPIC_LOCAL_SIGNATURE})
@@ -4875,7 +5009,8 @@ class APIHandler(BaseHTTPRequestHandler):
# lo splitter serve pure col ragionamento "spento", o il
# pensiero finisce incollato davanti alla risposta.
split = (ThinkingStreamSplit(emit_thinking, emit_answer, close_thinking)
- if starts_in_reasoning(enable_thinking) else None)
+ if starts_in_reasoning(enable_thinking, add_generation_prompt)
+ else None)
def on_text(chunk):
raw.append(chunk)
diff --git a/c/tests/test_anthropic_messages.py b/c/tests/test_anthropic_messages.py
index b8006b44..7d9bd6a5 100644
--- a/c/tests/test_anthropic_messages.py
+++ b/c/tests/test_anthropic_messages.py
@@ -11,6 +11,7 @@ is the reference client from the issue β€” not merely that the handler returns 2
- the Anthropic error envelope, which is not the OpenAI one.
"""
import json
+import os
import re
import threading
import unittest
@@ -19,7 +20,8 @@ from urllib.error import HTTPError
from urllib.request import Request, urlopen
from openai_server import (APIServer, anthropic_to_openai, anthropic_tools, APIError,
- render_chat, render_chat_inkling, render_chat_kimi, render_chat_v4)
+ render_chat, render_chat_glm53, render_chat_inkling,
+ render_chat_kimi, render_chat_v4)
class FakeEngine:
@@ -155,6 +157,24 @@ class MessagesHTTPTest(unittest.TestCase):
self.assertEqual(payload["usage"], {"input_tokens": 11, "output_tokens": 3})
self.assertTrue(payload["id"].startswith("msg_"))
+ def test_trailing_assistant_turn_follows_shared_continuation_switch(self):
+ """Off preserves the existing cue; on continues the turn on both endpoints."""
+ messages = [{"role": "user", "content": "The capital of France is?"},
+ {"role": "assistant", "content": "The capital is"}]
+ with patch("openai_server.ARCH", "glm53"):
+ with patch.dict(os.environ, {"COLI_CONTINUE_ASSISTANT": "0"}):
+ with self.post(self.base_body(messages=messages)) as response:
+ self.assertEqual(response.status, 200)
+ self.assertEqual(self.engine.prompts[-1], render_chat_glm53(messages))
+ self.assertTrue(self.engine.prompts[-1].endswith("<|assistant|><think>"))
+
+ with patch.dict(os.environ, {"COLI_CONTINUE_ASSISTANT": "1"}):
+ with self.post(self.base_body(messages=messages)) as response:
+ self.assertEqual(response.status, 200)
+ self.assertEqual(self.engine.prompts[-1],
+ render_chat_glm53(messages, add_generation_prompt=False))
+ self.assertTrue(self.engine.prompts[-1].endswith("The capital is"))
+
def test_each_architecture_receives_its_native_chat_prompt(self):
messages = [{"role": "user", "content": "Hi"}]
renderers = {
diff --git a/c/tests/test_glm53_chat_template.py b/c/tests/test_glm53_chat_template.py
index 066bff6e..079bf5f1 100644
--- a/c/tests/test_glm53_chat_template.py
+++ b/c/tests/test_glm53_chat_template.py
@@ -13,7 +13,14 @@ con due, con una chiamata e il suo risultato, e con i livelli di ragionamento.
Serve chat_template.jinja del checkpoint. Se non c'e' il test si dichiara
saltato invece di passare: un test che non ha trovato il suo riferimento non ha
-verificato niente, e dirlo verde sarebbe peggio che non averlo.
+verificato niente, e dirlo verde sarebbe peggio che non averlo -- percio' un
+salto esce con codice 2, distinto dallo 0 di un confronto riuscito.
+
+RIFERIMENTO (scaricato 2026-09-10):
+ repo zai-org/GLM-5.3-Flash
+ file chat_template.jinja
+ sha256 34d5ee66b12fa6446cdae131c352b8f68cd85369e0e6fda115583805fada3891
+ hf download zai-org/GLM-5.3-Flash chat_template.jinja
USO:
python3 tests/test_glm53_chat_template.py --template PATH/chat_template.jinja
@@ -71,7 +78,8 @@ CASES = {
EFFORTS = {"low": "low", "high": "high", "xhigh": None, None: None}
-def reference(template_text, *, messages, tools=None, reasoning_effort=None):
+def reference(template_text, *, messages, tools=None, reasoning_effort=None,
+ add_generation_prompt=True):
import jinja2
environment = jinja2.Environment(trim_blocks=False, lstrip_blocks=False,
extensions=["jinja2.ext.loopcontrols"])
@@ -81,7 +89,7 @@ def reference(template_text, *, messages, tools=None, reasoning_effort=None):
environment.filters["tojson"] = (
lambda value, ensure_ascii=False, **kw: json.dumps(value, ensure_ascii=ensure_ascii))
rendered = environment.from_string(template_text)
- arguments = {"messages": messages, "add_generation_prompt": True}
+ arguments = {"messages": messages, "add_generation_prompt": add_generation_prompt}
if tools:
arguments["tools"] = tools
if reasoning_effort:
@@ -97,12 +105,12 @@ def main() -> int:
if not arguments.template.exists():
print(f"SKIP: manca {arguments.template}; il riferimento non c'e' e "
f"questo test non ha verificato nulla")
- return 0
+ return 2 # salto != successo (vedi docstring)
try:
import jinja2 # noqa: F401
except ImportError:
print("SKIP: jinja2 non installato; senza non c'e' riferimento")
- return 0
+ return 2 # salto != successo (vedi docstring)
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import openai_server
@@ -166,9 +174,55 @@ def main() -> int:
return 1
checked += 1
+ # Prosecuzione: l'ultimo turno assistant e' da CONTINUARE, non uno gia' finito. Nel
+ # template e' add_generation_prompt=False -- l'altro ramo dell'`if` che il gateway
+ # ha sempre fissato a True -- e il riferimento qui e' quel ramo reso con jinja2,
+ # non una forma scritta a mano. Il prompt finisce dentro il turno, sull'apertura
+ # del client, e non su un nuovo <|assistant|><think>.
+ #
+ # Nota su #1327: il blocco chiuso <think></think> in fondo al prompt e' fuori
+ # distribuzione, e infatti resta vietato -- ma la posizione qui e' un'altra,
+ # <think></think> seguito da contenuto vero, che e' la forma che il template
+ # scrive davanti a ogni turno passato. Per questo una prosecuzione vuota e'
+ # rifiutata a monte (tests/test_openai_server.py, TrailingAssistantTurnTest).
+ aperto = [{"role": "user", "content": "capitale della Francia?"},
+ {"role": "assistant", "content": "La capitale e'"}]
+ produced = openai_server.render_chat_for_arch(aperto, enable_thinking=True,
+ add_generation_prompt=False)
+ expected = reference(template_text, messages=aperto, add_generation_prompt=False)
+ if produced != expected:
+ print("FAIL prosecuzione: il turno aperto non rende come il template con "
+ "add_generation_prompt=False")
+ for position, (left, right) in enumerate(zip(produced, expected)):
+ if left != right:
+ start = max(0, position - 40)
+ print(f" primo scostamento a {position}")
+ print(f" gateway: ...{produced[start:position + 40]!r}")
+ print(f" template: ...{expected[start:position + 40]!r}")
+ break
+ else:
+ print(f" lunghezze diverse: {len(produced)} contro {len(expected)}")
+ return 1
+ if produced.endswith("<|assistant|><think>"):
+ print("FAIL prosecuzione: il prompt finisce su un nuovo turno invece di proseguire "
+ "quello del client")
+ return 1
+ if not produced.endswith("La capitale e'"):
+ print(f"FAIL prosecuzione: il prompt non finisce sull'apertura del client: "
+ f"{produced[-60:]!r}")
+ return 1
+ # Controllo negativo: col ramo True lo stesso scambio DEVE finire sulla cue, o il
+ # confronto qui sopra non starebbe distinguendo niente.
+ if not openai_server.render_chat_for_arch(
+ aperto, enable_thinking=True).endswith("<|assistant|><think>"):
+ print("FAIL prosecuzione: il ramo normale non emette piu' il prompt di generazione")
+ return 1
+ checked += 1
+
print(f"PASS GLM-5.3 chat template: {checked} rese identiche a "
f"chat_template.jinja, strumenti e livelli di ragionamento compresi, "
- f"piu' il livello minimo, che rende come il template con effort low")
+ f"piu' il livello minimo, che rende come il template con effort low, "
+ f"piu' il turno aperto (add_generation_prompt=False)")
return 0
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index 75e691e7..51428bbe 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -21,9 +21,11 @@ from openai_server import (APIError, APIHandler, APIServer, ClientCancelled,
_engine_error, _image_bytes_from_url, cap_for_arch, conversation_cache_slot, model_arch,
generation_options, parse_tool_calls, parse_dsv4_tool_calls,
parse_arch_tool_calls, parse_k3_tool_calls, parse_qwen38_tool_calls,
- read_engine_turn, render_chat, render_chat_kimi, render_chat_olmoe,
+ read_engine_turn, render_chat, render_chat_glm53, render_chat_kimi,
+ render_chat_olmoe,
render_chat_qwen38, render_chat_v4, _dsv4_tool_calls, serve,
- split_thinking_reply,
+ resolve_generation_prompt, split_thinking_reply,
+ starts_in_reasoning,
stop_policy, tune_child_env)
@@ -2054,6 +2056,137 @@ class ToolChoiceTest(unittest.TestCase):
generation_options({"messages": [], "tool_choice": "required"}, 128)
+class TrailingAssistantTurnTest(unittest.TestCase):
+ """A trailing `assistant` message is a turn to CONTINUE, not one already finished.
+
+ The gateway used to fold it into a completed turn and append a fresh generation cue,
+ so the model wrote a second assistant turn and the client's opening was dropped. These
+ pin what replaced that: the switch that turns continuation on, what the prompt looks
+ like when it is on, and what is refused rather than silently reshaped.
+
+ The switch is COLI_CONTINUE_ASSISTANT and not a request field on purpose -- a body
+ extension would only be reachable by hand-written JSON, and the clients that want this
+ send a message list and nothing else.
+ """
+
+ OPEN_TURN = [{"role": "user", "content": "capitale della Francia?"},
+ {"role": "assistant", "content": "La capitale e'"}]
+
+ @staticmethod
+ def on():
+ return patch.dict(os.environ, {"COLI_CONTINUE_ASSISTANT": "1"})
+
+ @staticmethod
+ def off():
+ return patch.dict(os.environ, {"COLI_CONTINUE_ASSISTANT": "0"})
+
+ def test_switch_on_leaves_the_turn_open(self):
+ with self.on(), patch("openai_server.ARCH", "glm53"):
+ self.assertFalse(resolve_generation_prompt(self.OPEN_TURN, {}))
+ prompt = render_chat_glm53(self.OPEN_TURN, enable_thinking=True,
+ add_generation_prompt=False)
+ # ends INSIDE the turn, on the client's opening -- no new cue after it
+ self.assertTrue(prompt.endswith("La capitale e'"), prompt[-60:])
+ self.assertFalse(prompt.endswith("<|assistant|><think>"))
+ self.assertEqual(prompt.count("<|assistant|>"), 1)
+
+ def test_switch_off_changes_nothing(self):
+ """The default deployment must behave exactly as it did before this existed."""
+ with self.off(), patch("openai_server.ARCH", "glm53"):
+ self.assertTrue(resolve_generation_prompt(self.OPEN_TURN, {}))
+ self.assertEqual(render_chat_glm53(self.OPEN_TURN, enable_thinking=True),
+ render_chat_glm53(self.OPEN_TURN, enable_thinking=True,
+ add_generation_prompt=True))
+ self.assertTrue(render_chat_glm53(self.OPEN_TURN,
+ enable_thinking=True).endswith("<|assistant|><think>"))
+
+ def test_no_trailing_assistant_turn_is_untouched_either_way(self):
+ messages = [{"role": "user", "content": "a"}]
+ for switch in (self.on(), self.off()):
+ with switch, patch("openai_server.ARCH", "glm53"):
+ self.assertTrue(resolve_generation_prompt(messages, {}))
+
+ def test_rejects_trailing_whitespace(self):
+ """The template strips it, so the model would resume from different bytes than the
+ ones sent -- the same reason Anthropic's own prefill validator refuses it."""
+ messages = [{"role": "user", "content": "a"},
+ {"role": "assistant", "content": "La capitale e' "}]
+ with self.on(), patch("openai_server.ARCH", "glm53"):
+ with self.assertRaises(APIError):
+ resolve_generation_prompt(messages, {})
+
+ def test_rejects_an_empty_continuation(self):
+ """An empty one ends the prompt on a closed, empty <think></think>: the
+ out-of-distribution position #1327 removed."""
+ for empty in ("", " ", None):
+ messages = [{"role": "user", "content": "a"},
+ {"role": "assistant", "content": empty}]
+ with self.on(), patch("openai_server.ARCH", "glm53"):
+ with self.assertRaises(APIError):
+ resolve_generation_prompt(messages, {})
+
+ def test_rejects_tools_and_tool_calls(self):
+ with self.on(), patch("openai_server.ARCH", "glm53"):
+ with self.assertRaises(APIError):
+ resolve_generation_prompt(self.OPEN_TURN, {"tools": ORDER_TOOL})
+ calls = [{"role": "user", "content": "a"},
+ {"role": "assistant", "content": "x", "tool_calls": [
+ {"type": "function", "function": {"name": "f", "arguments": "{}"}}]}]
+ with self.assertRaises(APIError):
+ resolve_generation_prompt(calls, {})
+
+ def test_rejects_families_without_an_open_turn_shape(self):
+ """Every family needs its own, derived from its own template. Until then, say so
+ instead of appending a cue to a turn the client meant to be continued."""
+ for arch in ("glm", "deepseek_v4", "kimi", "qwen38", "olmoe", "inkling", "qwen36"):
+ with self.on(), patch("openai_server.ARCH", arch):
+ with self.assertRaises(APIError):
+ resolve_generation_prompt(self.OPEN_TURN, {})
+
+ def test_splitter_starts_in_content_mode_on_a_continued_turn(self):
+ """Measured glm53 int4, CPU: content '' with reasoning_chars 10 and 109,
+ clean stop, and a byte-correct open turn on the wire. The model was fine; the
+ splitter was primed from enable_thinking alone, so it waited for a </think> the
+ prompt had already passed and filed the whole answer as reasoning.
+
+ starts_in_reasoning's own docstring is about exactly this invariant -- a continued
+ turn is the third state it did not model. Nothing else in the suite covers it:
+ every other thinking test runs against a prompt with the generation cue appended,
+ where enable_thinking really does say where the block was left.
+
+ Pinned to glm53 because that is the only family the switch serves. Left on the
+ module default (ARCH = "glm") this exercised the continuation path on a family
+ resolve_generation_prompt refuses, and passed for the wrong reason.
+
+ The family rule itself -- whether a NEW turn starts inside the block -- belongs to
+ #1278 and is pinned by its own test; asserted here it would only duplicate it. What
+ this test owns is the axis crossing it: a continued turn opens no block, whatever
+ the family rule says."""
+ with patch("openai_server.ARCH", "glm53"):
+ self.assertTrue(starts_in_reasoning(True)) # cue: block left open
+ self.assertFalse(starts_in_reasoning(True, add_generation_prompt=False))
+ self.assertFalse(starts_in_reasoning(False, add_generation_prompt=False))
+
+ # what the engine actually returns after "...The capital of France is": no
+ # markers, because the turn's <think></think> is already behind it in the prompt
+ emitted = " Paris."
+ reasoning, answer = split_thinking_reply(emitted, enable_thinking=True,
+ add_generation_prompt=False)
+ self.assertEqual(answer, emitted)
+ self.assertEqual(reasoning, "")
+ # and the bug it replaces, so this test fails if the priming is ever reverted
+ reasoning, answer = split_thinking_reply(emitted, enable_thinking=True,
+ add_generation_prompt=True)
+ self.assertEqual(answer, "")
+ self.assertEqual(reasoning, emitted)
+
+ def test_rejects_a_continuation_with_nothing_to_continue_from(self):
+ lone = [{"role": "assistant", "content": "La capitale e'"}]
+ with self.on(), patch("openai_server.ARCH", "glm53"):
+ with self.assertRaises(APIError):
+ resolve_generation_prompt(lone, {})
+
+
class AllowedHostsTest(unittest.TestCase):
"""#597: the DNS-rebinding guard must accept operator-trusted reverse-proxy
Host values, while still rejecting everything else by default."""
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index b32b2920..befe5a03 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -465,6 +465,7 @@ These are read by the Python programs (not the `glm` engine), so they don't appe
| `COLI_DEBUG` | `0` (off) | Tee the engine transaction to stderr, by level. **`1`** = decoded model output stream only (byte-by-byte, on both the tool-call and plain paths). **`2`** = both sides β€” the fully-rendered prompt the engine received *and* the output, bracketed and correlated by request id, so stderr reads as the whole conversation. Invaluable for seeing what the model received vs. emitted during an OpenCode session. |
| `COLI_TOOL_SALVAGE` | `0` (off) | Opt-in de-mangler: reconstruct a malformed int4 tool call by mapping its lone payload onto the tool's primary parameter. Never rewrites well-formed output; recommended for int4 deployments. |
| `COLI_THINK` | `0` (off) | Make thinking the default when the client sends *neither* `reasoning_effort` nor `enable_thinking`. Any explicit client value still wins. |
+| `COLI_CONTINUE_ASSISTANT` | `0` (off) | On the OpenAI- and Anthropic-compatible chat endpoints, continue a trailing `assistant` message instead of opening a new turn β€” the checkpoint's own template with `add_generation_prompt=False`. `glm53` only; refused with `tools`, and the turn must carry text not ending in whitespace. Off preserves each endpoint's existing behavior. Unrelated to `COLI_PREFILL_CHUNK`, which is the compute phase. |
| `COLI_MODEL` | unset | Default model directory (fallback for `--model`). |
| `COLI_MODEL_ID` | `glm-5.2-colibri` | Model id reported by the API. |
| `COLI_API_KEY` | unset | Required bearer token for the server. |
diff --git a/docs/api.md b/docs/api.md
index 76f39fac..fa7298c7 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -26,7 +26,33 @@ Implemented endpoints are `GET /v1/models`, `GET /v1/models/{model}`,
completion requests support JSON responses, SSE streaming, usage counts,
`max_tokens`/`max_completion_tokens`, `temperature`, `top_p`, and up to four
custom `stop` sequences. Stop sequences are removed from the response and end
-generation early in both JSON and streaming modes. The extension
+generation early in both JSON and streaming modes.
+
+The server-side switch `COLI_CONTINUE_ASSISTANT=1` makes a trailing `assistant`
+message *continue* that turn instead of starting a new one: the prompt ends
+inside it, which is the official template rendered with
+`add_generation_prompt=False`. There is no request field for this β€” a trailing
+assistant turn already says "continue me", and a body extension would only be
+reachable by hand-written JSON rather than from the clients that want it.
+Implemented for the `glm53` engine only (every family needs an open-turn shape
+derived from its own template) and refused together with `tools`, because the
+tool-call parsers read an assistant turn from its start. The continuation must
+carry text and must not end with whitespace: the template strips trailing
+whitespace, so the model would resume from different bytes than the ones sent.
+Off by default, and off means exactly today's behaviour.
+
+A continuation resumes from the exact bytes you send, which makes the split
+point part of the prompt. Splitting mid-word puts the model at a token boundary
+it would not have produced itself, and the first generated token is conditioned
+on that split: measured on GLM-5.3-Flash, `The capital of France is Par`
+completes to `ParΓ­s`, not `Paris` β€” deterministically, across every effort level
+and both endpoints. The continuation is real (the model finished the partial
+word rather than restarting, which is the behaviour this feature exists for);
+the spelling is an artefact of where the split fell. This is inherent to
+resuming from an arbitrary byte offset rather than specific to this engine, and
+it is the same hazard as the trailing whitespace above β€” in the one form that
+cannot be refused, because splitting mid-word is sometimes exactly what the
+caller wants. Split at a token-ish boundary when the spelling matters. The extension
`x_colibri_ignore_leading_stop: true` discards leading stop sequences until
the first non-whitespace response content, which is useful for local templates
that occasionally emit a role marker before the answer; strict OpenAI stop
@@ -132,11 +158,15 @@ tool declarations and choices explicitly instead of feeding another
architecture's markers to an incompatible tokenizer.
Not supported, and refused explicitly rather than ignored: `stop_sequences`,
-`top_k`, and non-text content blocks (images, documents). Errors use Anthropic's
-own `{"type":"error","error":{...}}` envelope on this path. Architecture-local
+`top_k`, and non-text content blocks (images, documents). Errors use Anthropic's own
+`{"type":"error","error":{...}}` envelope on this path. Architecture-local
features that have not been wired to this protocol are likewise rejected with
an explicit error.
+A trailing `assistant` message keeps this endpoint's existing behavior while
+`COLI_CONTINUE_ASSISTANT` is off. With the switch on, both the Anthropic- and
+OpenAI-compatible endpoints continue that turn on `glm53`.
+
> The prefill warning below applies here too, and applies *hardest* to Claude Code:
> its system prompt and tool catalog are large, and on a disk-streaming CPU path
> that is a long silent wait before the first token. Read it before you connect.