Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix: stop chat at implicit role boundaries

bill83fc0a71b8cb
diff --git a/c/openai_server.py b/c/openai_server.py
index 413ab2e0..fa336fcd 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -832,6 +832,8 @@ GENERIC_JSON_GBNF = (
'jws ::= ( " " | "\\t" | "\\n" | "\\r" )*\n'
)
+DEFAULT_CHAT_STOP_SEQUENCES = ("<|user|>", "<|observation|>")
+
def parse_stop_sequences(body):
value = body.get("stop")
@@ -854,6 +856,21 @@ def parse_stop_sequences(body):
return tuple(sequences)
+def stop_policy(body, chat):
+ sequences = parse_stop_sequences(body)
+ ignore_leading = body.get("x_colibri_ignore_leading_stop", False)
+ if not isinstance(ignore_leading, bool):
+ raise APIError(400, "`x_colibri_ignore_leading_stop` must be a boolean.",
+ "x_colibri_ignore_leading_stop", "invalid_value")
+ if chat and not sequences:
+ # The chat template owns these role boundaries, so generic OpenAI
+ # clients should not need model-specific stop knowledge. Treat an
+ # occasional leading marker patiently; client-provided stops remain
+ # strict unless the extension is explicitly requested.
+ return DEFAULT_CHAT_STOP_SEQUENCES, True
+ return sequences, ignore_leading
+
+
class StopFilter:
"""Stream text without exposing a full or partial stop sequence."""
def __init__(self, sequences, emit, ignore_leading=False):
@@ -1533,17 +1550,14 @@ class APIHandler(BaseHTTPRequestHandler):
if dbg >= 2:
sys.stderr.write(f"\n===== PROMPT [{request_id}] =====\n{prompt}\n===== OUTPUT [{request_id}] =====\n")
sys.stderr.flush()
- maximum, temperature, top_p, grammar, stop_sequences = generation_options(
+ maximum, temperature, top_p, grammar, _requested_stop_sequences = generation_options(
body, self.server.max_tokens)
if grammar is not None and ARCH == "inkling":
# inkling.c's serve loop speaks the 6-field SUBMIT header only; sending the
# grammar payload extension would desync its stdin framing.
raise APIError(400, "`response_format` grammars are not supported by the Inkling "
"engine yet.", "response_format", "unsupported_parameter")
- ignore_leading_stop = body.get("x_colibri_ignore_leading_stop", False)
- if not isinstance(ignore_leading_stop, bool):
- raise APIError(400, "`x_colibri_ignore_leading_stop` must be a boolean.",
- "x_colibri_ignore_leading_stop", "invalid_value")
+ stop_sequences, ignore_leading_stop = stop_policy(body, chat)
# tools and tool_choice come from chat_completion() already processed/filtered
if chat and tool_choice == "none":
tools = None # client forbade tools: never surface tool_calls
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index eb40ce44..c3c999d1 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -10,9 +10,10 @@ from urllib.error import HTTPError
from urllib.request import Request, urlopen
from pathlib import Path
-from openai_server import (APIError, APIHandler, APIServer, ClientCancelled, END, GenerationScheduler,
+from openai_server import (APIError, APIHandler, APIServer, ClientCancelled,
+ DEFAULT_CHAT_STOP_SEQUENCES, END, GenerationScheduler,
READY, Engine, StopFilter, _engine_error, generation_options,
- parse_tool_calls, read_engine_turn, render_chat, serve)
+ parse_tool_calls, read_engine_turn, render_chat, serve, stop_policy)
class FakeEngine:
@@ -122,6 +123,16 @@ class TemplateTest(unittest.TestCase):
with self.subTest(value=value), self.assertRaises(APIError):
generation_options({"stop": value}, 8)
+ def test_chat_defaults_role_stops_without_changing_client_or_completion_policy(self):
+ self.assertEqual(stop_policy({}, True), (DEFAULT_CHAT_STOP_SEQUENCES, True))
+ self.assertEqual(stop_policy({}, False), ((), False))
+ self.assertEqual(stop_policy({"stop": "END"}, True), (("END",), False))
+ self.assertEqual(stop_policy({
+ "stop": "END", "x_colibri_ignore_leading_stop": True,
+ }, True), (("END",), True))
+ with self.assertRaises(APIError):
+ stop_policy({"x_colibri_ignore_leading_stop": "yes"}, True)
+
class StopFilterTest(unittest.TestCase):
def test_hides_match_split_across_chunks(self):
diff --git a/docs/api.md b/docs/api.md
index 80ecbbe7..8522f3be 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -30,7 +30,11 @@ generation early in both JSON and streaming modes. The extension
`x_colibri_ignore_leading_stop: true` discards leading stop sequences until
the first non-whitespace response content, which is useful for local templates
that occasionally emit a role marker before the answer; strict OpenAI stop
-behavior remains the default. The extension
+behavior remains the default for client-provided sequences. Chat requests with
+no client `stop` automatically use the template's `<|user|>` and
+`<|observation|>` role markers, patiently ignoring only leading markers; this
+prevents a model-completed turn from silently generating a new user or tool
+turn. Legacy completion requests receive no implicit stop sequences. The extension
`enable_thinking: true` enables GLM-5.2's reasoning block; the standard
`reasoning_effort` field also enables it unless set to `none`.