Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix: stop chat at implicit role boundaries
bill83fc0a71b8cb
diff --git a/c/openai_server.py b/c/openai_server.pyindex 413ab2e0..fa336fcd 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -832,6 +832,8 @@ GENERIC_JSON_GBNF = ('jws ::= ( " " | "\\t" | "\\n" | "\\r" )*\n')+DEFAULT_CHAT_STOP_SEQUENCES = ("<|user|>", "<|observation|>")+def parse_stop_sequences(body):value = body.get("stop")@@ -854,6 +856,21 @@ def parse_stop_sequences(body):return tuple(sequences)+def stop_policy(body, chat):+ sequences = parse_stop_sequences(body)+ ignore_leading = body.get("x_colibri_ignore_leading_stop", False)+ if not isinstance(ignore_leading, bool):+ raise APIError(400, "`x_colibri_ignore_leading_stop` must be a boolean.",+ "x_colibri_ignore_leading_stop", "invalid_value")+ if chat and not sequences:+ # The chat template owns these role boundaries, so generic OpenAI+ # clients should not need model-specific stop knowledge. Treat an+ # occasional leading marker patiently; client-provided stops remain+ # strict unless the extension is explicitly requested.+ return DEFAULT_CHAT_STOP_SEQUENCES, True+ return sequences, ignore_leading++class StopFilter:"""Stream text without exposing a full or partial stop sequence."""def __init__(self, sequences, emit, ignore_leading=False):@@ -1533,17 +1550,14 @@ class APIHandler(BaseHTTPRequestHandler):if dbg >= 2:sys.stderr.write(f"\n===== PROMPT [{request_id}] =====\n{prompt}\n===== OUTPUT [{request_id}] =====\n")sys.stderr.flush()- maximum, temperature, top_p, grammar, stop_sequences = generation_options(+ maximum, temperature, top_p, grammar, _requested_stop_sequences = generation_options(body, self.server.max_tokens)if grammar is not None and ARCH == "inkling":# inkling.c's serve loop speaks the 6-field SUBMIT header only; sending the# grammar payload extension would desync its stdin framing.raise APIError(400, "`response_format` grammars are not supported by the Inkling ""engine yet.", "response_format", "unsupported_parameter")- ignore_leading_stop = body.get("x_colibri_ignore_leading_stop", False)- if not isinstance(ignore_leading_stop, bool):- raise APIError(400, "`x_colibri_ignore_leading_stop` must be a boolean.",- "x_colibri_ignore_leading_stop", "invalid_value")+ stop_sequences, ignore_leading_stop = stop_policy(body, chat)# tools and tool_choice come from chat_completion() already processed/filteredif chat and tool_choice == "none":tools = None # client forbade tools: never surface tool_callsdiff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex eb40ce44..c3c999d1 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -10,9 +10,10 @@ from urllib.error import HTTPErrorfrom urllib.request import Request, urlopenfrom pathlib import Path-from openai_server import (APIError, APIHandler, APIServer, ClientCancelled, END, GenerationScheduler,+from openai_server import (APIError, APIHandler, APIServer, ClientCancelled,+ DEFAULT_CHAT_STOP_SEQUENCES, END, GenerationScheduler,READY, Engine, StopFilter, _engine_error, generation_options,- parse_tool_calls, read_engine_turn, render_chat, serve)+ parse_tool_calls, read_engine_turn, render_chat, serve, stop_policy)class FakeEngine:@@ -122,6 +123,16 @@ class TemplateTest(unittest.TestCase):with self.subTest(value=value), self.assertRaises(APIError):generation_options({"stop": value}, 8)+ def test_chat_defaults_role_stops_without_changing_client_or_completion_policy(self):+ self.assertEqual(stop_policy({}, True), (DEFAULT_CHAT_STOP_SEQUENCES, True))+ self.assertEqual(stop_policy({}, False), ((), False))+ self.assertEqual(stop_policy({"stop": "END"}, True), (("END",), False))+ self.assertEqual(stop_policy({+ "stop": "END", "x_colibri_ignore_leading_stop": True,+ }, True), (("END",), True))+ with self.assertRaises(APIError):+ stop_policy({"x_colibri_ignore_leading_stop": "yes"}, True)+class StopFilterTest(unittest.TestCase):def test_hides_match_split_across_chunks(self):diff --git a/docs/api.md b/docs/api.mdindex 80ecbbe7..8522f3be 100644--- a/docs/api.md+++ b/docs/api.md@@ -30,7 +30,11 @@ generation early in both JSON and streaming modes. The extension`x_colibri_ignore_leading_stop: true` discards leading stop sequences untilthe first non-whitespace response content, which is useful for local templatesthat occasionally emit a role marker before the answer; strict OpenAI stop-behavior remains the default. The extension+behavior remains the default for client-provided sequences. Chat requests with+no client `stop` automatically use the template's `<|user|>` and+`<|observation|>` role markers, patiently ignoring only leading markers; this+prevents a model-completed turn from silently generating a new user or tool+turn. Legacy completion requests receive no implicit stop sequences. The extension`enable_thinking: true` enables GLM-5.2's reasoning block; the standard`reasoning_effort` field also enables it unless set to `none`.