Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat: support patient native stop handling
Bill Herald37c8fac9b5bd
diff --git a/c/openai_server.py b/c/openai_server.pyindex aa50986a..413ab2e0 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -856,27 +856,46 @@ def parse_stop_sequences(body):class StopFilter:"""Stream text without exposing a full or partial stop sequence."""- def __init__(self, sequences, emit):+ def __init__(self, sequences, emit, ignore_leading=False):self.sequences = tuple(sequences)self.emit = emit+ self.ignore_leading = ignore_leadingself.pending = ""self.matched = None+ self.useful_content_seen = False+ self.leading_matches_ignored = 0++ def _emit(self, text):+ if text:+ self.emit(text)+ if text.strip():+ self.useful_content_seen = Truedef feed(self, chunk):if self.matched is not None:returntext = self.pending + chunk- match = None- for order, sequence in enumerate(self.sequences):- offset = text.find(sequence)- candidate = (offset, order, sequence)- if offset >= 0 and (match is None or candidate[:2] < match[:2]):- match = candidate- if match is not None:- offset, _order, self.matched = match- if offset:- self.emit(text[:offset])- self.pending = ""+ self.pending = ""+ while True:+ match = None+ for order, sequence in enumerate(self.sequences):+ offset = text.find(sequence)+ candidate = (offset, order, sequence)+ if offset >= 0 and (match is None or candidate[:2] < match[:2]):+ match = candidate+ if match is None:+ break+ offset, _order, sequence = match+ prefix = text[:offset]+ if (self.ignore_leading and not self.useful_content_seen+ and not prefix.strip()):+ self.leading_matches_ignored += 1+ text = text[offset + len(sequence):]+ if not text:+ return+ continue+ self.matched = sequence+ self._emit(prefix)returnhold = 0@@ -887,12 +906,12 @@ class StopFilter:hold = sizeflush = len(text) - holdif flush:- self.emit(text[:flush])+ self._emit(text[:flush])self.pending = text[flush:]def finish(self):if self.matched is None and self.pending:- self.emit(self.pending)+ self._emit(self.pending)self.pending = ""def stopped(self):@@ -1521,6 +1540,10 @@ class APIHandler(BaseHTTPRequestHandler):# grammar payload extension would desync its stdin framing.raise APIError(400, "`response_format` grammars are not supported by the Inkling ""engine yet.", "response_format", "unsupported_parameter")+ ignore_leading_stop = body.get("x_colibri_ignore_leading_stop", False)+ if not isinstance(ignore_leading_stop, bool):+ raise APIError(400, "`x_colibri_ignore_leading_stop` must be a boolean.",+ "x_colibri_ignore_leading_stop", "invalid_value")# tools and tool_choice come from chat_completion() already processed/filteredif chat and tool_choice == "none":tools = None # client forbade tools: never surface tool_calls@@ -1547,7 +1570,7 @@ class APIHandler(BaseHTTPRequestHandler):queue_headers = {"x-colibri-queue-wait-ms": str(round(queue_wait * 1000))}if not stream:output = []- stop_filter = StopFilter(stop_sequences, output.append)+ stop_filter = StopFilter(stop_sequences, output.append, ignore_leading_stop)stats = self.server.engine.generate(prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,self.client_disconnected, grammar=grammar, stopped=stop_filter.stopped)@@ -1671,7 +1694,7 @@ class APIHandler(BaseHTTPRequestHandler):if flush:emit(sp["buf"][:flush])sp["buf"] = sp["buf"][flush:]- stop_filter = StopFilter(stop_sequences, emit_tools)+ stop_filter = StopFilter(stop_sequences, emit_tools, ignore_leading_stop)stats = self.server.engine.generate(prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,lambda: not connected, grammar=grammar, stopped=stop_filter.stopped)@@ -1690,7 +1713,7 @@ class APIHandler(BaseHTTPRequestHandler):if dbg_echo:sys.stderr.write(chunk); sys.stderr.flush()(splitter.feed if splitter else emit)(chunk)- stop_filter = StopFilter(stop_sequences, emit_plain)+ stop_filter = StopFilter(stop_sequences, emit_plain, ignore_leading_stop)stats = self.server.engine.generate(prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,lambda: not connected, grammar=grammar, stopped=stop_filter.stopped)diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex 24e6f47e..eb40ce44 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -140,6 +140,32 @@ class StopFilterTest(unittest.TestCase):stop_filter.finish()self.assertEqual("".join(output), "answer ST")+ def test_optional_patient_mode_ignores_only_leading_matches(self):+ output = []+ stop_filter = StopFilter(("<|user|>",), output.append, ignore_leading=True)+ for chunk in ("<|us", "er|>answer", "<|user|>ignored"):+ stop_filter.feed(chunk)+ stop_filter.finish()+ self.assertEqual("".join(output), "answer")+ self.assertEqual(stop_filter.matched, "<|user|>")+ self.assertEqual(stop_filter.leading_matches_ignored, 1)++ def test_patient_mode_preserves_remainder_after_same_chunk_leading_match(self):+ output = []+ stop_filter = StopFilter(("STOP",), output.append, ignore_leading=True)+ stop_filter.feed("STOPuseful STOPdiscarded")+ stop_filter.finish()+ self.assertEqual("".join(output), "useful ")+ self.assertEqual(stop_filter.matched, "STOP")++ def test_strict_mode_still_stops_on_a_leading_match(self):+ output = []+ stop_filter = StopFilter(("STOP",), output.append)+ stop_filter.feed("STOPignored")+ stop_filter.finish()+ self.assertEqual(output, [])+ self.assertEqual(stop_filter.matched, "STOP")+class ProtocolTest(unittest.TestCase):def test_reads_payload_and_extended_status(self):@@ -590,6 +616,24 @@ class HTTPTest(unittest.TestCase):self.assertEqual(body["choices"][0]["finish_reason"], "stop")self.assertEqual(self.engine.stop_requests, before + 1)+ def test_patient_stop_extension_ignores_a_leading_match(self):+ before = self.engine.stop_requests+ with self.request("/v1/chat/completions", {+ "model": "test-model", "messages": [{"role": "user", "content": "Hi"}],+ "stop": "H", "x_colibri_ignore_leading_stop": True,+ }) as response:+ body = json.load(response)+ self.assertEqual(body["choices"][0]["message"]["content"], "éllo")+ self.assertEqual(self.engine.stop_requests, before)++ def test_patient_stop_extension_requires_a_boolean(self):+ with self.assertRaises(HTTPError) as caught:+ self.request("/v1/chat/completions", {+ "model": "test-model", "messages": [{"role": "user", "content": "Hi"}],+ "stop": "H", "x_colibri_ignore_leading_stop": "yes",+ })+ self.assertEqual(caught.exception.code, 400)+def test_rejects_invalid_cache_slot(self):with self.assertRaises(HTTPError) as caught:self.request("/v1/chat/completions", {diff --git a/docs/api.md b/docs/api.mdindex f3bb16c3..80ecbbe7 100644--- a/docs/api.md+++ b/docs/api.md@@ -27,6 +27,10 @@ completion requests support JSON responses, SSE streaming, usage counts,`max_tokens`/`max_completion_tokens`, `temperature`, `top_p`, and up to fourcustom `stop` sequences. Stop sequences are removed from the response and endgeneration early in both JSON and streaming modes. The extension+`x_colibri_ignore_leading_stop: true` discards leading stop sequences until+the first non-whitespace response content, which is useful for local templates+that occasionally emit a role marker before the answer; strict OpenAI stop+behavior remains the default. The extension`enable_thinking: true` enables GLM-5.2's reasoning block; the standard`reasoning_effort` field also enables it unless set to `none`.