Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat: support OpenAI stop sequences
Bill Heraldcc5df232f763
diff --git a/c/colibri.c b/c/colibri.cindex 82eeac21..03d08b32 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -5471,6 +5471,16 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, GrDraft *gchar *line=NULL; size_t cap=0; ssize_t nr=getline(&line,&cap,stdin);if(nr<0){ free(line); return -1; }if(nr && line[nr-1]=='\n') line[--nr]=0;+ if(!strncmp(line,"STOP ",5)){+ unsigned long long id=0; char tail;+ if(sscanf(line+5,"%llu %c",&id,&tail)!=1 || id==0){+ printf("ERROR 0 BAD_REQUEST\n"); fflush(stdout); free(line); return 0;+ }+ for(int i=0;i<nctx;i++) if(req[i].active && req[i].id==id){+ mux_done(m,&ctx[i],&req[i]); free(line); return 0;+ }+ printf("ERROR %llu NOT_FOUND\n",id); fflush(stdout); free(line); return 0;+ }if(!strncmp(line,"CANCEL ",7)){unsigned long long id=0; char tail;if(sscanf(line+7,"%llu %c",&id,&tail)!=1 || id==0){diff --git a/c/openai_server.py b/c/openai_server.pyindex 074f5153..aa50986a 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -832,6 +832,72 @@ GENERIC_JSON_GBNF = ('jws ::= ( " " | "\\t" | "\\n" | "\\r" )*\n')++def parse_stop_sequences(body):+ value = body.get("stop")+ if value is None:+ return ()+ if isinstance(value, str):+ sequences = [value]+ elif isinstance(value, list):+ sequences = value+ else:+ raise APIError(400, "`stop` must be a string or an array of strings.",+ "stop", "invalid_value")+ if not 1 <= len(sequences) <= 4:+ raise APIError(400, "`stop` must contain between 1 and 4 sequences.",+ "stop", "invalid_value")+ for index, sequence in enumerate(sequences):+ if not isinstance(sequence, str) or not sequence:+ raise APIError(400, "Each `stop` sequence must be a non-empty string.",+ f"stop.{index}", "invalid_value")+ return tuple(sequences)+++class StopFilter:+ """Stream text without exposing a full or partial stop sequence."""+ def __init__(self, sequences, emit):+ self.sequences = tuple(sequences)+ self.emit = emit+ self.pending = ""+ self.matched = None++ def feed(self, chunk):+ if self.matched is not None:+ return+ text = self.pending + chunk+ match = None+ for order, sequence in enumerate(self.sequences):+ offset = text.find(sequence)+ candidate = (offset, order, sequence)+ if offset >= 0 and (match is None or candidate[:2] < match[:2]):+ match = candidate+ if match is not None:+ offset, _order, self.matched = match+ if offset:+ self.emit(text[:offset])+ self.pending = ""+ return++ hold = 0+ maximum = min(len(text), max((len(s) - 1 for s in self.sequences), default=0))+ for size in range(1, maximum + 1):+ suffix = text[-size:]+ if any(sequence.startswith(suffix) for sequence in self.sequences):+ hold = size+ flush = len(text) - hold+ if flush:+ self.emit(text[:flush])+ self.pending = text[flush:]++ def finish(self):+ if self.matched is None and self.pending:+ self.emit(self.pending)+ self.pending = ""++ def stopped(self):+ return self.matched is not None+def generation_options(body, limit):if body.get("n", 1) != 1:raise APIError(400, "Colibri currently supports `n=1` only.", "n", "unsupported_value")@@ -878,8 +944,7 @@ def generation_options(body, limit):"tool_choice", "invalid_value")if choice != "none" and not (body.get("tools") or body.get("functions")):raise APIError(400, "`tool_choice` requires `tools`.", "tool_choice", "invalid_value")- if body.get("stop") is not None:- raise APIError(400, "Custom stop sequences are not supported yet.", "stop", "unsupported_parameter")+ stop_sequences = parse_stop_sequences(body)if body.get("logprobs"):raise APIError(400, "Log probabilities are not supported yet.", "logprobs", "unsupported_parameter")if body.get("frequency_penalty", 0) or body.get("presence_penalty", 0):@@ -944,7 +1009,7 @@ def generation_options(body, limit):if (isinstance(top_p, bool) or not isinstance(top_p, (int, float)) ornot math.isfinite(top_p) or not 0 < top_p <= 1):raise APIError(400, "`top_p` must be greater than 0 and at most 1.", "top_p")- return maximum, float(temperature), float(top_p), grammar+ return maximum, float(temperature), float(top_p), grammar, stop_sequencesdef read_engine_turn(stream, sentinel, on_bytes):@@ -1108,7 +1173,7 @@ class Engine:self._fail_pending(error)def generate(self, prompt, max_tokens, temperature, top_p, on_text, cache_slot=0,- cancelled=None, grammar=None):+ cancelled=None, grammar=None, stopped=None):if isinstance(cache_slot, bool) or not isinstance(cache_slot, int) or not 0 <= cache_slot < self.kv_slots:raise APIError(400, "Invalid cache slot.", "cache_slot")payload = prompt.encode("utf-8")@@ -1150,12 +1215,18 @@ class Engine:raisecancel_sent = False+ stop_sent = Falsewhile True:kind, value = events.get()if kind == "data":- if not cancel_sent:+ if not cancel_sent and not stop_sent:decode(value)- if cancelled and cancelled():+ if stopped and stopped():+ stop_sent = True+ with self.write_lock:+ self.process.stdin.write(f"STOP {request_id}\n".encode())+ self.process.stdin.flush()+ elif cancelled and cancelled():cancel_sent = Truewith self.write_lock:self.process.stdin.write(f"CANCEL {request_id}\n".encode())@@ -1443,7 +1514,8 @@ class APIHandler(BaseHTTPRequestHandler):if dbg >= 2:sys.stderr.write(f"\n===== PROMPT [{request_id}] =====\n{prompt}\n===== OUTPUT [{request_id}] =====\n")sys.stderr.flush()- maximum, temperature, top_p, grammar = generation_options(body, self.server.max_tokens)+ maximum, temperature, top_p, grammar, stop_sequences = generation_options(+ body, self.server.max_tokens)if grammar is not None and ARCH == "inkling":# inkling.c's serve loop speaks the 6-field SUBMIT header only; sending the# grammar payload extension would desync its stdin framing.@@ -1475,9 +1547,11 @@ class APIHandler(BaseHTTPRequestHandler):queue_headers = {"x-colibri-queue-wait-ms": str(round(queue_wait * 1000))}if not stream:output = []+ stop_filter = StopFilter(stop_sequences, output.append)stats = self.server.engine.generate(- prompt, maximum, temperature, top_p, output.append, cache_slot,- self.client_disconnected, grammar=grammar)+ prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,+ self.client_disconnected, grammar=grammar, stopped=stop_filter.stopped)+ stop_filter.finish()text = "".join(output)reasoning = ""if ARCH == "inkling":@@ -1597,9 +1671,11 @@ class APIHandler(BaseHTTPRequestHandler):if flush:emit(sp["buf"][:flush])sp["buf"] = sp["buf"][flush:]+ stop_filter = StopFilter(stop_sequences, emit_tools)stats = self.server.engine.generate(- prompt, maximum, temperature, top_p, emit_tools, cache_slot,- lambda: not connected, grammar=grammar)+ prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,+ lambda: not connected, grammar=grammar, stopped=stop_filter.stopped)+ stop_filter.finish()if not sp["tool"] and sp["buf"]:emit(sp["buf"]) # no tool call happened: flush held tail_content, calls = parse_tool_calls("".join(raw), tools)@@ -1614,9 +1690,11 @@ class APIHandler(BaseHTTPRequestHandler):if dbg_echo:sys.stderr.write(chunk); sys.stderr.flush()(splitter.feed if splitter else emit)(chunk)+ stop_filter = StopFilter(stop_sequences, emit_plain)stats = self.server.engine.generate(- prompt, maximum, temperature, top_p, emit_plain, cache_slot,- lambda: not connected, grammar=grammar)+ prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,+ lambda: not connected, grammar=grammar, stopped=stop_filter.stopped)+ stop_filter.finish()if splitter:splitter.close()finish = "length" if stats["length_limited"] else "stop"@@ -1710,7 +1788,8 @@ class APIHandler(BaseHTTPRequestHandler):self.anthropic_generation(translated, prompt, request_id, tools, enable_thinking)def anthropic_generation(self, body, prompt, request_id, tools, enable_thinking):- maximum, temperature, top_p, grammar = generation_options(body, self.server.max_tokens)+ maximum, temperature, top_p, grammar, _stop_sequences = generation_options(+ body, self.server.max_tokens)cache_slot = body.get("cache_slot")if (cache_slot is not None and(isinstance(cache_slot, bool) or not isinstance(cache_slot, int) ordiff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex 98da4c18..24e6f47e 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -11,19 +11,23 @@ from urllib.request import Request, urlopenfrom pathlib import Pathfrom openai_server import (APIError, APIHandler, APIServer, ClientCancelled, END, GenerationScheduler,- READY, Engine, _engine_error, generation_options, parse_tool_calls,- read_engine_turn, render_chat, serve)+ READY, Engine, StopFilter, _engine_error, generation_options,+ parse_tool_calls, read_engine_turn, render_chat, serve)class FakeEngine:def __init__(self):self.calls = []+ self.stop_requests = 0def generate(self, prompt, maximum, temperature, top_p, on_text, cache_slot=0,- cancelled=None, grammar=None):+ cancelled=None, grammar=None, stopped=None):self.calls.append((prompt, maximum, temperature, top_p, cache_slot, grammar))- on_text("Hé")- on_text("llo")+ for chunk in ("Hé", "llo"):+ on_text(chunk)+ if stopped and stopped():+ self.stop_requests += 1+ breakreturn {"prompt_tokens": 7, "completion_tokens": 2, "length_limited": False}@@ -34,11 +38,11 @@ class BlockingEngine(FakeEngine):self.release = threading.Event()def generate(self, prompt, maximum, temperature, top_p, on_text, cache_slot=0,- cancelled=None, grammar=None):+ cancelled=None, grammar=None, stopped=None):self.entered.set()self.release.wait(2)return super().generate(prompt, maximum, temperature, top_p, on_text, cache_slot,- cancelled)+ cancelled, grammar, stopped)class TemplateTest(unittest.TestCase):@@ -71,11 +75,11 @@ class TemplateTest(unittest.TestCase):def test_validates_generation_limits(self):self.assertEqual(generation_options({"max_tokens": 4, "temperature": 0, "top_p": 1}, 8),- (4, 0.0, 1.0, None))+ (4, 0.0, 1.0, None, ()))# max_tokens above the server cap is clamped, not rejected (#260): OpenAI# clients default to large values; erroring breaks them.self.assertEqual(generation_options({"max_tokens": 9, "temperature": 0, "top_p": 1}, 8),- (8, 0.0, 1.0, None))+ (8, 0.0, 1.0, None, ()))# non-positive / non-int max_tokens is still a hard errorwith self.assertRaises(APIError):generation_options({"max_tokens": 0}, 8)@@ -84,7 +88,7 @@ class TemplateTest(unittest.TestCase):with self.assertRaises(APIError):generation_options({"top_p": math.inf}, 8)self.assertEqual(generation_options({"temperature": None, "top_p": None}, 8),- (8, 0.7, 0.9, None))+ (8, 0.7, 0.9, None, ()))# response_format -> grammar plumbing (draft source, never a constraint)opts = generation_options({"max_tokens": 4, "response_format": {"type": "json_object"}}, 8)self.assertIn("root ::=", opts[3])@@ -110,6 +114,32 @@ class TemplateTest(unittest.TestCase):opts = generation_options({"response_format": {"type": "gbnf", "grammar": "not a grammar ::="}}, 8)self.assertEqual(opts[3], "not a grammar ::=")+ def test_validates_stop_sequences(self):+ self.assertEqual(generation_options({"stop": "END"}, 8)[4], ("END",))+ self.assertEqual(generation_options({"stop": ["ONE", "TWO"]}, 8)[4],+ ("ONE", "TWO"))+ for value in ("", [], [""], ["1", "2", "3", "4", "5"], 7, ["ok", 7]):+ with self.subTest(value=value), self.assertRaises(APIError):+ generation_options({"stop": value}, 8)+++class StopFilterTest(unittest.TestCase):+ def test_hides_match_split_across_chunks(self):+ output = []+ stop_filter = StopFilter(("STOP",), output.append)+ for chunk in ("answer S", "TO", "Pignored"):+ stop_filter.feed(chunk)+ stop_filter.finish()+ self.assertEqual("".join(output), "answer ")+ self.assertEqual(stop_filter.matched, "STOP")++ def test_flushes_partial_prefix_when_generation_finishes(self):+ output = []+ stop_filter = StopFilter(("STOP",), output.append)+ stop_filter.feed("answer ST")+ stop_filter.finish()+ self.assertEqual("".join(output), "answer ST")+class ProtocolTest(unittest.TestCase):def test_reads_payload_and_extended_status(self):@@ -446,6 +476,30 @@ class DispatcherTest(unittest.TestCase):self.assertEqual(output, ["x"])self.assertEqual(process.writes[-1].split(), [b"CANCEL", request_id])+ def test_stops_generation_through_successful_done_path(self):+ request_id = None++ def respond(process, frame):+ nonlocal request_id+ fields = frame.split()+ if fields[0] == b"SUBMIT":+ request_id = fields[1]+ process.stdout.feed(b"DATA " + request_id + b" 1\nx\n")+ elif fields[0] == b"STOP":+ self.assertEqual(fields[1], request_id)+ process.stdout.feed(b"DONE " + request_id + b" STAT 1 1 0 1 2 0\n")++ process = FakeProcess(respond)+ with patch("openai_server.subprocess.Popen", return_value=process):+ engine = Engine("glm", "model")+ output = []+ stats = engine.generate("hello", 8, 0.7, 0.9, output.append,+ stopped=lambda: output == ["x"])+ engine.close()+ self.assertEqual(output, ["x"])+ self.assertEqual(stats["completion_tokens"], 1)+ self.assertEqual(process.writes[-1].split(), [b"STOP", request_id])+class HTTPTest(unittest.TestCase):@classmethod@@ -525,6 +579,17 @@ class HTTPTest(unittest.TestCase):self.assertIn("<|user|>Hi<|assistant|><think></think>", self.engine.calls[-1][0])self.assertEqual(self.engine.calls[-1][4], 1)+ def test_chat_completion_stops_across_engine_chunks(self):+ before = self.engine.stop_requests+ with self.request("/v1/chat/completions", {+ "model": "test-model", "messages": [{"role": "user", "content": "Hi"}],+ "stop": "éll",+ }) as response:+ body = json.load(response)+ self.assertEqual(body["choices"][0]["message"]["content"], "H")+ self.assertEqual(body["choices"][0]["finish_reason"], "stop")+ self.assertEqual(self.engine.stop_requests, before + 1)+def test_rejects_invalid_cache_slot(self):with self.assertRaises(HTTPError) as caught:self.request("/v1/chat/completions", {@@ -545,6 +610,21 @@ class HTTPTest(unittest.TestCase):self.assertIn('\"usage\":{\"prompt_tokens\":7,\"completion_tokens\":2,\"total_tokens\":9}', stream)self.assertTrue(stream.endswith("data: [DONE]\n\n"))+ def test_streaming_stop_never_exposes_partial_sequence(self):+ before = self.engine.stop_requests+ with self.request("/v1/chat/completions", {+ "model": "test-model", "messages": [{"role": "user", "content": "Hi"}],+ "stream": True, "stop": "éll",+ }) as response:+ raw = response.read().decode()+ payloads = [json.loads(line[6:]) for line in raw.splitlines()+ if line.startswith("data: ") and line != "data: [DONE]"]+ content = "".join((choice.get("delta") or {}).get("content", "")+ for payload in payloads for choice in payload["choices"])+ self.assertEqual(content, "H")+ self.assertEqual(payloads[-1]["choices"][0]["finish_reason"], "stop")+ self.assertEqual(self.engine.stop_requests, before + 1)+def test_legacy_completion(self):with self.request("/v1/completions", {"model": "test-model", "prompt": "Complete me", "temperature": 0,diff --git a/docs/api.md b/docs/api.mdindex 2b92bb6e..f3bb16c3 100644--- a/docs/api.md+++ b/docs/api.md@@ -24,14 +24,16 @@ curl http://127.0.0.1:8000/v1/chat/completions \Implemented endpoints are `GET /v1/models`, `GET /v1/models/{model}`,`POST /v1/chat/completions`, and legacy `POST /v1/completions`. Chat andcompletion requests support JSON responses, SSE streaming, usage counts,-`max_tokens`/`max_completion_tokens`, `temperature`, and `top_p`. The extension+`max_tokens`/`max_completion_tokens`, `temperature`, `top_p`, and up to four+custom `stop` sequences. Stop sequences are removed from the response and end+generation early in both JSON and streaming modes. The extension`enable_thinking: true` enables GLM-5.2's reasoning block; the standard`reasoning_effort` field also enables it unless set to `none`.The server is deliberately text-only and serves one generation at a time: the744B model stays in one persistent process, so concurrent HTTP requests queue-instead of loading duplicate model copies. Tools, image/audio input, custom-stop sequences, log probabilities, and token penalties return an explicit error+instead of loading duplicate model copies. Image/audio input, log probabilities,+and token penalties return an explicit errorrather than being silently ignored. The default bind address is localhost; set`COLI_API_KEY` before exposing the server beyond the machine.diff --git a/docs/serve_protocol.md b/docs/serve_protocol.mdindex b0eb61ec..8b9000b4 100644--- a/docs/serve_protocol.md+++ b/docs/serve_protocol.md@@ -33,6 +33,7 @@ recognize**; that is the protocol's forward-compatibility rule.```SUBMIT <id> <slot> <bytes> <max_tokens> <temperature> <top_p>\n<payload>\n+STOP <id>\nCANCEL <id>\n```@@ -44,6 +45,10 @@ CANCEL <id>\n- `bytes` — exact byte length of `payload` (UTF-8, may contain newlines). The enginereads exactly that many bytes after the header line, then one trailing `\n`.- `payload` — the fully rendered prompt (the server owns the chat template).+- `STOP` ends generation through the normal successful `DONE` path. Statistics,+ usage history, and KV state are persisted; the HTTP gateway uses it after a+ client-provided stop sequence matches.+- `CANCEL` aborts a request after its client disconnects and returns `CANCELLED`.- EOF on stdin = graceful shutdown: in-flight requests finish first.Prefill is serial; decode is continuously batched — every active slot contributes