Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat: support OpenAI stop sequences

Bill Heraldcc5df232f763
diff --git a/c/colibri.c b/c/colibri.c
index 82eeac21..03d08b32 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -5471,6 +5471,16 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, GrDraft *g
char *line=NULL; size_t cap=0; ssize_t nr=getline(&line,&cap,stdin);
if(nr<0){ free(line); return -1; }
if(nr && line[nr-1]=='\n') line[--nr]=0;
+ if(!strncmp(line,"STOP ",5)){
+ unsigned long long id=0; char tail;
+ if(sscanf(line+5,"%llu %c",&id,&tail)!=1 || id==0){
+ printf("ERROR 0 BAD_REQUEST\n"); fflush(stdout); free(line); return 0;
+ }
+ for(int i=0;i<nctx;i++) if(req[i].active && req[i].id==id){
+ mux_done(m,&ctx[i],&req[i]); free(line); return 0;
+ }
+ printf("ERROR %llu NOT_FOUND\n",id); fflush(stdout); free(line); return 0;
+ }
if(!strncmp(line,"CANCEL ",7)){
unsigned long long id=0; char tail;
if(sscanf(line+7,"%llu %c",&id,&tail)!=1 || id==0){
diff --git a/c/openai_server.py b/c/openai_server.py
index 074f5153..aa50986a 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -832,6 +832,72 @@ GENERIC_JSON_GBNF = (
'jws ::= ( " " | "\\t" | "\\n" | "\\r" )*\n'
)
+
+def parse_stop_sequences(body):
+ value = body.get("stop")
+ if value is None:
+ return ()
+ if isinstance(value, str):
+ sequences = [value]
+ elif isinstance(value, list):
+ sequences = value
+ else:
+ raise APIError(400, "`stop` must be a string or an array of strings.",
+ "stop", "invalid_value")
+ if not 1 <= len(sequences) <= 4:
+ raise APIError(400, "`stop` must contain between 1 and 4 sequences.",
+ "stop", "invalid_value")
+ for index, sequence in enumerate(sequences):
+ if not isinstance(sequence, str) or not sequence:
+ raise APIError(400, "Each `stop` sequence must be a non-empty string.",
+ f"stop.{index}", "invalid_value")
+ return tuple(sequences)
+
+
+class StopFilter:
+ """Stream text without exposing a full or partial stop sequence."""
+ def __init__(self, sequences, emit):
+ self.sequences = tuple(sequences)
+ self.emit = emit
+ self.pending = ""
+ self.matched = None
+
+ def feed(self, chunk):
+ if self.matched is not None:
+ return
+ text = self.pending + chunk
+ match = None
+ for order, sequence in enumerate(self.sequences):
+ offset = text.find(sequence)
+ candidate = (offset, order, sequence)
+ if offset >= 0 and (match is None or candidate[:2] < match[:2]):
+ match = candidate
+ if match is not None:
+ offset, _order, self.matched = match
+ if offset:
+ self.emit(text[:offset])
+ self.pending = ""
+ return
+
+ hold = 0
+ maximum = min(len(text), max((len(s) - 1 for s in self.sequences), default=0))
+ for size in range(1, maximum + 1):
+ suffix = text[-size:]
+ if any(sequence.startswith(suffix) for sequence in self.sequences):
+ hold = size
+ flush = len(text) - hold
+ if flush:
+ self.emit(text[:flush])
+ self.pending = text[flush:]
+
+ def finish(self):
+ if self.matched is None and self.pending:
+ self.emit(self.pending)
+ self.pending = ""
+
+ def stopped(self):
+ return self.matched is not None
+
def generation_options(body, limit):
if body.get("n", 1) != 1:
raise APIError(400, "Colibri currently supports `n=1` only.", "n", "unsupported_value")
@@ -878,8 +944,7 @@ def generation_options(body, limit):
"tool_choice", "invalid_value")
if choice != "none" and not (body.get("tools") or body.get("functions")):
raise APIError(400, "`tool_choice` requires `tools`.", "tool_choice", "invalid_value")
- if body.get("stop") is not None:
- raise APIError(400, "Custom stop sequences are not supported yet.", "stop", "unsupported_parameter")
+ stop_sequences = parse_stop_sequences(body)
if body.get("logprobs"):
raise APIError(400, "Log probabilities are not supported yet.", "logprobs", "unsupported_parameter")
if body.get("frequency_penalty", 0) or body.get("presence_penalty", 0):
@@ -944,7 +1009,7 @@ def generation_options(body, limit):
if (isinstance(top_p, bool) or not isinstance(top_p, (int, float)) or
not math.isfinite(top_p) or not 0 < top_p <= 1):
raise APIError(400, "`top_p` must be greater than 0 and at most 1.", "top_p")
- return maximum, float(temperature), float(top_p), grammar
+ return maximum, float(temperature), float(top_p), grammar, stop_sequences
def read_engine_turn(stream, sentinel, on_bytes):
@@ -1108,7 +1173,7 @@ class Engine:
self._fail_pending(error)
def generate(self, prompt, max_tokens, temperature, top_p, on_text, cache_slot=0,
- cancelled=None, grammar=None):
+ cancelled=None, grammar=None, stopped=None):
if isinstance(cache_slot, bool) or not isinstance(cache_slot, int) or not 0 <= cache_slot < self.kv_slots:
raise APIError(400, "Invalid cache slot.", "cache_slot")
payload = prompt.encode("utf-8")
@@ -1150,12 +1215,18 @@ class Engine:
raise
cancel_sent = False
+ stop_sent = False
while True:
kind, value = events.get()
if kind == "data":
- if not cancel_sent:
+ if not cancel_sent and not stop_sent:
decode(value)
- if cancelled and cancelled():
+ if stopped and stopped():
+ stop_sent = True
+ with self.write_lock:
+ self.process.stdin.write(f"STOP {request_id}\n".encode())
+ self.process.stdin.flush()
+ elif cancelled and cancelled():
cancel_sent = True
with self.write_lock:
self.process.stdin.write(f"CANCEL {request_id}\n".encode())
@@ -1443,7 +1514,8 @@ class APIHandler(BaseHTTPRequestHandler):
if dbg >= 2:
sys.stderr.write(f"\n===== PROMPT [{request_id}] =====\n{prompt}\n===== OUTPUT [{request_id}] =====\n")
sys.stderr.flush()
- maximum, temperature, top_p, grammar = generation_options(body, self.server.max_tokens)
+ maximum, temperature, top_p, grammar, stop_sequences = generation_options(
+ body, self.server.max_tokens)
if grammar is not None and ARCH == "inkling":
# inkling.c's serve loop speaks the 6-field SUBMIT header only; sending the
# grammar payload extension would desync its stdin framing.
@@ -1475,9 +1547,11 @@ class APIHandler(BaseHTTPRequestHandler):
queue_headers = {"x-colibri-queue-wait-ms": str(round(queue_wait * 1000))}
if not stream:
output = []
+ stop_filter = StopFilter(stop_sequences, output.append)
stats = self.server.engine.generate(
- prompt, maximum, temperature, top_p, output.append, cache_slot,
- self.client_disconnected, grammar=grammar)
+ prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,
+ self.client_disconnected, grammar=grammar, stopped=stop_filter.stopped)
+ stop_filter.finish()
text = "".join(output)
reasoning = ""
if ARCH == "inkling":
@@ -1597,9 +1671,11 @@ class APIHandler(BaseHTTPRequestHandler):
if flush:
emit(sp["buf"][:flush])
sp["buf"] = sp["buf"][flush:]
+ stop_filter = StopFilter(stop_sequences, emit_tools)
stats = self.server.engine.generate(
- prompt, maximum, temperature, top_p, emit_tools, cache_slot,
- lambda: not connected, grammar=grammar)
+ prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,
+ lambda: not connected, grammar=grammar, stopped=stop_filter.stopped)
+ stop_filter.finish()
if not sp["tool"] and sp["buf"]:
emit(sp["buf"]) # no tool call happened: flush held tail
_content, calls = parse_tool_calls("".join(raw), tools)
@@ -1614,9 +1690,11 @@ class APIHandler(BaseHTTPRequestHandler):
if dbg_echo:
sys.stderr.write(chunk); sys.stderr.flush()
(splitter.feed if splitter else emit)(chunk)
+ stop_filter = StopFilter(stop_sequences, emit_plain)
stats = self.server.engine.generate(
- prompt, maximum, temperature, top_p, emit_plain, cache_slot,
- lambda: not connected, grammar=grammar)
+ prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,
+ lambda: not connected, grammar=grammar, stopped=stop_filter.stopped)
+ stop_filter.finish()
if splitter:
splitter.close()
finish = "length" if stats["length_limited"] else "stop"
@@ -1710,7 +1788,8 @@ class APIHandler(BaseHTTPRequestHandler):
self.anthropic_generation(translated, prompt, request_id, tools, enable_thinking)
def anthropic_generation(self, body, prompt, request_id, tools, enable_thinking):
- maximum, temperature, top_p, grammar = generation_options(body, self.server.max_tokens)
+ maximum, temperature, top_p, grammar, _stop_sequences = generation_options(
+ body, self.server.max_tokens)
cache_slot = body.get("cache_slot")
if (cache_slot is not None and
(isinstance(cache_slot, bool) or not isinstance(cache_slot, int) or
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index 98da4c18..24e6f47e 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -11,19 +11,23 @@ from urllib.request import Request, urlopen
from pathlib import Path
from openai_server import (APIError, APIHandler, APIServer, ClientCancelled, END, GenerationScheduler,
- READY, Engine, _engine_error, generation_options, parse_tool_calls,
- read_engine_turn, render_chat, serve)
+ READY, Engine, StopFilter, _engine_error, generation_options,
+ parse_tool_calls, read_engine_turn, render_chat, serve)
class FakeEngine:
def __init__(self):
self.calls = []
+ self.stop_requests = 0
def generate(self, prompt, maximum, temperature, top_p, on_text, cache_slot=0,
- cancelled=None, grammar=None):
+ cancelled=None, grammar=None, stopped=None):
self.calls.append((prompt, maximum, temperature, top_p, cache_slot, grammar))
- on_text("Hé")
- on_text("llo")
+ for chunk in ("Hé", "llo"):
+ on_text(chunk)
+ if stopped and stopped():
+ self.stop_requests += 1
+ break
return {"prompt_tokens": 7, "completion_tokens": 2, "length_limited": False}
@@ -34,11 +38,11 @@ class BlockingEngine(FakeEngine):
self.release = threading.Event()
def generate(self, prompt, maximum, temperature, top_p, on_text, cache_slot=0,
- cancelled=None, grammar=None):
+ cancelled=None, grammar=None, stopped=None):
self.entered.set()
self.release.wait(2)
return super().generate(prompt, maximum, temperature, top_p, on_text, cache_slot,
- cancelled)
+ cancelled, grammar, stopped)
class TemplateTest(unittest.TestCase):
@@ -71,11 +75,11 @@ class TemplateTest(unittest.TestCase):
def test_validates_generation_limits(self):
self.assertEqual(generation_options({"max_tokens": 4, "temperature": 0, "top_p": 1}, 8),
- (4, 0.0, 1.0, None))
+ (4, 0.0, 1.0, None, ()))
# max_tokens above the server cap is clamped, not rejected (#260): OpenAI
# clients default to large values; erroring breaks them.
self.assertEqual(generation_options({"max_tokens": 9, "temperature": 0, "top_p": 1}, 8),
- (8, 0.0, 1.0, None))
+ (8, 0.0, 1.0, None, ()))
# non-positive / non-int max_tokens is still a hard error
with self.assertRaises(APIError):
generation_options({"max_tokens": 0}, 8)
@@ -84,7 +88,7 @@ class TemplateTest(unittest.TestCase):
with self.assertRaises(APIError):
generation_options({"top_p": math.inf}, 8)
self.assertEqual(generation_options({"temperature": None, "top_p": None}, 8),
- (8, 0.7, 0.9, None))
+ (8, 0.7, 0.9, None, ()))
# response_format -> grammar plumbing (draft source, never a constraint)
opts = generation_options({"max_tokens": 4, "response_format": {"type": "json_object"}}, 8)
self.assertIn("root ::=", opts[3])
@@ -110,6 +114,32 @@ class TemplateTest(unittest.TestCase):
opts = generation_options({"response_format": {"type": "gbnf", "grammar": "not a grammar ::="}}, 8)
self.assertEqual(opts[3], "not a grammar ::=")
+ def test_validates_stop_sequences(self):
+ self.assertEqual(generation_options({"stop": "END"}, 8)[4], ("END",))
+ self.assertEqual(generation_options({"stop": ["ONE", "TWO"]}, 8)[4],
+ ("ONE", "TWO"))
+ for value in ("", [], [""], ["1", "2", "3", "4", "5"], 7, ["ok", 7]):
+ with self.subTest(value=value), self.assertRaises(APIError):
+ generation_options({"stop": value}, 8)
+
+
+class StopFilterTest(unittest.TestCase):
+ def test_hides_match_split_across_chunks(self):
+ output = []
+ stop_filter = StopFilter(("STOP",), output.append)
+ for chunk in ("answer S", "TO", "Pignored"):
+ stop_filter.feed(chunk)
+ stop_filter.finish()
+ self.assertEqual("".join(output), "answer ")
+ self.assertEqual(stop_filter.matched, "STOP")
+
+ def test_flushes_partial_prefix_when_generation_finishes(self):
+ output = []
+ stop_filter = StopFilter(("STOP",), output.append)
+ stop_filter.feed("answer ST")
+ stop_filter.finish()
+ self.assertEqual("".join(output), "answer ST")
+
class ProtocolTest(unittest.TestCase):
def test_reads_payload_and_extended_status(self):
@@ -446,6 +476,30 @@ class DispatcherTest(unittest.TestCase):
self.assertEqual(output, ["x"])
self.assertEqual(process.writes[-1].split(), [b"CANCEL", request_id])
+ def test_stops_generation_through_successful_done_path(self):
+ request_id = None
+
+ def respond(process, frame):
+ nonlocal request_id
+ fields = frame.split()
+ if fields[0] == b"SUBMIT":
+ request_id = fields[1]
+ process.stdout.feed(b"DATA " + request_id + b" 1\nx\n")
+ elif fields[0] == b"STOP":
+ self.assertEqual(fields[1], request_id)
+ process.stdout.feed(b"DONE " + request_id + b" STAT 1 1 0 1 2 0\n")
+
+ process = FakeProcess(respond)
+ with patch("openai_server.subprocess.Popen", return_value=process):
+ engine = Engine("glm", "model")
+ output = []
+ stats = engine.generate("hello", 8, 0.7, 0.9, output.append,
+ stopped=lambda: output == ["x"])
+ engine.close()
+ self.assertEqual(output, ["x"])
+ self.assertEqual(stats["completion_tokens"], 1)
+ self.assertEqual(process.writes[-1].split(), [b"STOP", request_id])
+
class HTTPTest(unittest.TestCase):
@classmethod
@@ -525,6 +579,17 @@ class HTTPTest(unittest.TestCase):
self.assertIn("<|user|>Hi<|assistant|><think></think>", self.engine.calls[-1][0])
self.assertEqual(self.engine.calls[-1][4], 1)
+ def test_chat_completion_stops_across_engine_chunks(self):
+ before = self.engine.stop_requests
+ with self.request("/v1/chat/completions", {
+ "model": "test-model", "messages": [{"role": "user", "content": "Hi"}],
+ "stop": "éll",
+ }) as response:
+ body = json.load(response)
+ self.assertEqual(body["choices"][0]["message"]["content"], "H")
+ self.assertEqual(body["choices"][0]["finish_reason"], "stop")
+ self.assertEqual(self.engine.stop_requests, before + 1)
+
def test_rejects_invalid_cache_slot(self):
with self.assertRaises(HTTPError) as caught:
self.request("/v1/chat/completions", {
@@ -545,6 +610,21 @@ class HTTPTest(unittest.TestCase):
self.assertIn('\"usage\":{\"prompt_tokens\":7,\"completion_tokens\":2,\"total_tokens\":9}', stream)
self.assertTrue(stream.endswith("data: [DONE]\n\n"))
+ def test_streaming_stop_never_exposes_partial_sequence(self):
+ before = self.engine.stop_requests
+ with self.request("/v1/chat/completions", {
+ "model": "test-model", "messages": [{"role": "user", "content": "Hi"}],
+ "stream": True, "stop": "éll",
+ }) as response:
+ raw = response.read().decode()
+ payloads = [json.loads(line[6:]) for line in raw.splitlines()
+ if line.startswith("data: ") and line != "data: [DONE]"]
+ content = "".join((choice.get("delta") or {}).get("content", "")
+ for payload in payloads for choice in payload["choices"])
+ self.assertEqual(content, "H")
+ self.assertEqual(payloads[-1]["choices"][0]["finish_reason"], "stop")
+ self.assertEqual(self.engine.stop_requests, before + 1)
+
def test_legacy_completion(self):
with self.request("/v1/completions", {
"model": "test-model", "prompt": "Complete me", "temperature": 0,
diff --git a/docs/api.md b/docs/api.md
index 2b92bb6e..f3bb16c3 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -24,14 +24,16 @@ curl http://127.0.0.1:8000/v1/chat/completions \
Implemented endpoints are `GET /v1/models`, `GET /v1/models/{model}`,
`POST /v1/chat/completions`, and legacy `POST /v1/completions`. Chat and
completion requests support JSON responses, SSE streaming, usage counts,
-`max_tokens`/`max_completion_tokens`, `temperature`, and `top_p`. The extension
+`max_tokens`/`max_completion_tokens`, `temperature`, `top_p`, and up to four
+custom `stop` sequences. Stop sequences are removed from the response and end
+generation early in both JSON and streaming modes. The extension
`enable_thinking: true` enables GLM-5.2's reasoning block; the standard
`reasoning_effort` field also enables it unless set to `none`.
The server is deliberately text-only and serves one generation at a time: the
744B model stays in one persistent process, so concurrent HTTP requests queue
-instead of loading duplicate model copies. Tools, image/audio input, custom
-stop sequences, log probabilities, and token penalties return an explicit error
+instead of loading duplicate model copies. Image/audio input, log probabilities,
+and token penalties return an explicit error
rather than being silently ignored. The default bind address is localhost; set
`COLI_API_KEY` before exposing the server beyond the machine.
diff --git a/docs/serve_protocol.md b/docs/serve_protocol.md
index b0eb61ec..8b9000b4 100644
--- a/docs/serve_protocol.md
+++ b/docs/serve_protocol.md
@@ -33,6 +33,7 @@ recognize**; that is the protocol's forward-compatibility rule.
```
SUBMIT <id> <slot> <bytes> <max_tokens> <temperature> <top_p>\n<payload>\n
+STOP <id>\n
CANCEL <id>\n
```
@@ -44,6 +45,10 @@ CANCEL <id>\n
- `bytes` — exact byte length of `payload` (UTF-8, may contain newlines). The engine
reads exactly that many bytes after the header line, then one trailing `\n`.
- `payload` — the fully rendered prompt (the server owns the chat template).
+- `STOP` ends generation through the normal successful `DONE` path. Statistics,
+ usage history, and KV state are persisted; the HTTP gateway uses it after a
+ client-provided stop sequence matches.
+- `CANCEL` aborts a request after its client disconnects and returns `CANCELLED`.
- EOF on stdin = graceful shutdown: in-flight requests finish first.
Prefill is serial; decode is continuously batched — every active slot contributes