Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
OpenAI-compatible HTTP API: stdlib-only gateway over SERVE with KV prefix reuse across stateless requests (#21)
ZacharyZcR3e4d08b6bfe6
diff --git a/README.md b/README.mdindex deaed7af..2bf9eb5b 100644--- a/README.md+++ b/README.md@@ -93,6 +93,45 @@ COLI_MODEL=/nvme/glm52_i4 ./coli chatThe engine at runtime is pure C — python is only used by the one-time converter.+### OpenAI-compatible API++`coli serve` keeps one model process loaded and exposes a text-only OpenAI-compatible+HTTP API. The gateway uses only the Python standard library; inference still runs in+the same dependency-free C engine.++```bash+cd c+COLI_MODEL=/nvme/glm52_i4 COLI_API_KEY=local-secret ./coli serve \+ --host 127.0.0.1 --port 8000 --model-id glm-5.2-colibri++curl http://127.0.0.1:8000/v1/chat/completions \+ -H 'Authorization: Bearer local-secret' \+ -H 'Content-Type: application/json' \+ -d '{+ "model": "glm-5.2-colibri",+ "messages": [{"role": "user", "content": "Hello"}],+ "stream": true+ }'+```++Implemented endpoints are `GET /v1/models`, `GET /v1/models/{model}`,+`POST /v1/chat/completions`, and legacy `POST /v1/completions`. Chat and+completion requests support JSON responses, SSE streaming, usage counts,+`max_tokens`/`max_completion_tokens`, `temperature`, and `top_p`. The extension+`enable_thinking: true` enables GLM-5.2's reasoning block; the standard+`reasoning_effort` field also enables it unless set to `none`.++The first version is deliberately text-only and serves one generation at a time:+the 744B model stays in one persistent process, so concurrent HTTP requests queue+instead of loading duplicate model copies. Tools, image/audio input, custom stop+sequences, log probabilities, and token penalties return an explicit error rather+than being silently ignored. The default bind address is localhost; set+`COLI_API_KEY` before exposing the server beyond the machine.++Browser access from the Vite development server and Tauri local origins is enabled+by default. Repeat `--cors-origin https://your-ui.example` to allow another exact+origin, or use `--cors-origin '*'` only on a trusted local network.+### Experimental resident CUDA backendcolibrì includes an opt-in CUDA backend for model-resident tensors. Streaming@@ -266,6 +305,7 @@ c/├── backend_cuda.* optional CUDA tier├── Makefile build and local checks├── coli user-facing CLI+├── openai_server.py OpenAI-compatible HTTP gateway├── setup.sh one-command local setup├── tools/ offline conversion, fixtures and benchmarks├── scripts/ long-running conversion helpersdiff --git a/c/coli b/c/coliindex 5e304b7c..052d8570 100755--- a/c/coli+++ b/c/coli@@ -4,6 +4,7 @@ colibrì — piccolo motore, modello immenso.CLI per far girare GLM-5.2 (744B) in locale, su CPU, in ~15-26 GB di RAM.coli chat chat interattiva (carica il modello UNA volta)+ coli serve API HTTP compatibile OpenAI (motore persistente)coli run "prompt" generazione singolacoli info stato: modello, RAM, disco, configcoli bench [task...] benchmark di qualità (MMLU/HellaSwag/...)@@ -375,6 +376,12 @@ def cmd_chat(a):except Exception: passprint(f" {C.teal}ciao{C.r} {C.dim}— il colibrì torna al nido{C.r} 🐦\n")+def cmd_serve(a):+ need_model(a.model)+ from openai_server import serve+ serve(a.model, a.host, a.port, a.model_id, a.api_key,+ a.cap, a.ngen, GLM, env_for(a), a.cors_origin)+def cmd_bench(a):need_model(a.model)banner("bench")@@ -427,13 +434,18 @@ def main():sub.add_parser("build", parents=[common]); sub.add_parser("info", parents=[common])pr=sub.add_parser("run", parents=[common]); pr.add_argument("prompt", nargs="*")sub.add_parser("chat", parents=[common])+ ps=sub.add_parser("serve", parents=[common])+ ps.add_argument("--host",default="127.0.0.1"); ps.add_argument("--port",type=int,default=8000)+ ps.add_argument("--model-id",default=os.environ.get("COLI_MODEL_ID","glm-5.2-colibri"))+ ps.add_argument("--api-key",default=os.environ.get("COLI_API_KEY"))+ ps.add_argument("--cors-origin",action="append",default=None)pb=sub.add_parser("bench", parents=[common]); pb.add_argument("tasks", nargs="*")pb.add_argument("--limit",type=int,default=40); pb.add_argument("--data",default=os.path.join(HERE,"bench"))pc=sub.add_parser("convert", parents=[common]); pc.add_argument("--repo",default="zai-org/GLM-5.2-FP8")pc.add_argument("--ebits",type=int,default=4); pc.add_argument("--io-bits",type=int,default=8); pc.add_argument("--xbits",type=int,default=0)pc.add_argument("--no-mtp",action="store_true",help="salta la testa MTP (niente draft speculativi)")a=ap.parse_args()- {"build":cmd_build,"info":cmd_info,"run":cmd_run,"chat":cmd_chat,"bench":cmd_bench,+ {"build":cmd_build,"info":cmd_info,"run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"bench":cmd_bench,"convert":cmd_convert}.get(a.cmd, lambda _:(banner(),print(__doc__)))(a)if __name__=="__main__":diff --git a/c/glm.c b/c/glm.cindex 63cb9d7d..95f35319 100644--- a/c/glm.c+++ b/c/glm.c@@ -1828,12 +1828,14 @@ static void kv_hdr(Model *m, int32_t *h, int nrec){h[0]=c->n_layers; h[1]=c->kv_lora; h[2]=c->qk_rope;h[3]=m->has_dsa?c->index_hd:0; h[4]=nic; h[5]=c->vocab; h[6]=nrec; h[7]=0;}-static void kv_disk_reset(void){+static void kv_disk_truncate(int nrec){if(!g_kvsave) return;- FILE *f=fopen(g_kv_path,"r+b"); if(!f) return;- int32_t nz=0; fseek(f,8+6*4,SEEK_SET); fwrite(&nz,4,1,f); fclose(f);- g_kv_nrec=0;+ FILE *f=fopen(g_kv_path,"r+b");+ if(!f){ g_kv_nrec=0; return; }+ g_kv_nrec=nrec;+ int32_t nr=nrec; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); fclose(f);}+static void kv_disk_reset(void){ kv_disk_truncate(0); }static void kv_disk_append(Model *m, const int *hist, int len){if(!g_kvsave || len<=g_kv_nrec) return;Cfg *c=&m->c;@@ -1930,33 +1932,79 @@ static void run_serve(Model *m, const char *snap){printf("STAT %d %.2f %.1f %.2f\n", prod, prod/tdt, (dh+dm)>0?100.0*dh/(dh+dm):0.0, rss_gb());fflush(stdout); kv_disk_append(m,hist,len); repin_pass(m); continue; } /* RFC: re-pin a caldo tra i turni / live re-pin between turns */if(nr<1){ printf("\x01\x01" "END" "\x01\x01\n"); printf("STAT 0 0.00 0.0 %.2f\n", rss_gb()); fflush(stdout); continue; }- int bl=0; /* costruisce il testo del turno (con template) */+ /* API mode: an exact, length-prefixed prompt. Unlike the interactive+ * line protocol this accepts newlines. The tokenized prompt is matched+ * against hist so the common KV prefix survives stateless HTTP turns.+ * Per-request generation controls follow the byte count:+ * \x02PROMPT <bytes> <max_tokens> <temperature> <top_p>\n<prompt>\n */+ char *raw=NULL, *input=line;+ int input_n=(int)nr, raw_mode=0, req_ngen=ngen, prompt_tokens=0;+ float base_temp=g_temp, base_nuc=g_nuc;+ if(!strncmp(line,"\x02PROMPT ",8)){+ unsigned long long nb=0; double rt=0, rp=0;+ if(sscanf(line+8,"%llu %d %lf %lf",&nb,&req_ngen,&rt,&rp)!=4 ||+ nb>(16u<<20) || req_ngen<1 || rt<0 || rt>2 || rp<=0 || rp>1){+ printf("\x01\x01" "END" "\x01\x01\n"); printf("STAT 0 0.00 0.0 %.2f 0 0\n",rss_gb()); fflush(stdout); continue;+ }+ raw=malloc((size_t)nb+1); if(!raw){fprintf(stderr,"OOM raw prompt\n");exit(1);}+ if(fread(raw,1,(size_t)nb,stdin)!=(size_t)nb){free(raw);break;}+ int delim=fgetc(stdin); if(delim!='\n' && delim!=EOF) ungetc(delim,stdin);+ if(memchr(raw,0,(size_t)nb)){free(raw); printf("\x01\x01" "END" "\x01\x01\n");+ printf("STAT 0 0.00 0.0 %.2f 0 0\n",rss_gb()); fflush(stdout); continue;}+ raw[nb]=0; input=raw; input_n=(int)nb; raw_mode=1;+ if(req_ngen>ngen) req_ngen=ngen;+ g_temp=(float)rt; g_nuc=(float)rp;+ }+ int bl=0, k=0; /* costruisce/tokenizza il turno *//* template UFFICIALE GLM-5.2 (chat_template.jinja): niente \n dopo i ruoli, e dopo* <|assistant|> serve SEMPRE il blocco think — <think></think> lo DISATTIVA (nothink):* col template sbagliato il modello farfuglia e non emette mai lo stop. THINK=1 lo abilita. */const char *tk = getenv("THINK")&&atoi(getenv("THINK"))? "<think>" : "<think></think>";- if(templ){ if(first) bl+=snprintf(buf+bl,(1<<16)-bl,"[gMASK]<sop>");- bl+=snprintf(buf+bl,(1<<16)-bl,"<|user|>%s<|assistant|>%s",line,tk); }- else bl+=snprintf(buf+bl,(1<<16)-bl,"%s",line);- int k=tok_encode(&T,buf,bl,hist+len,maxctx-len);- if(k<1){ printf("\x01\x01" "END" "\x01\x01\n"); printf("STAT 0 0.00 0.0 %.2f\n", rss_gb()); fflush(stdout); continue; }- if(len+k+8+g_draft>=maxctx){ len=0; first=1; kv_disk_reset(); /* contesto pieno: azzera e ricomincia */- bl=0; if(templ){ bl+=snprintf(buf+bl,(1<<16)-bl,"[gMASK]<sop><|user|>%s<|assistant|>%s",line,tk); }- else bl+=snprintf(buf+bl,(1<<16)-bl,"%s",line);- k=tok_encode(&T,buf,bl,hist,maxctx); if(k>maxctx-8-g_draft) k=maxctx-8-g_draft; }+ if(raw_mode){+ int *tmp=malloc(maxctx*sizeof(int)); if(!tmp){fprintf(stderr,"OOM raw tokens\n");exit(1);}+ prompt_tokens=tok_encode(&T,input,input_n,tmp,maxctx-8-g_draft);+ int old_len=len, prefix=0;+ while(prefix<old_len && prefix<prompt_tokens && hist[prefix]==tmp[prefix]) prefix++;+ if(prefix<old_len){+ len=prefix;+ if(m->has_mtp) m->kv_start[m->c.n_layers]=-1;+ kv_disk_truncate(len); /* il prossimo append sovrascrive solo la coda */+ }+ k=prompt_tokens-len;+ if(k>0) memcpy(hist+len,tmp+len,k*sizeof(int));+ fprintf(stderr,"[API] KV prefix %d/%d token, prefill %d\n",len,prompt_tokens,k);+ free(tmp);+ } else {+ if(templ){ if(first) bl+=snprintf(buf+bl,(1<<16)-bl,"[gMASK]<sop>");+ bl+=snprintf(buf+bl,(1<<16)-bl,"<|user|>%s<|assistant|>%s",input,tk); }+ else bl+=snprintf(buf+bl,(1<<16)-bl,"%s",input);+ k=tok_encode(&T,buf,bl,hist+len,maxctx-len); prompt_tokens=k;+ if(len+k+8+g_draft>=maxctx){ len=0; first=1; kv_disk_reset();+ bl=0; if(templ){ bl+=snprintf(buf+bl,(1<<16)-bl,"[gMASK]<sop><|user|>%s<|assistant|>%s",input,tk); }+ else bl+=snprintf(buf+bl,(1<<16)-bl,"%s",input);+ k=tok_encode(&T,buf,bl,hist,maxctx); if(k>maxctx-8-g_draft) k=maxctx-8-g_draft;+ prompt_tokens=k;+ }+ }+ if(prompt_tokens<1){ free(raw); g_temp=base_temp; g_nuc=base_nuc;+ printf("\x01\x01" "END" "\x01\x01\n"); printf("STAT 0 0.00 0.0 %.2f 0 0\n", rss_gb()); fflush(stdout); continue; }first=0;- int cur=ngen; if(len+k+cur+g_draft+2>=maxctx) cur=maxctx-len-k-g_draft-2;+ int cur=req_ngen; if(len+k+cur+g_draft+2>=maxctx) cur=maxctx-len-k-g_draft-2;uint64_t h0=m->hits, ms0=m->miss; double tt0=now_s();- float *logit=step(m,hist+len,k,len); len+=k;+ float *logit;+ if(k>0){ logit=step(m,hist+len,k,len); len+=k; }+ else logit=step(m,hist+len-1,1,len-1); /* prompt identico/prefisso: rigenera i logits */EmitStream es={&T,m,now_s(),0,1};int prod=0;if(cur>0) prod=spec_decode(m,hist,len,cur,eos,logit,emit_stream,&es,&len);else free(logit);double tdt=now_s()-tt0; if(tdt<1e-6) tdt=1e-6;double dh=(double)(m->hits-h0), dm=(double)(m->miss-ms0);- printf("\n\x01\x01" "END" "\x01\x01\n");- printf("STAT %d %.2f %.1f %.2f\n", prod, prod/tdt, (dh+dm)>0?100.0*dh/(dh+dm):0.0, rss_gb());+ printf("%s\x01\x01" "END" "\x01\x01\n",raw_mode?"":"\n");+ printf("STAT %d %.2f %.1f %.2f %d %d\n", prod, prod/tdt,+ (dh+dm)>0?100.0*dh/(dh+dm):0.0, rss_gb(), prompt_tokens, prod>=cur);fflush(stdout);+ free(raw); g_temp=base_temp; g_nuc=base_nuc;usage_save(m); /* la cache che impara: storia aggiornata a ogni turno */kv_disk_append(m,hist,len); /* KV su disco: il prossimo avvio riparte da qui */}diff --git a/c/openai_server.py b/c/openai_server.pynew file mode 100644index 00000000..d97bd64b--- /dev/null+++ b/c/openai_server.py@@ -0,0 +1,466 @@+#!/usr/bin/env python3+"""Dependency-free OpenAI-compatible HTTP gateway for the colibri engine."""++import argparse+import codecs+import json+import os+import signal+import subprocess+import sys+import threading+import time+import uuid+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from pathlib import Path+from urllib.parse import unquote, urlsplit+++HERE = Path(__file__).resolve().parent+END = b"\x01\x01END\x01\x01\n"+READY = b"\x01\x01READY\x01\x01\n"+MAX_BODY = 4 << 20+DEFAULT_CORS_ORIGINS = (+ "http://127.0.0.1:5173",+ "http://localhost:5173",+ "http://tauri.localhost",+ "tauri://localhost",+)+++class APIError(Exception):+ def __init__(self, status, message, param=None, code=None, error_type="invalid_request_error"):+ super().__init__(message)+ self.status = status+ self.message = message+ self.param = param+ self.code = code+ self.error_type = error_type+++def error_object(error):+ return {"error": {"message": error.message, "type": error.error_type,+ "param": error.param, "code": error.code}}+++def content_text(content, param):+ if isinstance(content, str):+ return content+ if not isinstance(content, list):+ raise APIError(400, "Message content must be a string or an array of text parts.", param)+ parts = []+ for index, part in enumerate(content):+ if not isinstance(part, dict) or part.get("type") not in ("text", "input_text"):+ raise APIError(400, "Colibri currently supports text message content only.",+ f"{param}.{index}", "unsupported_content_type")+ if not isinstance(part.get("text"), str):+ raise APIError(400, "Text content parts require a string `text` field.",+ f"{param}.{index}.text")+ parts.append(part["text"])+ return "".join(parts)+++def render_chat(messages, enable_thinking=False, reasoning_effort=None):+ """Render the text-only subset of the official GLM-5.2 chat template."""+ if not isinstance(messages, list) or not messages:+ raise APIError(400, "`messages` must be a non-empty array.", "messages")+ prompt = ["[gMASK]<sop>"]+ if enable_thinking:+ effort = "High" if reasoning_effort == "high" else "Max"+ prompt.append(f"<|system|>Reasoning Effort: {effort}")+ for index, message in enumerate(messages):+ if not isinstance(message, dict):+ raise APIError(400, "Each message must be an object.", f"messages.{index}")+ role = message.get("role")+ text = content_text(message.get("content"), f"messages.{index}.content")+ if role in ("system", "developer"):+ prompt.append(f"<|system|>{text}")+ elif role == "user":+ prompt.append(f"<|user|>{text}")+ elif role == "assistant":+ prompt.append(f"<|assistant|><think></think>{text.strip()}")+ else:+ raise APIError(400, f"Unsupported message role: {role!r}.",+ f"messages.{index}.role", "unsupported_role")+ prompt.append("<|assistant|><think>" if enable_thinking else+ "<|assistant|><think></think>")+ return "".join(prompt)+++def generation_options(body, limit):+ if body.get("n", 1) != 1:+ raise APIError(400, "Colibri currently supports `n=1` only.", "n", "unsupported_value")+ for name in ("tools", "functions"):+ if body.get(name):+ raise APIError(400, f"`{name}` is not supported yet.", name, "unsupported_parameter")+ if body.get("stop") is not None:+ raise APIError(400, "Custom stop sequences are not supported yet.", "stop", "unsupported_parameter")+ if body.get("logprobs"):+ raise APIError(400, "Log probabilities are not supported yet.", "logprobs", "unsupported_parameter")+ if body.get("frequency_penalty", 0) or body.get("presence_penalty", 0):+ raise APIError(400, "Token penalties are not supported yet.", None, "unsupported_parameter")+ if body.get("seed") is not None:+ raise APIError(400, "Per-request seeds are not supported yet.", "seed", "unsupported_parameter")+ response_format = body.get("response_format")+ if response_format not in (None, {"type": "text"}):+ raise APIError(400, "Only the default text response format is supported.",+ "response_format", "unsupported_parameter")++ maximum = body.get("max_completion_tokens")+ maximum_param = "max_completion_tokens"+ if maximum is None:+ maximum = body.get("max_tokens")+ maximum_param = "max_tokens"+ if maximum is None:+ maximum = min(256, limit)+ temperature = body.get("temperature")+ top_p = body.get("top_p")+ temperature = 0.7 if temperature is None else temperature+ top_p = 0.9 if top_p is None else top_p+ if isinstance(maximum, bool) or not isinstance(maximum, int) or not 1 <= maximum <= limit:+ raise APIError(400, f"`{maximum_param}` must be an integer between 1 and {limit}.", maximum_param)+ if isinstance(temperature, bool) or not isinstance(temperature, (int, float)) or not 0 <= temperature <= 2:+ raise APIError(400, "`temperature` must be between 0 and 2.", "temperature")+ if isinstance(top_p, bool) or not isinstance(top_p, (int, float)) or not 0 < top_p <= 1:+ raise APIError(400, "`top_p` must be greater than 0 and at most 1.", "top_p")+ return maximum, float(temperature), float(top_p)+++def read_engine_turn(stream, sentinel, on_bytes):+ pending = b""+ while True:+ byte = stream.read(1)+ if byte == b"":+ raise RuntimeError("colibri engine exited unexpectedly")+ pending += byte+ if pending.endswith(sentinel):+ data = pending[:-len(sentinel)]+ if data:+ on_bytes(data)+ break+ if len(pending) > len(sentinel):+ on_bytes(pending[:-len(sentinel)])+ pending = pending[-len(sentinel):]++ fields = stream.readline().decode("utf-8", "replace").strip().split()+ if len(fields) < 5 or fields[0] != "STAT":+ raise RuntimeError(f"invalid engine status: {' '.join(fields)}")+ return {+ "completion_tokens": int(fields[1]),+ "tokens_per_second": float(fields[2]),+ "cache_hit_percent": float(fields[3]),+ "rss_gb": float(fields[4]),+ "prompt_tokens": int(fields[5]) if len(fields) > 5 else 0,+ "length_limited": bool(int(fields[6])) if len(fields) > 6 else False,+ }+++class Engine:+ def __init__(self, executable, model, cap=8, max_tokens=1024, env=None):+ child_env = dict(env or os.environ, SNAP=str(model), SERVE="1", NGEN=str(max_tokens))+ self.process = subprocess.Popen(+ [str(executable), str(cap)], env=child_env, stdin=subprocess.PIPE,+ stdout=subprocess.PIPE, bufsize=0,+ )+ self.lock = threading.Lock()+ read_engine_turn(self.process.stdout, READY, lambda _: None)++ def generate(self, prompt, max_tokens, temperature, top_p, on_text):+ payload = prompt.encode("utf-8")+ if b"\0" in payload:+ raise APIError(400, "NUL bytes are not supported in prompts.", "messages")+ decoder = codecs.getincrementaldecoder("utf-8")("replace")++ def decode(data):+ text = decoder.decode(data)+ if text:+ on_text(text)++ with self.lock:+ if self.process.poll() is not None:+ raise RuntimeError("colibri engine is not running")+ header = f"\x02PROMPT {len(payload)} {max_tokens} {temperature:.8g} {top_p:.8g}\n".encode()+ self.process.stdin.write(header + payload + b"\n")+ self.process.stdin.flush()+ stats = read_engine_turn(self.process.stdout, END, decode)+ tail = decoder.decode(b"", final=True)+ if tail:+ on_text(tail)+ return stats++ def close(self):+ if self.process.poll() is None:+ self.process.terminate()+ try:+ self.process.wait(timeout=5)+ except subprocess.TimeoutExpired:+ self.process.kill()+++def model_object(model_id, created):+ return {"id": model_id, "object": "model", "created": created, "owned_by": "colibri"}+++class APIServer(ThreadingHTTPServer):+ daemon_threads = True++ def __init__(self, address, engine, model_id, api_key=None, max_tokens=1024,+ cors_origins=DEFAULT_CORS_ORIGINS):+ super().__init__(address, APIHandler)+ self.engine = engine+ self.model_id = model_id+ self.api_key = api_key+ self.max_tokens = max_tokens+ self.cors_origins = tuple(cors_origins)+ self.created = int(time.time())+++class APIHandler(BaseHTTPRequestHandler):+ protocol_version = "HTTP/1.1"+ server_version = "colibri"++ def log_message(self, fmt, *args):+ sys.stderr.write("[api] %s - %s\n" % (self.address_string(), fmt % args))++ def send_json(self, status, body, request_id=None):+ data = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode()+ self.send_response(status)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(data)))+ if request_id:+ self.send_header("x-request-id", request_id)+ self.send_cors_headers()+ self.end_headers()+ self.wfile.write(data)++ def send_cors_headers(self):+ origin = self.headers.get("Origin")+ if not origin or ("*" not in self.server.cors_origins and origin not in self.server.cors_origins):+ return+ self.send_header("Access-Control-Allow-Origin", "*" if "*" in self.server.cors_origins else origin)+ self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")+ self.send_header("Access-Control-Allow-Headers", "Authorization, Content-Type")+ self.send_header("Access-Control-Expose-Headers", "x-request-id")+ self.send_header("Access-Control-Max-Age", "600")+ if "*" not in self.server.cors_origins:+ self.send_header("Vary", "Origin")++ def require_auth(self):+ if self.server.api_key and self.headers.get("Authorization") != f"Bearer {self.server.api_key}":+ raise APIError(401, "Invalid or missing API key.", None, "invalid_api_key",+ "authentication_error")++ def read_json(self):+ try:+ length = int(self.headers.get("Content-Length", "0"))+ except ValueError:+ raise APIError(400, "Invalid Content-Length header.")+ if length < 1 or length > MAX_BODY:+ raise APIError(400, f"Request body must be between 1 and {MAX_BODY} bytes.")+ try:+ body = json.loads(self.rfile.read(length))+ except (json.JSONDecodeError, UnicodeDecodeError):+ raise APIError(400, "Request body must be valid JSON.")+ if not isinstance(body, dict):+ raise APIError(400, "Request body must be a JSON object.")+ return body++ def check_model(self, body):+ model = body.get("model")+ if model != self.server.model_id:+ raise APIError(404, f"The model `{model}` does not exist.", "model", "model_not_found")++ def do_GET(self):+ request_id = "req_" + uuid.uuid4().hex+ try:+ path = urlsplit(self.path).path+ if path == "/health":+ self.send_json(200, {"status": "ok"}, request_id)+ return+ self.require_auth()+ if path == "/v1/models":+ self.send_json(200, {"object": "list", "data": [model_object(+ self.server.model_id, self.server.created)]}, request_id)+ elif path.startswith("/v1/models/") and unquote(path[11:]) == self.server.model_id:+ self.send_json(200, model_object(self.server.model_id, self.server.created), request_id)+ else:+ raise APIError(404, "Not found.", None, "not_found")+ except APIError as error:+ self.send_json(error.status, error_object(error), request_id)++ def do_OPTIONS(self):+ self.send_response(204)+ self.send_header("Content-Length", "0")+ self.send_cors_headers()+ self.end_headers()++ def do_POST(self):+ request_id = "req_" + uuid.uuid4().hex+ try:+ self.require_auth()+ body = self.read_json()+ self.check_model(body)+ path = urlsplit(self.path).path+ if path == "/v1/chat/completions":+ self.chat_completion(body, request_id)+ elif path == "/v1/completions":+ self.completion(body, request_id)+ else:+ raise APIError(404, "Not found.", None, "not_found")+ except APIError as error:+ self.send_json(error.status, error_object(error), request_id)+ except (BrokenPipeError, ConnectionResetError):+ pass+ except Exception as error:+ self.log_error("request failed: %s", error)+ api_error = APIError(500, "The colibri engine failed to process the request.",+ None, "engine_error", "server_error")+ try:+ self.send_json(500, error_object(api_error), request_id)+ except OSError:+ pass++ def generation(self, body, prompt, request_id, chat):+ maximum, temperature, top_p = generation_options(body, self.server.max_tokens)+ stream = body.get("stream", False)+ if not isinstance(stream, bool):+ raise APIError(400, "`stream` must be a boolean.", "stream")+ object_name = "chat.completion" if chat else "text_completion"+ id_prefix = "chatcmpl-" if chat else "cmpl-"+ completion_id = id_prefix + uuid.uuid4().hex+ created = int(time.time())++ if not stream:+ output = []+ stats = self.server.engine.generate(prompt, maximum, temperature, top_p, output.append)+ text = "".join(output)+ finish = "length" if stats["length_limited"] else "stop"+ choice = ({"index": 0, "message": {"role": "assistant", "content": text,+ "refusal": None}, "logprobs": None, "finish_reason": finish} if chat else+ {"index": 0, "text": text, "logprobs": None, "finish_reason": finish})+ self.send_json(200, {"id": completion_id, "object": object_name, "created": created,+ "model": self.server.model_id, "choices": [choice], "usage": self.usage(stats)}, request_id)+ return++ stream_options = body.get("stream_options")+ if stream_options is not None and not isinstance(stream_options, dict):+ raise APIError(400, "`stream_options` must be an object.", "stream_options")+ include_usage = bool((stream_options or {}).get("include_usage"))+ stream_object = "chat.completion.chunk" if chat else object_name+ self.send_response(200)+ self.send_header("Content-Type", "text/event-stream")+ self.send_header("Cache-Control", "no-cache")+ self.send_header("X-Accel-Buffering", "no")+ self.send_header("x-request-id", request_id)+ self.send_cors_headers()+ self.end_headers()+ connected = True++ def event(choices, usage_marker=False):+ nonlocal connected+ if not connected:+ return+ event_body = {"id": completion_id, "object": stream_object, "created": created,+ "model": self.server.model_id, "choices": choices}+ if include_usage:+ event_body["usage"] = None if not usage_marker else usage_marker+ try:+ data = json.dumps(event_body, ensure_ascii=False, separators=(",", ":"))+ self.wfile.write(f"data: {data}\n\n".encode())+ self.wfile.flush()+ except OSError:+ connected = False++ if chat:+ event([{"index": 0, "delta": {"role": "assistant", "content": ""},+ "logprobs": None, "finish_reason": None}])++ def emit(text):+ choice = ({"index": 0, "delta": {"content": text}, "logprobs": None,+ "finish_reason": None} if chat else+ {"index": 0, "text": text, "logprobs": None, "finish_reason": None})+ event([choice])++ stats = self.server.engine.generate(prompt, maximum, temperature, top_p, emit)+ finish = "length" if stats["length_limited"] else "stop"+ final_choice = ({"index": 0, "delta": {}, "logprobs": None, "finish_reason": finish}+ if chat else {"index": 0, "text": "", "logprobs": None,+ "finish_reason": finish})+ event([final_choice])+ if include_usage:+ event([], self.usage(stats))+ if connected:+ try:+ self.wfile.write(b"data: [DONE]\n\n")+ self.wfile.flush()+ except OSError:+ pass+ self.close_connection = True++ @staticmethod+ def usage(stats):+ prompt = stats["prompt_tokens"]+ completion = stats["completion_tokens"]+ return {"prompt_tokens": prompt, "completion_tokens": completion,+ "total_tokens": prompt + completion}++ def chat_completion(self, body, request_id):+ reasoning_effort = body.get("reasoning_effort")+ efforts = (None, "none", "minimal", "low", "medium", "high", "xhigh")+ if reasoning_effort not in efforts:+ raise APIError(400, "`reasoning_effort` must be none, minimal, low, medium, high, or xhigh.",+ "reasoning_effort")+ enable_thinking = body.get("enable_thinking", reasoning_effort not in (None, "none"))+ if not isinstance(enable_thinking, bool):+ raise APIError(400, "`enable_thinking` must be a boolean.", "enable_thinking")+ prompt = render_chat(body.get("messages"), enable_thinking, reasoning_effort)+ self.generation(body, prompt, request_id, True)++ def completion(self, body, request_id):+ prompt = body.get("prompt")+ if not isinstance(prompt, str):+ raise APIError(400, "Colibri currently requires `prompt` to be a string.", "prompt")+ self.generation(body, prompt, request_id, False)+++def serve(model, host="127.0.0.1", port=8000, model_id="glm-5.2-colibri", api_key=None,+ cap=8, max_tokens=1024, engine=HERE / "glm", env=None, cors_origins=None):+ if not 1 <= max_tokens:+ raise ValueError("max_tokens must be positive")+ if not 1 <= port <= 65535:+ raise ValueError("port must be between 1 and 65535")+ if host not in ("127.0.0.1", "localhost", "::1") and not api_key:+ print("WARNING: API is listening beyond localhost without COLI_API_KEY", file=sys.stderr)+ runtime = Engine(engine, model, cap, max_tokens, env)+ origins = DEFAULT_CORS_ORIGINS if cors_origins is None else tuple(cors_origins)+ server = APIServer((host, port), runtime, model_id, api_key, max_tokens, origins)+ print(f"OpenAI-compatible API listening on http://{host}:{port}/v1", file=sys.stderr)+ previous_sigterm = signal.getsignal(signal.SIGTERM)+ signal.signal(signal.SIGTERM, lambda *_: threading.Thread(target=server.shutdown, daemon=True).start())+ try:+ server.serve_forever()+ finally:+ signal.signal(signal.SIGTERM, previous_sigterm)+ server.server_close()+ runtime.close()+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--model", default=os.environ.get("COLI_MODEL"), required=not os.environ.get("COLI_MODEL"))+ parser.add_argument("--engine", default=str(HERE / "glm"))+ parser.add_argument("--host", default="127.0.0.1")+ parser.add_argument("--port", type=int, default=8000)+ parser.add_argument("--model-id", default=os.environ.get("COLI_MODEL_ID", "glm-5.2-colibri"))+ parser.add_argument("--api-key", default=os.environ.get("COLI_API_KEY"))+ parser.add_argument("--cors-origin", action="append", default=None,+ help="allowed browser origin; repeat as needed (use '*' for any origin)")+ parser.add_argument("--cap", type=int, default=8)+ parser.add_argument("--max-tokens", type=int, default=1024)+ args = parser.parse_args()+ serve(args.model, args.host, args.port, args.model_id, args.api_key,+ args.cap, args.max_tokens, args.engine, cors_origins=args.cors_origin)+++if __name__ == "__main__":+ main()diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pynew file mode 100644index 00000000..06382ea4--- /dev/null+++ b/c/tests/test_openai_server.py@@ -0,0 +1,152 @@+import io+import json+import threading+import unittest+from urllib.error import HTTPError+from urllib.request import Request, urlopen++from openai_server import APIError, APIServer, END, generation_options, read_engine_turn, render_chat+++class FakeEngine:+ def __init__(self):+ self.calls = []++ def generate(self, prompt, maximum, temperature, top_p, on_text):+ self.calls.append((prompt, maximum, temperature, top_p))+ on_text("Hé")+ on_text("llo")+ return {"prompt_tokens": 7, "completion_tokens": 2, "length_limited": False}+++class TemplateTest(unittest.TestCase):+ def test_renders_text_subset_of_official_template(self):+ prompt = render_chat([+ {"role": "system", "content": "System"},+ {"role": "developer", "content": "Developer"},+ {"role": "user", "content": [{"type": "text", "text": "Hi"}]},+ {"role": "assistant", "content": " Hello "},+ {"role": "user", "content": "Again"},+ ])+ self.assertEqual(+ prompt,+ "[gMASK]<sop><|system|>System<|system|>Developer<|user|>Hi"+ "<|assistant|><think></think>Hello<|user|>Again"+ "<|assistant|><think></think>",+ )++ def test_rejects_non_text_content(self):+ with self.assertRaisesRegex(APIError, "text message content only"):+ render_chat([{"role": "user", "content": [+ {"type": "image_url", "image_url": {"url": "x"}}+ ]}])++ def test_renders_thinking_prefix(self):+ self.assertEqual(+ render_chat([{"role": "user", "content": "Hi"}], True, "high"),+ "[gMASK]<sop><|system|>Reasoning Effort: High<|user|>Hi<|assistant|><think>",+ )++ def test_validates_generation_limits(self):+ self.assertEqual(generation_options({"max_tokens": 4, "temperature": 0, "top_p": 1}, 8),+ (4, 0.0, 1.0))+ with self.assertRaises(APIError):+ generation_options({"max_tokens": 9}, 8)+ self.assertEqual(generation_options({"temperature": None, "top_p": None}, 8),+ (8, 0.7, 0.9))+++class ProtocolTest(unittest.TestCase):+ def test_reads_payload_and_extended_status(self):+ stream = io.BytesIO(b"hello" + END + b"STAT 2 3.5 44 1.2 7 1\n")+ chunks = []+ stats = read_engine_turn(stream, END, chunks.append)+ self.assertEqual(b"".join(chunks), b"hello")+ self.assertEqual(stats["prompt_tokens"], 7)+ self.assertTrue(stats["length_limited"])+++class HTTPTest(unittest.TestCase):+ @classmethod+ def setUpClass(cls):+ cls.engine = FakeEngine()+ cls.server = APIServer(("127.0.0.1", 0), cls.engine, "test-model", "secret", 16)+ cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True)+ cls.thread.start()+ cls.base = f"http://127.0.0.1:{cls.server.server_port}"++ @classmethod+ def tearDownClass(cls):+ cls.server.shutdown()+ cls.server.server_close()+ cls.thread.join(timeout=2)++ def request(self, path, body=None, key="secret"):+ headers = {"Authorization": f"Bearer {key}"}+ data = None+ if body is not None:+ data = json.dumps(body).encode()+ headers["Content-Type"] = "application/json"+ return urlopen(Request(self.base + path, data=data, headers=headers), timeout=2)++ def test_lists_models_and_checks_auth(self):+ with self.request("/v1/models") as response:+ self.assertEqual(json.load(response)["data"][0]["id"], "test-model")+ with self.assertRaises(HTTPError) as caught:+ self.request("/v1/models", key="wrong")+ self.assertEqual(caught.exception.code, 401)++ def test_browser_preflight(self):+ request = Request(self.base + "/v1/chat/completions", method="OPTIONS", headers={+ "Origin": "http://localhost:5173",+ "Access-Control-Request-Method": "POST",+ "Access-Control-Request-Headers": "authorization,content-type",+ })+ with urlopen(request, timeout=2) as response:+ self.assertEqual(response.status, 204)+ self.assertEqual(response.headers["Access-Control-Allow-Origin"], "http://localhost:5173")+ self.assertIn("Authorization", response.headers["Access-Control-Allow-Headers"])++ def test_chat_completion(self):+ with self.request("/v1/chat/completions", {+ "model": "test-model", "messages": [{"role": "user", "content": "Hi"}],+ "max_tokens": 4,+ }) as response:+ body = json.load(response)+ self.assertEqual(body["object"], "chat.completion")+ self.assertEqual(body["choices"][0]["message"]["content"], "Héllo")+ self.assertEqual(body["usage"], {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9})+ self.assertIn("<|user|>Hi<|assistant|><think></think>", self.engine.calls[-1][0])++ def test_streaming_chat_completion(self):+ with self.request("/v1/chat/completions", {+ "model": "test-model", "messages": [{"role": "user", "content": "Hi"}],+ "stream": True, "stream_options": {"include_usage": True},+ }) as response:+ stream = response.read().decode()+ self.assertIn('\"delta\":{\"role\":\"assistant\",\"content\":\"\"}', stream)+ self.assertIn('\"object\":\"chat.completion.chunk\"', stream)+ self.assertIn('\"content\":\"Hé\"', stream)+ self.assertIn('\"usage\":{\"prompt_tokens\":7,\"completion_tokens\":2,\"total_tokens\":9}', stream)+ self.assertTrue(stream.endswith("data: [DONE]\n\n"))++ def test_legacy_completion(self):+ with self.request("/v1/completions", {+ "model": "test-model", "prompt": "Complete me", "temperature": 0,+ }) as response:+ body = json.load(response)+ self.assertEqual(body["object"], "text_completion")+ self.assertEqual(body["choices"][0]["text"], "Héllo")+ self.assertEqual(self.engine.calls[-1][0], "Complete me")++ def test_rejects_invalid_stream_options(self):+ with self.assertRaises(HTTPError) as caught:+ self.request("/v1/chat/completions", {+ "model": "test-model", "messages": [{"role": "user", "content": "Hi"}],+ "stream": True, "stream_options": "usage",+ })+ self.assertEqual(caught.exception.code, 400)+++if __name__ == "__main__":+ unittest.main()