Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
qwen-image: POST /v1/images/generations in the gateway
JustVugg397fee2f9e03
diff --git a/c/openai_server.py b/c/openai_server.pyindex 221427b3..787db467 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -22,6 +22,7 @@ import uuidimport v4_dsml # vendored DeepSeek V4 DSML reference primitivesimport v41_dsml # ...and V4.1's, whose tag names differ by a space+import image_engine # the qwenimage serve protocol, PNG and request rulesfrom family_registry import (FamilyConfigError, UnknownFamilyError, family_by_id,family_ids, resolve_model)from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer@@ -3920,6 +3921,21 @@ def model_object(model_id, created):return {"id": model_id, "object": "model", "created": created, "owned_by": "colibri"}+def is_image_engine(engine):+ """True when the server fronts a text-to-image engine rather than a chat one.+ Decided by the engine object, not by the ARCH global, so a server built in a+ test (or embedded) with an ImageEngine behaves as one."""+ return getattr(engine, "modality", "text") == "image"+++# The endpoints that only make sense for a chat model, refused with a pointer+# when the model draws images instead.+TEXT_ENDPOINTS = ("/v1/chat/completions", "/v1/completions", "/v1/messages", "/v1/brio",+ "/v1/systemone")+IMAGE_OPTION_KEYS = ("default_width", "default_height", "default_steps", "min_side",+ "max_side", "multiple")++def _positive_env(name, default):try:value = int(os.environ.get(name, "") or default)@@ -3974,6 +3990,17 @@ class APIServer(ThreadingHTTPServer):self._conn_by_ip = {}self._conn_owner = {}+ def model_entry(self):+ """The /v1/models object. An image model says so, and carries the size+ rules its engine announced, so a client can build a valid request+ without trial and error."""+ entry = model_object(self.model_id, self.created)+ if is_image_engine(self.engine):+ info = getattr(self.engine, "info", None) or {}+ entry["capabilities"] = ["image_generation"]+ entry["image"] = {key: info.get(key) for key in IMAGE_OPTION_KEYS}+ return entry+def generate(self, prompt, max_tokens, temperature, top_p, on_text, *args, **kwargs):started = time.monotonic()first_output = False@@ -4362,6 +4389,9 @@ class APIHandler(BaseHTTPRequestHandler):if tiers: payload["tiers"] = tiershwinfo = getattr(self.server.engine, "hwinfo", None) if self.server.engine else Noneif hwinfo: payload["hwinfo"] = hwinfo+ if is_image_engine(self.server.engine):+ payload["capabilities"] = ["image_generation"]+ payload["image"] = dict(getattr(self.server.engine, "info", None) or {})self.send_json(200, payload, request_id)returnif path == "/experts":@@ -4392,10 +4422,10 @@ class APIHandler(BaseHTTPRequestHandler):returnself.require_auth()if path == "/v1/models":- self.send_json(200, {"object": "list", "data": [model_object(- self.server.model_id, self.server.created)]}, request_id)+ self.send_json(200, {"object": "list", "data": [self.server.model_entry()]},+ request_id)elif path.startswith("/v1/models/") and unquote(path[11:]) == self.server.model_id:- self.send_json(200, model_object(self.server.model_id, self.server.created), request_id)+ self.send_json(200, self.server.model_entry(), request_id)else:raise APIError(404, "Not found.", None, "not_found")except APIError as error:@@ -4425,7 +4455,13 @@ class APIHandler(BaseHTTPRequestHandler):# served model answers whatever name was asked for.if path != "/v1/systemone":self.check_model(body)- if path == "/v1/chat/completions":+ if is_image_engine(self.server.engine) and path in TEXT_ENDPOINTS:+ raise APIError(400, f"`{self.server.model_id}` is an image generation model and "+ "does not chat: POST /v1/images/generations instead.",+ "model", "unsupported_endpoint")+ if path == "/v1/images/generations":+ self.image_generation(body, request_id)+ elif path == "/v1/chat/completions":self.chat_completion(body, request_id)elif path == "/v1/completions":self.completion(body, request_id)@@ -4869,6 +4905,178 @@ class APIHandler(BaseHTTPRequestHandler):"output_tokens": result["usage"]["read_tokens"]}}self.send_json(200, reply, request_id, result.get("_headers"))+ # ------------------------------------------------------------- images+ #+ # POST /v1/images/generations, the OpenAI shape plus a `colibri` block with+ # what was actually drawn (the seed above all: without it an image cannot be+ # reproduced). One image at a time through the same scheduler as chat, so+ # the queue, its limits and /metrics mean the same thing for both.+ def image_generation(self, body, request_id):+ engine = self.server.engine+ if not is_image_engine(engine):+ raise APIError(400, f"`{self.server.model_id}` does not generate images: it is a "+ "chat model (POST /v1/chat/completions).",+ "model", "unsupported_endpoint")+ try:+ request = image_engine.image_request(body, engine.info)+ except image_engine.ImageRequestError as error:+ raise APIError(400, str(error), error.param, "invalid_value")+ stream = body.get("stream", False)+ if not isinstance(stream, bool):+ raise APIError(400, "`stream` must be a boolean.", "stream")+ partial_images = body.get("partial_images")+ if partial_images is not None and (isinstance(partial_images, bool) or+ not isinstance(partial_images, int) or+ not 0 <= partial_images <= 16):+ raise APIError(400, "`partial_images` must be an integer between 0 and 16.",+ "partial_images")+ created = int(time.time())+ with contextlib.ExitStack() as stack:+ try:+ queue_wait, _slot = stack.enter_context(+ self.server.scheduler.admit(self.client_disconnected))+ except APIError as error:+ if error.status == 429:+ # The contract's word for "busy past the queue" is 503: an+ # image takes minutes, and a client that retries after a+ # second, as 429 invites, only refills the queue.+ raise APIError(503, error.message, None, error.code, "server_error",+ error.headers) from None+ raise+ if not engine.alive:+ raise APIError(503, f"The image engine is not running ({engine.dead or 'exited'}).",+ None, "engine_unavailable", "server_error")+ queue_headers = {"x-colibri-queue-wait-ms": str(round(queue_wait * 1000))}+ if stream:+ self._image_stream(engine, request, request_id, created, queue_headers,+ partial_images)+ return+ try:+ result = engine.generate(request["prompt"], request["width"], request["height"],+ request["steps"], request["seed"], preview=False,+ cancelled=self.client_disconnected)+ except image_engine.ImageCancelled:+ raise ClientCancelled() from None+ except image_engine.ImageEngineError as error:+ raise APIError(500, f"The image engine failed: {error}", None, "engine_error",+ "server_error") from None+ self.send_json(200, self._image_payload(result, created), request_id, queue_headers)++ @staticmethod+ def _image_payload(result, created):+ import base64+ png = image_engine.encode_png(result["width"], result["height"], result["rgba"], 4)+ return {"created": created,+ "data": [{"b64_json": base64.b64encode(png).decode("ascii"),+ "revised_prompt": None}],+ "colibri": {"width": result["width"], "height": result["height"],+ "seed": result["seed"], "steps": result["steps"],+ "timings": result["timings"]}}++ def _image_stream(self, engine, request, request_id, created, queue_headers,+ partial_images):+ """SSE: progress, optional partial images, completed, [DONE].++ The 200 is committed on the engine's first frame, not before: an engine+ that refuses the request outright still answers with a real HTTP error+ instead of an error event inside a stream that already said OK."""+ import base64+ lock = threading.Lock()+ state = {"started": False, "connected": True, "last": time.monotonic(), "partials": 0}++ def start():+ if state["started"]:+ return+ state["started"] = True+ self.send_response(200)+ self.send_header("Content-Type", "text/event-stream")+ self.send_header("Cache-Control", "no-cache")+ self.send_header("X-Accel-Buffering", "no")+ self.send_header("Connection", "close")+ self.close_connection = True+ self.send_header("x-request-id", request_id)+ for name, value in queue_headers.items():+ self.send_header(name, value)+ self.send_cors_headers()+ self.end_headers()++ def write(data):+ with lock:+ if not state["connected"]:+ return+ try:+ self.wfile.write(data)+ self.wfile.flush()+ state["last"] = time.monotonic()+ except OSError:+ state["connected"] = False++ def event(name, data):+ start()+ write(f"event: {name}\ndata: "+ f"{json.dumps(data, ensure_ascii=False, separators=(',', ':'))}\n\n".encode())++ def cancelled():+ return not state["connected"] or self.client_disconnected()++ def on_progress(frame):+ event("image_generation.progress",+ {key: frame.get(key) for key in ("stage", "step", "steps", "elapsed")})++ def on_preview(frame, pixels):+ limit = partial_images+ if limit is not None:+ # Spread a requested number of partial images over the run+ # instead of spending them on the first steps.+ steps = max(1, int(frame.get("steps") or request["steps"]))+ due = (state["partials"] + 1) * steps / (limit + 1)+ if state["partials"] >= limit or int(frame.get("step") or 0) < due:+ return+ channels = frame.get("channels", 3)+ try:+ png = image_engine.encode_png(frame["width"], frame["height"], pixels, channels)+ except (KeyError, TypeError, ValueError) as error:+ sys.stderr.write(f"[api] {request_id}: unusable PREVIEW frame ({error})\n")+ return+ event("image_generation.partial_image",+ {"b64_json": base64.b64encode(png).decode("ascii"),+ "partial_image_index": state["partials"]})+ state["partials"] += 1++ def on_idle():+ # SSE comments keep proxies and idle timers from closing a stream+ # that is legitimately silent through a long denoising step.+ if state["started"] and time.monotonic() - state["last"] >= 10:+ write(b": keepalive\n\n")++ try:+ result = engine.generate(request["prompt"], request["width"], request["height"],+ request["steps"], request["seed"],+ preview=partial_images != 0, on_progress=on_progress,+ on_preview=on_preview, cancelled=cancelled,+ on_idle=on_idle)+ except image_engine.ImageCancelled:+ if not state["started"]:+ raise ClientCancelled() from None+ # Once the 200 is out, every ending is an event: a client still+ # listening (the web UI) is told the image will not come, then the+ # stream closes the way every stream closes.+ event("error", {"error": {"message": "cancelled", "type": "cancelled",+ "param": None, "code": "cancelled"}})+ write(b"data: [DONE]\n\n")+ return+ except image_engine.ImageEngineError as error:+ if not state["started"]:+ raise APIError(500, f"The image engine failed: {error}", None, "engine_error",+ "server_error") from None+ event("error", {"error": {"message": f"The image engine failed: {error}",+ "type": "server_error", "param": None,+ "code": "engine_error"}})+ write(b"data: [DONE]\n\n")+ return+ event("image_generation.completed", self._image_payload(result, created))+ write(b"data: [DONE]\n\n")+def _fail(self, error, request_id):"""Report an error, unless the response is already on the wire. Once a streaming 200is committed, a second status line would be framed as SSE body -- clients saw a whole@@ -5643,7 +5851,15 @@ def serve(model, host="127.0.0.1", port=8000, model_id=None, api_key=None,if kv_slots > family.limits.max_kv_slots:raise ValueError(f"{family.id} engine supports at most "f"{family.limits.max_kv_slots} KV slot(s)")- runtime = Engine(engine,model,cap,max_tokens,env,kv_slots,family)+ if family.modality == "image":+ # Its own process class: the image engine speaks a line-and-JSON+ # protocol with binary frames, not the text engines' byte stream.+ runtime = image_engine.ImageEngine(engine, model, env=env)+ print(f"[image] {model_id}: default {runtime.info['default_width']}x"+ f"{runtime.info['default_height']}, {runtime.info['default_steps']} steps",+ file=sys.stderr)+ else:+ runtime = Engine(engine,model,cap,max_tokens,env,kv_slots,family)server.engine = runtimeprint(f"OpenAI-compatible API listening on http://{host}:{port}/v1", file=sys.stderr)signal.signal(signal.SIGTERM, lambda *_: threading.Thread(target=server.shutdown, daemon=True).start())