Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
#!/usr/bin/env python3
"""Dependency-free OpenAI-compatible HTTP gateway for the colibri engine."""
import argparse
import codecs
import collections
import contextlib
import hashlib
import json
import math
import mimetypes
import os
import select
import queue
import signal
import stat
import socket
import subprocess
import sys
import threading
import time
import uuid
import v4_dsml # vendored DeepSeek V4 DSML reference primitives
import v41_dsml # ...and V4.1's, whose tag names differ by a space
import image_engine # the qwenimage serve protocol, PNG and request rules
from family_registry import (FamilyConfigError, UnknownFamilyError, family_by_id,
display_for, family_ids, resolve_model)
from family_registry import default_model_id as registry_default_model_id
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import unquote, urlsplit
HERE = Path(__file__).resolve().parent
def default_engine(family=None):
"""Resolve the registered family's engine next to this file."""
family = family or family_by_id("glm")
names = (family.engine_artifact, *family.engine_aliases)
candidates = tuple(name + suffix for name in names for suffix in ("", ".exe"))
for name in candidates:
candidate = HERE / name
if candidate.exists():
return candidate
return HERE / family.engine_artifact
END = b"\x01\x01END\x01\x01\n"
READY = b"\x01\x01READY\x01\x01\n"
MAX_BODY = 4 << 20
PROFILE_TURNS = 120 # rolling window of per-turn PROF snapshots kept for /profile
DEFAULT_CORS_ORIGINS = (
"http://127.0.0.1:8000",
"http://localhost:8000",
"http://127.0.0.1:5173",
"http://localhost:5173",
"http://tauri.localhost",
"tauri://localhost",
)
class APIError(Exception):
def __init__(self, status, message, param=None, code=None, error_type="invalid_request_error",
headers=None):
super().__init__(message)
self.status = status
self.message = message
self.param = param
self.code = code
self.error_type = error_type
self.headers = headers or {}
class ClientCancelled(Exception):
pass
class EchoPrefixPinned(Exception):
"""The engine echoed a contiguous run of prompt positions that does not start at 0,
which is the shape a pin snapshot produces: the snapshot already covers `prefix`
tokens, so the prefill reads out `prefix..nt-1` and there is nothing to read out
before that.
This is not corruption and must not be answered as if it were, but it is also not
something this surface can serve: `echo: true` promises a logprob and a text offset for
every prompt position. Re-basing the run to start at 0 would attach every logprob to
the wrong prompt token -- a 200 carrying silently wrong data -- and filling the gap is
not available either, since the engine holds no logits for a pinned prefix. So the
caller is told by name what the engine could not do. Carries `prefix` so the message
can name it."""
def __init__(self, prefix):
super().__init__(f"the engine echoed prompt positions from {prefix}, not 0")
self.prefix = prefix
def error_object(error):
return {"error": {"message": error.message, "type": error.error_type,
"param": error.param, "code": error.code}}
def _malformed_logprob_tail(detail):
"""The engine's numeric tail did not parse, named for the client.
Every shape of it is the same event from the client's side: the engine sent a per-token
logprob frame this gateway cannot read, so the numbers the request asked for do not
exist. `engine_logprob_tail_malformed` says what the engine did rather than naming the
reaction to it, so the code stays true if the reaction ever changes."""
return APIError(500, "The colibri engine sent a malformed per-token logprob "
"tail: %s" % detail, "logprobs",
"engine_logprob_tail_malformed", "server_error")
def _malformed_echo_position(detail):
"""The `pos` field of an ECHO frame did not parse, named for the client.
Its own code rather than the numeric tail's: the two are different fields of the frame
and a client branching on the code should not be told the tail was unreadable when the
position was."""
return APIError(500, "The colibri engine sent an unreadable prompt-echo position: %s"
% detail, "echo", "engine_echo_position_malformed", "server_error")
def _with_engine_reason(fault, message):
"""`fault` again, with the engine's own terminal error text appended.
The recorded fault happened first and names the framing defect, so it is what the client
is told; the engine's parting word is kept in the message rather than dropped, because it
is often the only account of what the turn did after the frame this server could not
read."""
return APIError(fault.status, f"{fault.message} The engine then reported: {message}",
fault.param, fault.code, fault.error_type)
def _engine_error(fields, message):
"""Turn an engine ERROR frame into the right exception type.
CONTEXT_EXCEEDED is a client mistake, not a server fault: the prompt is longer than the
engine's context. Report it the way every OpenAI-compatible server does, so clients that
know how to compact a conversation actually get the chance to (previously the engine
silently truncated the prompt instead, which is #401)."""
if fields and fields[0] == "DECIDE_INVALID":
# A decision engine refusing the record: the caller's mistake, named by the
# field the engine points at ("questions.q: only 3 of its 30 option markers
# fit ..."), answered as every other /v1/systemone validation error is.
reason = " ".join(fields[1:]) or "The decision engine refused the request."
head = reason.split(":", 1)[0] if ":" in reason else ""
param = head if head and " " not in head else "questions"
return APIError(422, reason, param, "invalid_question")
if fields and fields[0] == "CONTEXT_EXCEEDED":
# Two spellings of the same frame. colibri and deepseek_v4 write the
# original `CONTEXT_EXCEEDED <used> <limit>`; qwen36 and qwen38 write
# `prompt_tokens=N requested=M capacity=C`. Reading the second by
# position took "requested=M" (the completion budget) as the limit and
# printed it raw: "maximum context length is requested=4 tokens", with
# the real ceiling nowhere (#1376). Unifying the engines' spelling is
# a separate change; the server must read both meanwhile.
kv = dict(f.split("=", 1) for f in fields[1:] if "=" in f)
if kv:
limit = kv.get("capacity") or "the context"
used = kv.get("prompt_tokens") or "?"
else:
limit = fields[2] if len(fields) > 2 else "the context"
used = fields[1] if len(fields) > 1 else "?"
return APIError(400,
f"This model's maximum context length is {limit} tokens, however your "
f"messages resulted in at least {used} tokens. Please shorten the "
f"conversation, or restart the server with a larger CTX.",
"messages", "context_length_exceeded")
return RuntimeError(message)
class GenerationScheduler:
"""Bounded FIFO admission for the engine's independent KV contexts."""
_buckets = (0.001, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60, 300, math.inf)
def __init__(self, max_queue=8, queue_timeout=300, capacity=1):
if max_queue < 0:
raise ValueError("max_queue cannot be negative")
if queue_timeout <= 0:
raise ValueError("queue_timeout must be positive")
if capacity < 1:
raise ValueError("capacity must be positive")
self.max_queue = max_queue
self.queue_timeout = queue_timeout
self.capacity = capacity
self.free_slots = set(range(capacity))
self.condition = threading.Condition()
self.queue = collections.deque()
self.active = 0
self.closed = False
self.admitted = 0
self.completed = 0
self.failed = 0
self.rejected = 0
self.timed_out = 0
self.cancelled = 0
self.timings = {name: {"sum": 0.0, "buckets": [0] * len(self._buckets)}
for name in ("queue_wait_seconds", "slot_duration_seconds",
"first_output_seconds", "engine_call_seconds")}
@contextlib.contextmanager
def admit(self, cancelled=None, slot=None):
ticket = object()
entry = (ticket, slot) # (#B2) remember each waiter's target slot for fair, per-slot admission
queued_at = time.monotonic()
with self.condition:
if self.closed:
raise APIError(503, "The inference scheduler is shutting down.", None,
"scheduler_closed", "server_error")
if self._available_slot(slot) is None and len(self.queue) >= self.max_queue:
self.rejected += 1
raise APIError(429, "The inference queue is full.", None, "queue_full",
"rate_limit_error", {"Retry-After": "1"})
self.queue.append(entry)
deadline = queued_at + self.queue_timeout
while True:
if self.closed:
self.queue.remove(entry)
self.condition.notify_all()
raise APIError(503, "The inference scheduler is shutting down.", None,
"scheduler_closed", "server_error")
if cancelled and cancelled():
self.queue.remove(entry)
self.cancelled += 1
self.condition.notify_all()
raise ClientCancelled()
remaining = deadline - time.monotonic()
if remaining <= 0:
self.queue.remove(entry)
self.timed_out += 1
self.condition.notify_all()
raise APIError(429, "Timed out waiting for the inference engine.", None,
"queue_timeout", "rate_limit_error", {"Retry-After": "1"})
available = self._available_slot(slot, ticket)
if available is not None:
break
self.condition.wait(min(remaining, 0.25))
self.queue.remove(entry)
self.free_slots.remove(available)
self.active += 1
self.admitted += 1
admitted_at = time.monotonic()
wait_seconds = admitted_at - queued_at
self._observe("queue_wait_seconds", wait_seconds)
outcome = "failed"
try:
yield wait_seconds, available
outcome = "completed"
except ClientCancelled:
outcome = "cancelled"
raise
finally:
with self.condition:
self.active -= 1
self.free_slots.add(available)
setattr(self, outcome, getattr(self, outcome) + 1)
self._observe("slot_duration_seconds", time.monotonic() - admitted_at)
self.condition.notify_all()
def _available_slot(self, slot, ticket=None):
# Caller holds the condition lock. Pinned waiters reserve only their
# target; an older any-slot waiter has priority over every free slot.
candidates = self.free_slots.copy() if slot is None else self.free_slots & {slot}
for earlier_ticket, earlier_slot in self.queue:
if earlier_ticket is ticket:
break
if earlier_slot is None:
return None
candidates.discard(earlier_slot)
return min(candidates, default=None)
def snapshot(self):
with self.condition:
return {"active": self.active, "queued": len(self.queue),
"capacity": self.capacity,
"max_queue": self.max_queue, "queue_timeout_seconds": self.queue_timeout,
"admitted": self.admitted, "completed": self.completed, "failed": self.failed,
"rejected": self.rejected, "timed_out": self.timed_out,
"cancelled": self.cancelled}
def _observe(self, name, seconds):
# Called with condition held. Cumulative buckets need no request history.
timing = self.timings[name]
timing["sum"] += seconds
for i, bound in enumerate(self._buckets):
if seconds <= bound:
timing["buckets"][i] += 1
def observe_timing(self, name, seconds):
with self.condition:
self._observe(name, seconds)
def prometheus(self):
"""One consistent, bounded snapshot; no prompt or request-ID labels."""
gauges = {"active": "Currently admitted requests.",
"queued": "Requests waiting for a KV slot.",
"capacity": "Concurrent KV slots configured.",
"max_queue": "Maximum waiting requests configured."}
counters = {"admitted": "Requests admitted to a KV slot.",
"completed": "Admitted requests that returned normally.",
"failed": "Admitted requests that raised an error.",
"rejected": "Requests rejected because the queue was full.",
"timed_out": "Requests that timed out waiting for a slot.",
"cancelled": "Requests cancelled while queued or admitted."}
lines = []
with self.condition:
for kind, fields in (("gauge", gauges), ("counter", counters)):
for field, help_text in fields.items():
name = "colibri_scheduler_" + field + ("_total" if kind == "counter" else "")
value = len(self.queue) if field == "queued" else getattr(self, field)
lines.extend((f"# HELP {name} {help_text}", f"# TYPE {name} {kind}",
f"{name} {value}"))
for field, help_text in (
("queue_wait_seconds", "Queue wait of admitted requests only."),
("slot_duration_seconds", "Slot occupancy of finished admitted requests, including errors and cancellation."),
("first_output_seconds", "Engine-call start to first nonempty text or tool callback, excluding queue wait."),
("engine_call_seconds", "Duration of finished engine generation calls, including errors and cancellation.")):
name = "colibri_scheduler_" + field
timing = self.timings[field]
lines.extend((f"# HELP {name} {help_text}", f"# TYPE {name} histogram"))
for bound, count in zip(self._buckets, timing["buckets"]):
label = "+Inf" if math.isinf(bound) else str(bound)
lines.append(f'{name}_bucket{{le="{label}"}} {count}')
lines.extend((f'{name}_sum {timing["sum"]}',
f'{name}_count {timing["buckets"][-1]}'))
return "\n".join(lines) + "\n"
def close(self):
with self.condition:
self.closed = True
self.condition.notify_all()
def content_text(content, param):
if isinstance(content, str):
return content
if not isinstance(content, list):
raise APIError(400, "Message content must be a string or an array of text parts.", param)
parts = []
for index, part in enumerate(content):
if not isinstance(part, dict) or part.get("type") not in ("text", "input_text"):
raise APIError(400, "Colibri currently supports text message content only.",
f"{param}.{index}", "unsupported_content_type")
if not isinstance(part.get("text"), str):
raise APIError(400, "Text content parts require a string `text` field.",
f"{param}.{index}.text")
parts.append(part["text"])
return "".join(parts)
# ---- GLM-5.2 tool calling -----------------------------------------------------------------
# The model expresses tool calls as ordinary text (from chat_template.jinja):
# <tool_call>{name}<arg_key>{k}</arg_key><arg_value>{v}</arg_value>...</tool_call>
# and tool results come back as <|observation|><tool_response>{content}</tool_response>.
# We render those markers into the prompt and parse them back into OpenAI `tool_calls`.
import re
BOX_START, BOX_END = "<tool_call>", "</tool_call>"
TR_OPEN, TR_CLOSE = "<tool_response>", "</tool_response>"
THINK_OPEN, THINK_CLOSE = "<think>", "</think>"
_BOX_RE = re.compile(re.escape(BOX_START) + r"(.*?)" + re.escape(BOX_END), re.DOTALL)
_ARG_RE = re.compile(r"<arg_key>([^<]*)</arg_key><arg_value>(.*?)</arg_value>", re.DOTALL)
_NAME_RE = re.compile(r"\s*([A-Za-z0-9_.\-]+)")
_TAG_RE = re.compile(r"</?arg_key>|</?arg_value>")
def _fallback_tool_preamble(tools):
"""Tool declaration for a family with no native tool tokens.
Mirrors the GLM-5.2 block because ``parse_tool_calls`` -- the parser these
families fall back to in ``parse_arch_tool_calls`` -- reads exactly that
wire format. Asking for a format the parser does not accept would produce
tool calls nobody can read back.
"""
out = ["You have access to the following functions. Call one only when it "
"is needed to answer the user.\n\n<tools>\n"]
for tool in tools:
fn = tool.get("function", tool) if isinstance(tool, dict) else {}
out.append(json.dumps(fn, ensure_ascii=False) + "\n")
out.append("</tools>\n\nTo call a function, reply with the call and nothing "
"else, in this exact format:\n" + BOX_START + "{function-name}"
"<arg_key>{arg-key}</arg_key><arg_value>{arg-value}</arg_value>"
+ BOX_END)
return "".join(out)
def _history_tool_calls(tool_calls, index):
"""A past assistant message's tool_calls, checked as _fallback_tool_calls and
_qwen38_tool_calls check them, for the renderers that read `function` without
looking: GLM, GLM-5.3 and DeepSeek V4/V4.1 raised AttributeError on a
`function` that is not an object, which do_POST answers with HTTP 500."""
if tool_calls is None:
return []
if not isinstance(tool_calls, list):
raise APIError(400, "`tool_calls` must be an array.", f"messages.{index}.tool_calls")
for position, call in enumerate(tool_calls):
where = f"messages.{index}.tool_calls.{position}"
if not isinstance(call, dict):
raise APIError(400, "Each tool call must be an object.", where)
fn = call.get("function", call)
if not isinstance(fn, dict):
raise APIError(400, "`function` must be an object.", f"{where}.function")
name = fn.get("name")
if name is not None and not isinstance(name, str):
raise APIError(400, "`function.name` must be a string.", f"{where}.function.name")
return tool_calls
def _fallback_tool_calls(tool_calls, index):
"""Render assistant tool_calls in the format parse_tool_calls() reads."""
out = []
for position, call in enumerate(tool_calls or []):
if not isinstance(call, dict):
raise APIError(400, "Each tool call must be an object.",
f"messages.{index}.tool_calls.{position}")
fn = call.get("function", call)
if not isinstance(fn, dict):
raise APIError(400, "`function` must be an object.",
f"messages.{index}.tool_calls.{position}.function")
name = fn.get("name")
if not isinstance(name, str) or not name:
raise APIError(400, "`function.name` must be a non-empty string.",
f"messages.{index}.tool_calls.{position}.function.name")
args = fn.get("arguments", "{}")
if isinstance(args, str):
try:
args = json.loads(args) if args else {}
except (json.JSONDecodeError, TypeError, ValueError):
raise APIError(400, "`function.arguments` must be a JSON object.",
f"messages.{index}.tool_calls.{position}.function.arguments")
out.append(BOX_START + name)
for key, value in (args or {}).items():
rendered = value if isinstance(value, str) else json.dumps(
value, ensure_ascii=False)
out.append(f"<arg_key>{key}</arg_key><arg_value>{rendered}</arg_value>")
out.append(BOX_END)
return "".join(out)
def _fallback_tool_result(message, index):
"""Render a role:"tool" message as prose these templates can carry."""
body = content_text(message.get("content"), f"messages.{index}.content")
return TR_OPEN + body + TR_CLOSE
# A closing tag the model started but never finished ("</tool_cal", "</tool"), at end of reply.
_PARTIAL_END_RE = re.compile(r"<(?:/(?:t(?:o(?:o(?:l(?:_(?:c(?:a(?:l)?)?)?)?)?)?)?)?)?\Z")
# De-mangler: opt-in recovery for heavily-quantized models that drop the
# <arg_key>K</arg_key><arg_value> structure. Default OFF (never rewrites well-formed output).
_SALVAGE = os.environ.get("COLI_TOOL_SALVAGE", "0") == "1"
# Families whose chat template has no tool syntax at all (OLMoE) refuse
# tools[] and role:"tool" rather than invent a format. COLI_TOOL_FALLBACK=1 opts
# into a prompt-injected translation for them: the declaration block, the prior
# assistant calls and the tool results are written as ordinary turns, in the
# same wire format parse_tool_calls() already reads back (#1378). Default OFF --
# these models were never trained on tool syntax, so this trades a clean 400 for
# output the parser may or may not recognise.
_TOOL_FALLBACK = os.environ.get("COLI_TOOL_FALLBACK", "0") == "1"
# ---- raw-stream span maps ---------------------------------------------------------------
# Every transformation between the engine's raw stream and the string a client receives is
# a DELETION: the stop filter withholds a matched sequence, the thinking split routes each
# character into one bucket or the other, the tool-call parse removes box syntax. None
# writes a character its input did not hold and none reorders, so a stage can report what
# it did as an ordered list of surviving intervals -- [(in_start, in_end, out_start), ...],
# monotonic and non-overlapping -- in ITS OWN INPUT's coordinates. Output positions are
# assigned in input order with no gaps, which keeps the image of a contiguous range
# contiguous and composition associative.
def _append_span(spans, in_start, in_end, out_start):
"""Append one surviving interval, merging it into the previous one when the two are
adjacent on both sides so the map stays canonical."""
if in_end <= in_start:
return
if spans:
previous_start, previous_end, previous_out = spans[-1]
if previous_end == in_start and previous_out + (previous_end - previous_start) == out_start:
spans[-1] = (previous_start, in_end, previous_out)
return
spans.append((in_start, in_end, out_start))
def _cut_span_map(length, cuts):
"""The deletion map for removing `cuts` -- possibly overlapping [start, end) ranges --
from a string of `length` characters.
The cuts are sorted here rather than required to arrive sorted: every caller today
produces them in order, and an out-of-order list silently produced a wrong map, which
is the kind of precondition a later caller discovers the expensive way."""
spans, cursor, out = [], 0, 0
for start, end in sorted(cuts):
if start > cursor:
_append_span(spans, cursor, start, out)
out += start - cursor
cursor = max(cursor, end)
if cursor < length:
_append_span(spans, cursor, length, out)
return spans
def _apply_cuts(text, cuts):
"""`(remaining text, deletion map)` for removing `cuts` from `text`. One step of a
multi-step stage: the caller composes the steps' maps in the same order it applies
them, because each step's cuts are found in the text the previous step produced."""
spans = _cut_span_map(len(text), cuts)
return "".join(text[start:end] for start, end, _out in spans), spans
def _compose_span_maps(first, second):
"""Chain a stage's map with the map of the stage that consumes its output: `first` is
X -> Y, `second` is Y -> Z, and the result is X -> Z."""
composed, index = [], 0
for in_start, in_end, out_start in first:
middle_end = out_start + (in_end - in_start)
while index < len(second) and second[index][1] <= out_start:
index += 1 # `first` is monotonic in Y, so this never rewinds
probe = index
while probe < len(second) and second[probe][0] < middle_end:
middle_start, middle_stop, final_out = second[probe]
low, high = max(out_start, middle_start), min(middle_end, middle_stop)
if low < high:
_append_span(composed, in_start + (low - out_start), in_start + (high - out_start),
final_out + (low - middle_start))
probe += 1
return composed
def _project_point(span_map, position):
"""How many input characters strictly before `position` survive `span_map`."""
low, high = 0, len(span_map)
while low < high:
middle = (low + high) // 2
if span_map[middle][0] <= position:
low = middle + 1
else:
high = middle
if low == 0:
return 0
in_start, in_end, out_start = span_map[low - 1]
return out_start + min(position, in_end) - in_start
def _project_span(span_map, start, end):
"""Where input range [start, end) lands in the stage's output, or None when every
character of it was deleted.
One interval, not a list: a deletion map assigns output positions in input order with
no gaps, so even a deletion inside the range leaves a contiguous image -- the survivors
on either side of the hole become adjacent once the hole is gone."""
low, high = _project_point(span_map, start), _project_point(span_map, end)
return (low, high) if high > low else None
# `tool_choice: "required"` as one line of prompt, for every renderer that has a tool
# block to put it after. It is a prompt-level instruction and nothing more: no
# renderer here constrains sampling, so a model that answers in prose anyway has done
# nothing the API promised would be impossible. Grammar forcing is not a remedy -- that
# path feeds a draft the engine then verifies, so it degrades to "no speedup" rather
# than to an enforced tool call.
#
# One constant, because it used to live in three copies (GLM-5.2, DeepSeek V4,
# DeepSeek V4.1) and be absent from four renderers that accept `required`, render the
# tools and then drop the instruction on the floor -- the "accepted in silence" outcome
# #1698 rules out. A renderer with no tool block to attach it to (Inkling, OLMoE
# without COLI_TOOL_FALLBACK) answers HTTP 400 instead, the honest third outcome. Kimi
# K3 spells its own, because its wire format carries a dedicated tool-choice message
# rather than prose.
TOOL_CHOICE_REQUIRED_INSTRUCTION = (
"\n\nYou must call one of the functions above. Do not answer directly.")
def _tool_choice_name(tool_choice):
"""The tool name a dict `tool_choice` forces, or None.
`.get` is taken only from a dict. Writing the name where the object goes --
{"type": "function", "function": "search"} instead of
{"function": {"name": "search"}} -- raised AttributeError in the five
renderers that read this and in generation_options() itself, and do_POST
answered HTTP 500 "The colibri engine failed to process the request." for a
payload generation_options() already has a 400 for. Same shape as the
json_schema fix (#1587): read the member, then check it.
"""
function = tool_choice.get("function")
return ((function if isinstance(function, dict) else {}).get("name")
or tool_choice.get("name"))
def _tool_function(tool):
"""The function object on a tools[] entry, or {} if it is missing or not an object.
OpenAI dual spelling: {"function": {"name": ...}} or a bare function object.
.items() is taken only from a dict. Writing the name where the object goes
({"type": "function", "function": "search"}) raised AttributeError in the GLM
and DeepSeek declaration blocks, and do_POST answered HTTP 500 "The colibri
engine failed to process the request." for a payload generation_options()
already has a 400 for. Same shape as the tool_choice fix (#1598): read the
member, then check it.
"""
fn = tool.get("function", tool) if isinstance(tool, dict) else {}
return fn if isinstance(fn, dict) else {}
def _tool_param_order(tools):
"""name -> ordered param names (required first) from the request schema, for de-mangling."""
out = {}
for tool in (tools or []):
fn = tool.get("function", tool) if isinstance(tool, dict) else {}
name = fn.get("name")
if not name:
continue
params = ((fn.get("parameters") or {}).get("properties") or {})
required = list((fn.get("parameters") or {}).get("required") or [])
out[name] = required + [p for p in params if p not in required]
return out
def _tool_param_types(tools):
"""name -> {param: declared JSON-schema type}. The model emits every argument as text;
without the schema a string-typed value that happens to look numeric ("12345" for an
order id, an SKU, a phone number) would be json.loads()'d into an int and the tool would
receive the wrong type."""
out = {}
for tool in (tools or []):
fn = tool.get("function", tool) if isinstance(tool, dict) else {}
name = fn.get("name")
if not name:
continue
props = ((fn.get("parameters") or {}).get("properties") or {})
types = {}
for key, spec in props.items():
if isinstance(spec, dict):
t = spec.get("type")
if isinstance(t, list): # {"type": ["string", "null"]}
t = next((x for x in t if x != "null"), None)
types[key] = t
out[name] = types
return out
def _coerce_arg(value, declared):
"""Decode a raw <arg_value> according to the declared schema type.
A string-typed parameter is kept verbatim -- never parsed as JSON. Everything else keeps
the previous permissive behaviour (parse if it parses, otherwise leave as text)."""
if declared == "string":
return value
try:
parsed = json.loads(value)
except (json.JSONDecodeError, TypeError):
return value
if declared in ("integer", "number") and isinstance(parsed, bool):
return value # `true` is not a number
if declared and declared not in ("integer", "number", "boolean", "object", "array"):
return value
return parsed
def _unclosed_tail(reply, tools):
"""Body of a trailing <tool_call> that was never closed, or None.
Only returned when the recovery is unambiguous, so ordinary prose that merely mentions
"<tool_call>" can never be turned into a call. Both conditions must hold:
* the last BOX_START is not followed by a BOX_END (a closed box is the strict parser's job);
* the tail carries a complete <arg_key>..</arg_value> pair, OR it is exactly the name of a
tool the client declared (the zero-argument case).
"""
start = reply.rfind(BOX_START)
if start < 0 or BOX_END in reply[start:]:
return None
inner = _PARTIAL_END_RE.sub("", reply[start + len(BOX_START):])
if _ARG_RE.search(inner):
return inner
declared = {(t.get("function", t) if isinstance(t, dict) else {}).get("name")
for t in (tools or []) if isinstance(t, dict)}
return inner if inner.strip() in declared else None
def _marker_cuts(text, marker):
"""Every non-overlapping occurrence of `marker`, left to right -- what
str.replace(marker, "") removes, found as ranges instead of applied as a rewrite."""
if not marker:
return [] # find("") never advances past `index`
cuts, index = [], text.find(marker)
while index >= 0:
cuts.append((index, index + len(marker)))
index = text.find(marker, index + len(marker))
return cuts
def _tool_call_content_spans(reply, tail, track_spans=True):
"""parse_tool_calls' content derivation as `(content, box_map, content_map)`, both maps
in `reply`'s own coordinates.
Every step is a deletion or a slice, so the whole derivation is one composed map. The
steps are applied in order and composed in the same order, because each step's cuts are
found in the text the previous step produced. `box_map` stops after the tool-call
removals, which is what tells a character consumed as tool-call syntax apart from one
consumed as thinking markup.
Both maps are None for inkling, whose marker stripping is not modelled here: an inkling
engine does not support the numeric logprobs channel, so no logprobs object can reach
this text to be aligned. They are also None when `track_spans` is off, which is how a
request that never asked for logprobs skips composing them.
The per-step survivor lists `_apply_cuts` returns are not gated, because they are the
deletion itself -- the text is built by slicing them -- rather than a map kept for a
later reader. Only the composition and the exported maps are optional."""
def chain(previous, step):
return previous and track_spans and _compose_span_maps(previous, step)
cuts = [(match.start(), match.end()) for match in _BOX_RE.finditer(reply)]
text, box_map = _apply_cuts(reply, cuts)
if tail is not None: # drop the recovered tail from the visible content
text, step = _apply_cuts(text, [(text.rindex(BOX_START), len(text))])
box_map = chain(box_map, step)
content_map = box_map
if ARCH == "inkling":
text = strip_inkling_markers(text) # thinking is reasoning, not answer
box_map = content_map = None
if THINK_CLOSE in text:
text, step = _apply_cuts(text, [(0, text.index(THINK_CLOSE) + len(THINK_CLOSE))])
content_map = chain(content_map, step)
for marker in (THINK_OPEN, THINK_CLOSE):
text, step = _apply_cuts(text, _marker_cuts(text, marker))
content_map = chain(content_map, step)
stripped = text.strip()
head = len(text) - len(text.lstrip())
text, step = _apply_cuts(text, [(0, head), (head + len(stripped), len(text))])
content_map = chain(content_map, step)
if not track_spans:
return text, None, None
return text, box_map, content_map
def parse_tool_calls(reply, tools=None):
"""Return (content, tool_calls). Strict GLM parse; optional de-mangler (COLI_TOOL_SALVAGE=1)
rescues malformed int4 output by mapping a lone payload onto the tool's primary parameter."""
content, calls, _box_map, _content_map = _parse_tool_calls(reply, tools, False)
return content, calls
def parse_glm_tool_calls_strict(reply, tools):
"""Parse complete GLM calls against declared schemas, without recovery or salvage.
A client can opt into this when executing a malformed call would be worse than
receiving a named model-output error. The default parser remains permissive.
"""
def invalid(reason):
raise APIError(502, "Invalid GLM tool call: " + reason, code="invalid_model_tool_call",
error_type="server_error")
declared = {_tool_function(tool).get("name"): _tool_function(tool).get("parameters") or {}
for tool in tools}
calls = []
boxes = list(_BOX_RE.finditer(reply))
outside = _BOX_RE.sub("", reply)
if re.search(r"</?tool_call|</?arg_(?:key|value)", outside):
invalid("incomplete or stray tool marker")
for box in boxes:
inner = box.group(1)
name_match = _NAME_RE.match(inner)
if not name_match or name_match.group(1) not in declared:
invalid("undeclared function")
name = name_match.group(1)
params = declared[name]
properties = params.get("properties") or {}
args = {}
cursor = name_match.end()
while inner[cursor:].strip():
prefix = len(inner[cursor:]) - len(inner[cursor:].lstrip())
cursor += prefix
match = _ARG_RE.match(inner, cursor)
if not match:
invalid("malformed argument syntax")
key, value = match.groups()
if key not in properties or key in args or re.search(r"</?arg_(?:key|value)|</?tool_call", value):
invalid("unknown, duplicate, or malformed argument")
spec = properties[key] if isinstance(properties[key], dict) else {}
declared_type = spec.get("type")
if isinstance(declared_type, list):
declared_type = next((item for item in declared_type if item != "null"), None)
parsed = _coerce_arg(value, declared_type)
valid = (declared_type in (None, "string") or
(declared_type == "integer" and type(parsed) is int) or
(declared_type == "number" and type(parsed) in (int, float)) or
(declared_type == "boolean" and type(parsed) is bool) or
(declared_type == "array" and isinstance(parsed, list)) or
(declared_type == "object" and isinstance(parsed, dict)))
if not valid:
invalid("argument does not match its declared type")
args[key] = parsed
cursor = match.end()
if any(key not in args for key in params.get("required") or []):
invalid("missing required argument")
calls.append({"id": "call_" + uuid.uuid4().hex[:24], "type": "function",
"function": {"name": name, "arguments": json.dumps(args, ensure_ascii=False)}})
content, _box_map, _content_map = _tool_call_content_spans(reply, None, False)
return content, calls
def parse_tool_calls_spans(reply, tools=None):
"""parse_tool_calls plus the tool-call stage's maps."""
return _parse_tool_calls(reply, tools, True)
def _parse_tool_calls(reply, tools, track_spans):
"""The parse itself: `(content, tool_calls, box_map, content_map)`. One implementation
behind both spellings, so the maps cannot drift from the text they describe."""
param_order = _tool_param_order(tools)
param_types = _tool_param_types(tools)
calls, salvaged = [], []
# #401: a box the model opened but never closed -- it ran out of budget, or the closing tag
# came out mangled ("</tool_cal"). The call itself is often perfectly well-formed, but the
# strict regex needs BOTH tags, so the client used to get *zero* tool_calls. Recover the tail,
# but only when it is unambiguous (see _unclosed_tail) so prose can never fabricate a call.
boxes = [m.group(1) for m in _BOX_RE.finditer(reply)]
tail = _unclosed_tail(reply, tools)
if tail is not None:
boxes.append(tail)
for inner in boxes:
name_match = _NAME_RE.match(inner)
name = name_match.group(1) if name_match else inner.strip()
args = {}
types = param_types.get(name, {})
for arg in _ARG_RE.finditer(inner):
key, value = arg.group(1), arg.group(2)
args[key] = _coerce_arg(value, types.get(key))
if not args and _SALVAGE:
rest = inner[name_match.end():] if name_match else ""
payload = _TAG_RE.sub("", rest).strip()
if payload.startswith("(") and payload.endswith(")"):
payload = payload[1:-1].strip()
if payload:
key = (param_order.get(name) or ["input"])[0]
try:
payload = json.loads(payload)
except (json.JSONDecodeError, TypeError, ValueError):
pass
args = {key: payload}
salvaged.append(name)
calls.append({"id": "call_" + uuid.uuid4().hex[:24], "type": "function",
"function": {"name": name, "arguments": json.dumps(args, ensure_ascii=False)}})
if tools and not calls and re.search(r"</?tool_call>|</?arg_key>|</?arg_value>", reply):
# Diagnosi per la #401: il client ha dichiarato i tools e il modello ha PROVATO la
# sintassi, ma il parse rigoroso non ha agganciato nulla (tipico output int4 storpiato).
# EN: #401 field diagnosis: tools were declared and the model attempted the syntax,
# EN: but the strict parse matched nothing (typically quantization-mangled output).
sys.stderr.write("[api] tools declared and tool-call markers present, but no call "
"parsed -- output may be quantization-mangled; try COLI_TOOL_SALVAGE=1\n")
sys.stderr.flush()
text, box_map, content_map = _tool_call_content_spans(reply, tail, track_spans)
if calls:
dm, rec = len(salvaged), (1 if tail is not None else 0)
sys.stderr.write("[api] tool-calls: %d total, %d strict, %d unclosed-recovered, "
"%d de-mangled [%s]%s\n"
% (len(calls), max(0, len(calls) - dm - rec), rec, dm,
"CLEAN" if dm == 0 and rec == 0 else "RECOVERED",
(" -> " + ", ".join(salvaged)) if dm else ""))
sys.stderr.flush()
return text, calls, box_map, content_map
# ---- DeepSeek V4 tool calling (DSML) -------------------------------------------------------
# V4 expresses tool calls as DSML blocks (see encoding/encoding_dsv4.py):
# <|DSML|tool_calls>\n<|DSML|invoke name="fn">\n
# <|DSML|parameter name="k" string="true">v</|DSML|parameter>\n</|DSML|invoke>\n</|DSML|tool_calls>
# preceded by "\n\n". There is no standalone "tool" role: results are <tool_result>{content}
# </tool_result> blocks merged into the following user turn. DSML = U+FF5C (|) + ASCII.
DSV4_DSML = v4_dsml.dsml_token
DSV4_EOS = v4_dsml.eos_token
# OpenAI-style reasoning_effort levels -> the V4 vocabulary (encoding_dsv4.py has only
# low/high/max; `low` adds nothing). In thinking mode the level's prompt is prepended at the
# very start of the conversation, byte-matching REASONING_EFFORT_PROMPTS.
DSV4_REASONING_EFFORT = {"minimal": "low", "low": "low", "medium": "high",
"high": "high", "xhigh": "max", "max": "max"}
DSV4_REASONING_EFFORT_PROMPTS = {
"high": ("Reasoning Effort: High.\n"
"Reason thoroughly, decompose the problem, and verify the relevant edge cases before acting. "
"Avoid repeating settled points or narrating redundant alternatives. "
"Keep the analysis proportional to the task. HARD LIMIT: finish reasoning within about "
"1,500 tokens, close the thinking section, and then emit the next tool call or a complete "
"final response. Never consume the whole output budget with reasoning.\n\n"),
"max": ("Reasoning Effort: Maximum.\n"
"Analyze the problem with maximum depth, trace root causes, and independently verify the "
"solution from multiple relevant angles. Do not repeat settled reasoning or pursue "
"irrelevant branches. Reserve sufficient tokens for the required tool call or final "
"response, and always terminate reasoning before the token budget is exhausted.\n\n"),
}
def _dsv4_tools_block(tools):
"""V4 tool-declaration block, rendered by the vendored reference template."""
schemas = []
for tool in (tools or []):
fn = _tool_function(tool)
# Gateway-side scrub: OpenAI clients attach routing hints the model
# schema must not carry.
schemas.append({k: v for k, v in fn.items() if k not in ("defer_loading", "strict")})
return v4_dsml.render_tools(schemas)
def _dsv4_tool_calls(tool_calls):
"""Render OpenAI-format tool_calls into a V4 DSML block (incl. the leading
)."""
return v4_dsml.render_tool_calls(tool_calls)
def parse_dsv4_tool_calls(reply):
"""Parse DeepSeek V4 DSML tool calls out of one assistant reply.
The block itself is decoded by the vendored reference parser (strict, so a
malformed block degrades to no calls instead of half-parsed arguments).
Gateway hardening on top: an incomplete block (e.g. length-truncated
output) is cut from the visible content so raw DSML syntax never leaks,
and any thinking/eos markers around the block are scrubbed.
"""
content, calls = v4_dsml.parse_completion_text(reply)
if not calls:
cut = len(content)
for marker in ("<" + DSV4_DSML + "tool_calls", "<" + DSV4_DSML + "invoke"):
pos = content.find(marker)
if 0 <= pos < cut:
cut = pos
if cut < len(content):
content = content[:cut]
for marker in (DSV4_EOS, THINK_OPEN, THINK_CLOSE):
content = content.replace(marker, "")
return content.strip(), calls
# K3 tool-call XTML carried on the engine-authenticated TOOL sideband (#1147).
# Older #1144 engines placed the same bytes on DATA; parse_arch_tool_calls keeps
# that path only when a request did not declare an authoritative sideband.
K3_TOOLS_OPEN = "<|open|>tools<|sep|>"
_K3_TOOLS_RE = re.compile(r"<\|open\|>tools<\|sep\|>(.*?)<\|close\|>tools<\|sep\|>", re.DOTALL)
_K3_CALL_RE = re.compile(
r'<\|open\|>call tool="([^"]*)" index="(\d+)"<\|sep\|>(.*?)<\|close\|>call<\|sep\|>', re.DOTALL)
_K3_ARG_RE = re.compile(
r'<\|open\|>argument key="([^"]*)" type="([^"]*)"<\|sep\|>(.*?)<\|close\|>argument<\|sep\|>',
re.DOTALL)
_K3_JSON_RE = re.compile(r'<\|open\|>json type="object"<\|sep\|>(.*?)<\|close\|>json<\|sep\|>',
re.DOTALL)
def _k3_unescape_attr(s):
return s.replace(""", '"').replace("&", "&") # & last, mirroring the escape
def parse_k3_tool_calls(reply, tools=None):
"""Return (content, tool_calls) from K3's engine-proven XTML tool block."""
calls = []
blocks = [m.group(1) for m in _K3_TOOLS_RE.finditer(reply)]
text = _K3_TOOLS_RE.sub("", reply)
# An opened-but-unclosed tools block (token budget ran out): parse the complete
# calls inside it, drop the fragment from the visible text — same recovery
# posture as the GLM path's _unclosed_tail.
tail_at = text.rfind(K3_TOOLS_OPEN)
if tail_at >= 0:
blocks.append(text[tail_at + len(K3_TOOLS_OPEN):])
text = text[:tail_at]
for block in blocks:
for m in _K3_CALL_RE.finditer(block):
name = _k3_unescape_attr(m.group(1))
inner = m.group(3)
jm = _K3_JSON_RE.search(inner)
if jm is not None:
arguments = jm.group(1)
else:
args = {}
for am in _K3_ARG_RE.finditer(inner):
key = _k3_unescape_attr(am.group(1))
typ, val = am.group(2), am.group(3)
if typ == "string":
args[key] = val
else:
try:
args[key] = json.loads(val)
except (json.JSONDecodeError, ValueError):
args[key] = val
arguments = json.dumps(args, ensure_ascii=False)
calls.append({"id": "call_" + uuid.uuid4().hex[:24], "type": "function",
"function": {"name": name, "arguments": arguments}})
if THINK_CLOSE in text:
text = text.split(THINK_CLOSE, 1)[1]
text = text.replace(THINK_OPEN, "").replace(THINK_CLOSE, "")
if calls:
sys.stderr.write("[api] tool-calls: %d total (kimi_k3 XTML)\n" % len(calls))
sys.stderr.flush()
elif tools and K3_TOOLS_OPEN in reply:
sys.stderr.write("[api] K3 tool markers present but no call parsed -- "
"possibly truncated or mangled output\n")
sys.stderr.flush()
return text.strip(), calls
def parse_dsv41_tool_calls(reply):
"""Parse DeepSeek V4.1 DSML tool calls out of one assistant reply.
Same posture as the V4 path: the vendored reference parser decodes the block
strictly, and the gateway then cuts an incomplete block out of the visible
content so raw DSML never reaches the client, whatever the model did with
its token budget.
"""
content, calls = v41_dsml.parse_completion_text(reply)
if not calls:
cut = len(content)
for marker in (v41_dsml.TOOL_CALLS_PREFIX, v41_dsml.TOOL_CALL_PREFIX):
pos = content.find(marker)
if 0 <= pos < cut:
cut = pos
if cut < len(content):
content = content[:cut]
for marker in (v41_dsml.eos_token, THINK_OPEN, THINK_CLOSE):
content = content.replace(marker, "")
return content.strip(), calls
def parse_arch_tool_calls(reply, tools, tool_reply=None):
"""Architecture-appropriate tool-call parser. Returns (content, tool_calls)."""
content, calls, _box_map, _content_map = _parse_arch_tool_calls(
reply, tools, tool_reply, False)
return content, calls
def parse_arch_tool_calls_spans(reply, tools, tool_reply=None):
"""parse_arch_tool_calls plus the tool-call stage's maps."""
return _parse_arch_tool_calls(reply, tools, tool_reply, True)
def _parse_arch_tool_calls(reply, tools, tool_reply, track_spans):
"""parse_arch_tool_calls plus the tool-call stage's maps: `(content, tool_calls,
box_map, content_map)`.
Only the glm/default and mimo parsers report maps; the others return None for both. A
request that opted into the numeric logprobs channel is refused with a named 400 on any
other architecture, so no reply reaching those branches can carry a logprobs object for
a map to align."""
if ARCH == "deepseek_v4":
return parse_dsv4_tool_calls(reply) + (None, None)
if ARCH == "deepseek_v41":
return parse_dsv41_tool_calls(reply) + (None, None)
if ARCH == "kimi":
if tool_reply is not None:
_sideband_text, calls = parse_k3_tool_calls(tool_reply, tools)
return reply.strip(), calls, None, None
return parse_k3_tool_calls(reply, tools) + (None, None) # pre-#1147 engines
if chat_flavor() in ("qwen36", "qwen38", "qwen3_coder"):
return parse_qwen_tool_calls(reply, tools) + (None, None)
if ARCH == "mimo":
return _parse_mimo_tool_calls(reply, tools, track_spans)
return _parse_tool_calls(reply, tools, track_spans)
def _tool_stream_markers():
"""Marker(s) that open a model tool-call block, in match order (arch-specific)."""
if ARCH == "deepseek_v4":
return ("<" + DSV4_DSML + "tool_calls", "<" + DSV4_DSML + "invoke")
if ARCH == "deepseek_v41":
# V4.1's tag names lead with a space (" calls", " invoke"): building these the
# V4 way would suppress nothing and leak the block into delta.content.
return (v41_dsml.TOOL_CALLS_PREFIX, v41_dsml.TOOL_CALL_PREFIX)
if ARCH == "kimi":
return (K3_TOOLS_OPEN,)
return (BOX_START,)
def _tool_cut(buf):
"""Earliest position of a tool-call marker in buf, or -1."""
found = -1
for marker in _tool_stream_markers():
pos = buf.find(marker)
if pos >= 0 and (found < 0 or pos < found):
found = pos
return found
def _tool_hold():
"""Bytes to hold back while scanning for a tool-call marker split across chunks."""
return max(len(m) for m in _tool_stream_markers()) - 1
ARCH = "glm" # set in main(): a family id from family_registry (glm | inkling |
# kimi | olmoe | qwen36 | qwen38 | deepseek_v4)
# The chat template the model was trained on, when it is not its engine family's (#1757):
# Qwen3.8-27B is a dense Qwen3.5-architecture model, so the qwen36 engine runs it, but it
# ships Qwen3.8's chat_template.jinja (the same file, sha256 c3cf9e34, that the qwen38
# renderer is pinned to): xhigh reasoning by default, the XML tool-call form, history
# that keeps its thinking. None means "the family's own", which is what every other
# checkpoint is. Only rendering follows it; what the engine can do (images) stays ARCH's.
CHAT_FLAVOR = None
def chat_flavor():
return CHAT_FLAVOR or ARCH
def detect_chat_flavor(family_id, model_dir):
"""The template a checkpoint ships, when it differs from its family's (see CHAT_FLAVOR)."""
if family_id != "qwen36" or not model_dir:
return None
for name in ("chat_template.jinja", "tokenizer_config.json"):
try:
text = (Path(model_dir) / name).read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError):
continue
# the line that makes a template Qwen3.8's: reasoning is on by default, at xhigh
if "reasoning_effort|default('xhigh')" in text:
return "qwen38"
# Qwen3-Coder (qwen3_moe): its own XML tool block, no thinking
if "interact with a computer to solve tasks" in text:
return "qwen3_coder"
return None
INK_THINK, INK_TEXT = "<|content_thinking|>", "<|content_text|>"
class InklingStreamSplit:
"""Strips Inkling's content markers from the visible stream and withholds
<|content_thinking|> sections from `content` (they are reasoning, not
answer). Buffers partial markers across chunk boundaries so a marker split
between two DATA frames never leaks."""
def __init__(self, on_content, on_reasoning=None, on_reasoning_end=None):
self.on_content = on_content
self.on_reasoning = on_reasoning
self.on_reasoning_end = on_reasoning_end
self.mode = "content"
self.buf = ""
def feed(self, piece):
self.buf += piece
while True:
hits = [(i, m) for i, m in ((self.buf.find(INK_THINK), INK_THINK),
(self.buf.find(INK_TEXT), INK_TEXT)) if i >= 0]
if not hits:
hold = self._tail_hold()
out = self.buf[:len(self.buf) - hold] if hold else self.buf
self.buf = self.buf[len(self.buf) - hold:] if hold else ""
self._emit(out)
return
i, m = min(hits)
self._emit(self.buf[:i])
if m == INK_TEXT and self.mode == "reasoning" and self.on_reasoning_end:
self.on_reasoning_end()
self.mode = "reasoning" if m == INK_THINK else "content"
self.buf = self.buf[i + len(m):]
def _tail_hold(self):
for k in range(min(len(self.buf), 24), 0, -1):
if INK_THINK.startswith(self.buf[-k:]) or INK_TEXT.startswith(self.buf[-k:]):
return k
return 0
def _emit(self, text):
if not text:
return
text = _INK_MARKER.sub("", text)
if not text:
return
if self.mode == "content":
self.on_content(text)
elif self.on_reasoning:
self.on_reasoning(text)
def close(self):
self._emit(self.buf)
self.buf = ""
import re as _re
_INK_MARKER = _re.compile(r"<\|(?:content_\w+|end_message|message_\w+|audio_end|unused_\d+)\|>")
def strip_inkling_markers(text):
"""Remove <|content_thinking|>…<|content_text|> sections, then any stray
control markers (end_message, role/content tokens) the model emits."""
while INK_THINK in text:
pre, _, rest = text.partition(INK_THINK)
_, _, after = rest.partition(INK_TEXT)
text = pre + after
return _INK_MARKER.sub("", text)
def split_inkling(text):
"""Split raw Inkling output into (content, reasoning). Thinking blocks
(<|content_thinking|>…<|content_text|>) become reasoning — including an
UNTERMINATED trailing block (budget ran out mid-thought), which partitions
to everything after the opener — so a think-only generation surfaces its
reasoning instead of collapsing to an empty answer."""
reasoning = []
while INK_THINK in text:
pre, _, rest = text.partition(INK_THINK)
think, _, after = rest.partition(INK_TEXT)
reasoning.append(think)
text = pre + after
return _INK_MARKER.sub("", text), _INK_MARKER.sub("", "".join(reasoning))
# ---- Inkling DMel audio input ------------------------------------------------------------
# Inkling takes audio as discretized log-mel frames ("DMel"): 80 slaney mel
# bands per 50 ms hop, quantized to 16 levels in log10 [-7, 2]. One frame = one
# <|audio|> placeholder token; the engine swaps in the frame's embedding at that
# position. The DSP below matches tml-renderers 0.1.0 (via tinkernel-audio,
# which is byte-golden against the official wheel): 100 ms periodic-Hann window
# centered on i*hop with zero edge padding, magnitude-domain mel projection,
# and a turn-level RMS boost for quiet audio (rms < 0.01).
AUDIO_SAMPLE_RATE = 16_000
AUDIO_HOP = 800
AUDIO_WINDOW = 1_600
AUDIO_MEL_BANDS = 80
AUDIO_DMEL_LEVELS = 16
AUDIO_DMEL_MIN, AUDIO_DMEL_MAX = -7.0, 2.0
AUDIO_RMS_FLOOR = 0.01
AUDIO_LOG_FLOOR = 1.0e-10
_MEL_FILTERS = None
def _np():
try:
import numpy
except ImportError:
raise APIError(400, "Audio input needs numpy on the gateway (pip install numpy).",
None, "unsupported_content_type")
return numpy
def _mel_filters(np):
"""Slaney mel filter bank, [80, 801], normalization 2/(upper-lower)."""
global _MEL_FILTERS
if _MEL_FILTERS is not None:
return _MEL_FILTERS
fft_freqs = np.arange(AUDIO_WINDOW // 2 + 1, dtype=np.float64) * AUDIO_SAMPLE_RATE / AUDIO_WINDOW
def hz_to_mel(hz):
hz = np.asarray(hz, dtype=np.float64)
return np.where(hz >= 1000.0, 15.0 + np.log(np.maximum(hz, 1e-30) / 1000.0) / 0.06875177742094912,
hz / 66.66666666666667)
def mel_to_hz(mel):
mel = np.asarray(mel, dtype=np.float64)
return np.where(mel >= 15.0, 1000.0 * np.exp(0.06875177742094912 * (mel - 15.0)),
66.66666666666667 * mel)
max_mel = hz_to_mel(AUDIO_SAMPLE_RATE / 2.0)
mel_points = mel_to_hz(np.linspace(0.0, float(max_mel), AUDIO_MEL_BANDS + 2))
lower, center, upper = mel_points[:-2], mel_points[1:-1], mel_points[2:]
rising = (fft_freqs[None, :] - lower[:, None]) / (center - lower)[:, None]
falling = (upper[:, None] - fft_freqs[None, :]) / (upper - center)[:, None]
weights = np.maximum(np.minimum(rising, falling), 0.0) * (2.0 / (upper - lower))[:, None]
_MEL_FILTERS = weights.astype(np.float32)
return _MEL_FILTERS
def dmel_encode(samples):
"""Mono 16 kHz f32 PCM -> u8 DMel bytes, [ceil(n/800), 80] row-major."""
np = _np()
samples = np.asarray(samples, dtype=np.float32)
n = samples.shape[0]
if n == 0:
raise APIError(400, "Audio clip is empty.", None, "invalid_value")
frames = -(-n // AUDIO_HOP)
half = AUDIO_WINDOW // 2
padded = np.zeros(half + frames * AUDIO_HOP + half, dtype=np.float32)
padded[half:half + n] = samples
idx = (np.arange(frames)[:, None] * AUDIO_HOP) + np.arange(AUDIO_WINDOW)[None, :]
hann = (0.5 - 0.5 * np.cos(2.0 * np.pi * np.arange(AUDIO_WINDOW, dtype=np.float64)
/ AUDIO_WINDOW)).astype(np.float32)
windows = padded[idx] * hann[None, :]
sqmag = np.abs(np.fft.rfft(windows, axis=1)) ** 2 # [frames, 801]
rms = math.sqrt(float(np.sum(samples.astype(np.float64) ** 2)) / n)
scale = AUDIO_RMS_FLOOR / rms if 0.0 < rms < AUDIO_RMS_FLOOR else 1.0
mag = np.sqrt(np.maximum(sqmag * (scale * scale), AUDIO_LOG_FLOOR)).astype(np.float32)
energy = mag @ _mel_filters(np).T # [frames, 80]
logmel = np.log10(np.maximum(energy, AUDIO_LOG_FLOOR))
norm = np.clip((np.clip(logmel, AUDIO_DMEL_MIN, AUDIO_DMEL_MAX) - AUDIO_DMEL_MIN)
/ (AUDIO_DMEL_MAX - AUDIO_DMEL_MIN), 0.0, 1.0)
q = np.clip(np.ceil(norm * (AUDIO_DMEL_LEVELS - 1) - 0.5), 0, AUDIO_DMEL_LEVELS - 1)
return q.astype(np.uint8).tobytes()
def decode_wav_mono16k(data, param):
"""Minimal RIFF/WAVE reader: PCM16 or float32, any channel count (mixed
down), sample rate must already be 16 kHz — resampling belongs at the
capture edge, not in the gateway."""
import struct as _struct
np = _np()
if len(data) < 44 or data[:4] != b"RIFF" or data[8:12] != b"WAVE":
raise APIError(400, "Audio must be a RIFF/WAVE file.", param, "invalid_value")
pos, fmt, raw = 12, None, None
while pos + 8 <= len(data):
cid, size = data[pos:pos + 4], _struct.unpack_from("<I", data, pos + 4)[0]
body = data[pos + 8:pos + 8 + size]
if cid == b"fmt ":
fmt = _struct.unpack_from("<HHIIHH", body, 0)
elif cid == b"data":
raw = body
pos += 8 + size + (size & 1)
if fmt is None or raw is None:
raise APIError(400, "WAV file is missing fmt/data chunks.", param, "invalid_value")
audio_format, channels, rate, _, _, bits = fmt
if audio_format == 0xFFFE: # WAVE_FORMAT_EXTENSIBLE: trust the bit width
audio_format = 3 if bits == 32 else 1
if rate != AUDIO_SAMPLE_RATE:
raise APIError(400, f"Audio must be {AUDIO_SAMPLE_RATE} Hz (got {rate}). "
"Resample at the capture edge.", param, "invalid_value")
if audio_format == 1 and bits == 16:
samples = np.frombuffer(raw, dtype="<i2").astype(np.float32) / 32768.0
elif audio_format == 3 and bits == 32:
samples = np.frombuffer(raw, dtype="<f4").astype(np.float32)
else:
raise APIError(400, f"Unsupported WAV encoding (format {audio_format}, {bits}-bit); "
"use PCM16 or float32.", param, "invalid_value")
if channels > 1:
samples = samples[:len(samples) - len(samples) % channels]
samples = samples.reshape(-1, channels).mean(axis=1)
return samples
def inkling_content_segments(content, param, audio_out):
"""Split OpenAI message content into ordered TMLv0 segments:
("text", str) for merged text runs, ("audio", n_frames) per input_audio
part (its DMel bytes appended to audio_out in prompt order)."""
if isinstance(content, str):
return [("text", content)]
if not isinstance(content, list):
raise APIError(400, "Message content must be a string or an array of parts.", param)
segments = []
for index, part in enumerate(content):
ptype = part.get("type") if isinstance(part, dict) else None
if ptype in ("text", "input_text"):
if not isinstance(part.get("text"), str):
raise APIError(400, "Text content parts require a string `text` field.",
f"{param}.{index}.text")
if segments and segments[-1][0] == "text":
segments[-1] = ("text", segments[-1][1] + part["text"])
else:
segments.append(("text", part["text"]))
elif ptype == "input_audio":
spec = part.get("input_audio")
if not isinstance(spec, dict) or not isinstance(spec.get("data"), str):
raise APIError(400, "`input_audio` parts need base64 `data`.",
f"{param}.{index}.input_audio")
if spec.get("format", "wav") != "wav":
raise APIError(400, "Only WAV audio is supported (mono, 16 kHz, PCM16/float32).",
f"{param}.{index}.input_audio.format", "unsupported_content_type")
import base64
try:
wav = base64.b64decode(spec["data"], validate=True)
except Exception:
raise APIError(400, "`input_audio.data` is not valid base64.",
f"{param}.{index}.input_audio.data")
dmel = dmel_encode(decode_wav_mono16k(wav, f"{param}.{index}.input_audio.data"))
audio_out.append(dmel)
segments.append(("audio", len(dmel) // AUDIO_MEL_BANDS))
else:
raise APIError(400, "Unsupported content part type for the Inkling engine.",
f"{param}.{index}", "unsupported_content_type")
return segments
# ---- Kimi K3 tool calling (#1143) ----------------------------------------------------------
# K3's normative renderer is encoding_k3.py in the checkpoint repo: tools are pure XTML over
# the four special tokens. The gateway ships typed K3CHAT1 records; kimi_k3.c constructs the
# XTML (tags/attrs are ordinary text whose segment boundaries are token boundaries).
# Y <type-len> <body-len> typed system message (tool-declare / tool-choice)
# O <index> <name-len> <n> tool-result message
# B <think> <nr> <nt> <ncalls> assistant turn with tool calls, then per call
# F <name-len> <nargs> + nargs of V <key-len> <type-len> <val-len>
# J <name-len> <json-len> json fallback for an unparseable arguments string
def _k3_xtml_type(value):
if isinstance(value, bool): return "boolean"
if value is None: return "null"
if isinstance(value, (int, float)): return "number"
if isinstance(value, str): return "string"
if isinstance(value, dict): return "object"
return "array"
def _k3_parse_arguments(raw):
"""OpenAI `arguments` -> list of (key, xtml_type, text), or None for the json fallback.
Mirrors the reference's one-level-deep parse: non-string values keep their exact JSON
bytes (1e2 stays 1e2), string values are the decoded string. A dict input is rendered
per-argument with compact re-serialization for nested values (no original bytes exist).
"""
if isinstance(raw, dict):
return [(k, _k3_xtml_type(v),
v if isinstance(v, str) else json.dumps(v, ensure_ascii=False, separators=(",", ":")))
for k, v in raw.items()]
if raw is None or raw == "":
return []
if not isinstance(raw, str):
return None
s, idx = raw, 0
dec = json.JSONDecoder(strict=False)
def skip():
nonlocal idx
while idx < len(s) and s[idx] in " \t\n\r": idx += 1
skip()
if idx >= len(s) or s[idx] != "{": return None
idx += 1; skip()
if idx < len(s) and s[idx] == "}": return []
out = []
try:
while True:
key, idx = dec.raw_decode(s, idx)
if not isinstance(key, str): return None
skip()
if idx >= len(s) or s[idx] != ":": return None
idx += 1; skip()
vstart = idx
value, idx = dec.raw_decode(s, idx)
text = value if isinstance(value, str) else s[vstart:idx]
out.append((key, _k3_xtml_type(value), text))
skip()
if idx >= len(s): return None
c = s[idx]; idx += 1; skip()
if c == "}": return out
if c != ",": return None
except (json.JSONDecodeError, ValueError):
return None
def _k3_call_records(tool_calls, where):
"""K3CHAT1 records for one assistant message's tool_calls (F/V or J per call)."""
parts = []
for ci, tc in enumerate(tool_calls):
if not isinstance(tc, dict):
raise APIError(400, "Each tool call must be an object.", f"{where}.{ci}")
fn = tc.get("function", tc)
name = fn.get("name") if isinstance(fn, dict) else None
if not isinstance(name, str) or not name or len(name.encode("utf-8")) > 256:
raise APIError(400, "Tool call needs a function name (<=256 bytes).",
f"{where}.{ci}.function.name")
args = _k3_parse_arguments(fn.get("arguments") if isinstance(fn, dict) else None)
nb = name.encode("utf-8")
if args is None:
js = fn.get("arguments")
js = js if isinstance(js, str) else json.dumps(js or {}, ensure_ascii=False)
jb = js.encode("utf-8")
parts.append(f"J {len(nb)} {len(jb)}\n{name}{js}")
else:
if len(args) > 64:
raise APIError(400, "Too many arguments for one tool call (max 64).",
f"{where}.{ci}.function.arguments")
parts.append(f"F {len(nb)} {len(args)}\n{name}")
for key, typ, text in args:
kb, tb, vb = key.encode("utf-8"), typ.encode("utf-8"), text.encode("utf-8")
if len(kb) < 1 or len(kb) > 256:
raise APIError(400, "Argument keys must be 1..256 bytes.",
f"{where}.{ci}.function.arguments")
parts.append(f"V {len(kb)} {len(tb)} {len(vb)}\n{key}{typ}{text}")
return parts
def _k3_order_tool_results(messages):
"""Re-sort each run of tool messages into the preceding assistant's tool_calls order,
resolving names from tool_call_id — the reference renderer's normalization. A run with
any unresolvable id is left untouched (order-based name fallback still applies)."""
out, index_map, i = [], {}, 0
while i < len(messages):
msg = messages[i]
if isinstance(msg, dict) and msg.get("role") == "assistant":
index_map = {}
calls = msg.get("tool_calls")
if calls is not None and not isinstance(calls, list):
raise APIError(400, "`tool_calls` must be an array.",
f"messages.{i}.tool_calls")
for pos, tc in enumerate(calls or [], start=1):
if isinstance(tc, dict) and tc.get("id") is not None:
fn = tc.get("function", tc)
nm = fn.get("name") if isinstance(fn, dict) else None
index_map[str(tc["id"])] = (pos, nm)
out.append(msg); i += 1; continue
if not (isinstance(msg, dict) and msg.get("role") == "tool"):
out.append(msg); i += 1; continue
run = []
while i < len(messages) and isinstance(messages[i], dict) and messages[i].get("role") == "tool":
tm = messages[i]
hit = index_map.get(str(tm.get("tool_call_id"))) if tm.get("tool_call_id") is not None else None
run.append((hit[0] if hit else None, len(run), tm, hit[1] if hit else None))
i += 1
if any(pos is None for pos, _, _, _ in run):
out.extend(tm for _, _, tm, _ in run)
else:
run.sort()
for _, _, tm, nm in run:
fixed = dict(tm)
if nm: fixed["name"] = nm
out.append(fixed)
return out
def render_chat_kimi(messages, enable_thinking=False, reasoning_effort=None, tools=None,
tool_choice=None, add_generation_prompt=True):
"""Validated multi-turn K3 payload for the C engine.
K3's rank-BPE makes ordinary-text segment boundaries part of the tokenizer
contract. This private length-framed payload preserves roles, UTF-8 bytes,
and message boundaries; kimi_k3.c constructs the native XTML tokens.
add_generation_prompt=False continues a trailing assistant turn. Kimi frames turns
engine-side, so unlike the string renderers there is no terminator to drop here: the final
assistant turn is emitted as a `C` record (reasoning + text), which kimi_k3.c renders as
the open turn -- no <|close|>/<|end_of_msg|>, and no fresh generation cue. An engine that
predates the record rejects the payload rather than miswiring it.
"""
if not isinstance(messages, list) or not messages:
raise APIError(400, "`messages` must be a non-empty array.", "messages")
forced = None
if isinstance(tool_choice, dict):
forced = _tool_choice_name(tool_choice)
if forced:
tools = [t for t in (tools or [])
if ((t.get("function", t) if isinstance(t, dict) else {}).get("name") == forced)]
elif tool_choice == "none":
tools = None # the client forbade tools: do not offer them
messages = _k3_order_tool_results(messages)
parts = ["K3CHAT1\n"]
if tools:
body = ("# Tools\nHere are the available tools, described in JSONSchema.\n\n"
"```json\n" + json.dumps(tools, ensure_ascii=False, separators=(",", ":"),
sort_keys=True) + "\n```")
parts.append(f"Y 12 {len(body.encode('utf-8'))}\ntool-declare{body}")
tool_index = 0
last_calls = []
for index, message in enumerate(messages):
if not isinstance(message, dict):
raise APIError(400, "Each message must be an object.", f"messages.{index}")
role = message.get("role")
if role not in ("system", "developer", "user", "assistant", "tool"):
raise APIError(400, f"Unsupported role {role!r}.", f"messages.{index}.role")
raw = message.get("content")
text = content_text(raw, f"messages.{index}.content") if raw is not None else ""
if role == "tool":
tool_index += 1
name = message.get("name") or message.get("tool")
if not name and tool_index <= len(last_calls):
fn = last_calls[tool_index - 1]
fn = fn.get("function", fn) if isinstance(fn, dict) else {}
name = fn.get("name")
if not isinstance(name, str) or not name:
raise APIError(400, "Kimi K3 tool messages need a resolvable tool name: "
"carry `name`, or match a preceding assistant tool_call "
"by id or order.", f"messages.{index}.name")
nb = name.encode("utf-8")
if len(nb) > 256:
raise APIError(400, "Tool name too long (max 256 bytes).", f"messages.{index}.name")
parts.append(f"O {tool_index} {len(nb)} {len(text.encode('utf-8'))}\n{name}{text}")
continue
reasoning = message.get("reasoning_content") if role == "assistant" else None
if reasoning is not None and not isinstance(reasoning, str):
raise APIError(400, "`reasoning_content` must be a string.",
f"messages.{index}.reasoning_content")
calls = message.get("tool_calls") if role == "assistant" else None
if role == "assistant":
last_calls = calls or []
tool_index = 0
if not add_generation_prompt and index == len(messages) - 1:
# Continuation: the trailing assistant turn is left OPEN. resolve_generation_prompt
# has already refused tools/tool_calls and a non-assistant trailing turn, so this is
# a plain assistant turn; the C record carries its reasoning (if any) and text, and
# kimi_k3.c renders it as the open turn with no cue.
r = reasoning or ""
parts.append(f"C {len(r.encode('utf-8'))} {len(text.encode('utf-8'))}\n{r}{text}")
continue
if calls:
if len(calls) > 64:
raise APIError(400, "Too many tool calls in one message (max 64).",
f"messages.{index}.tool_calls")
reasoning = reasoning or ""
parts.append(f"B {1 if enable_thinking else 0} {len(reasoning.encode('utf-8'))} "
f"{len(text.encode('utf-8'))} {len(calls)}\n{reasoning}{text}")
parts.extend(_k3_call_records(calls, f"messages.{index}.tool_calls"))
elif role == "assistant" and enable_thinking:
reasoning = reasoning or ""
parts.append(f"A {len(reasoning.encode('utf-8'))} {len(text.encode('utf-8'))}\n"
f"{reasoning or ''}{text}")
else:
r = "system" if role == "developer" else role
parts.append(f"M {r} {len(text.encode('utf-8'))}\n{text}")
if tool_choice == "required" and tools:
body = ("The system is invoked with `tool_choice=required`.\n"
"You MUST call tools in the next message.")
parts.append(f"Y 11 {len(body.encode('utf-8'))}\ntool-choice{body}")
elif forced and tools:
body = (f"The system is invoked with a forced tool choice.\n"
f"You MUST call the tool `{forced}` in the next message.")
parts.append(f"Y 11 {len(body.encode('utf-8'))}\ntool-choice{body}")
parts.append(f"G {1 if enable_thinking else 0}\n")
return "".join(parts)
def render_chat_v4(messages, enable_thinking=False, reasoning_effort=None, tools=None,
tool_choice=None, add_generation_prompt=True):
"""DeepSeek V4's native multi-turn chat template.
The target engine receives this as a raw prompt. Prior assistant turns end
with the checkpoint's EOS marker; the final assistant marker selects the
thinking or direct-answer prefix for the new turn.
add_generation_prompt=False continues a trailing assistant turn: the last assistant turn
is rendered open, i.e. without its closing EOS and with no cue, the position the model
occupies mid-turn. EOS is the terminator to drop here, as <|im_end|> is for ChatML.
Tool use follows the official DSML format (encoding/encoding_dsv4.py): tool
schemas are declared on the first system/developer message, assistant tool
calls are DSML blocks, and tool results are <tool_result> blocks merged into
user turns (V4 has no standalone "tool" role).
"""
if not isinstance(messages, list) or not messages:
raise APIError(400, "`messages` must be a non-empty array.", "messages")
forced = None
if isinstance(tool_choice, dict):
forced = _tool_choice_name(tool_choice)
if forced:
tools = [t for t in (tools or [])
if ((t.get("function", t) if isinstance(t, dict) else {}).get("name") == forced)]
elif tool_choice == "none":
tools = None # the client forbade tools: do not offer them
# Merge tool messages into <tool_result> blocks on the following user turn (V4 has no
# tool role); validate every message on the original list for accurate field-level errors.
merged = []
for index, message in enumerate(messages):
if not isinstance(message, dict):
raise APIError(400, "Each message must be an object.", f"messages.{index}")
role = message.get("role")
if role not in ("system", "developer", "user", "assistant", "tool"):
raise APIError(400, f"Unsupported role {role!r}.", f"messages.{index}.role")
if role == "assistant":
reasoning = message.get("reasoning_content")
if reasoning is not None and not isinstance(reasoning, str):
raise APIError(400, "`reasoning_content` must be a string.",
f"messages.{index}.reasoning_content")
raw = message.get("content")
content = content_text(raw, f"messages.{index}.content") if raw is not None else ""
merged.append({"role": role, "content": content,
"reasoning_content": message.get("reasoning_content"),
"tool_calls": _history_tool_calls(message.get("tool_calls"), index)})
continue
raw = message.get("content")
text = content_text(raw, f"messages.{index}.content") if raw is not None else ""
if role == "tool":
block = "<tool_result>" + text + "</tool_result>"
if merged and merged[-1].get("_parts") is not None:
merged[-1]["_parts"].append(block)
else:
merged.append({"role": "user", "_parts": [block]})
elif role == "user":
if merged and merged[-1].get("_parts") is not None:
merged[-1]["_parts"].append(text)
else:
merged.append({"role": "user", "content": text})
else: # system / developer
merged.append({"role": role, "content": text})
if tools:
tools_text = _dsv4_tools_block(tools)
if forced:
tools_text += f"\n\nYou must call the function `{forced}`. Do not answer directly."
elif tool_choice == "required":
tools_text += TOOL_CHOICE_REQUIRED_INSTRUCTION
for msg in merged:
if msg["role"] in ("system", "developer"):
msg["content"] += "\n\n" + tools_text
break
else:
# No system/developer message: the official encoder renders tools on an empty
# system message, i.e. "bos" + "\n\n" + tools. Keep that exact byte layout.
merged.insert(0, {"role": "system", "content": "\n\n" + tools_text})
bos = "<\uff5cbegin\u2581of\u2581sentence\uff5c>"
user = "<\uff5cUser\uff5c>"
assistant = "<\uff5cAssistant\uff5c>"
eos = "<\uff5cend\u2581of\u2581sentence\uff5c>"
parts = [bos]
if enable_thinking:
effort = DSV4_REASONING_EFFORT.get(reasoning_effort, "low")
if effort != "low":
parts.append(DSV4_REASONING_EFFORT_PROMPTS[effort])
for m_index, message in enumerate(merged):
role = message["role"]
if role in ("system", "developer"):
if role == "developer":
parts.append(user) # V4 wraps developer messages like user turns
parts.append(message["content"])
elif role == "user":
parts.append(user)
if message.get("_parts") is not None:
parts.append("\n\n".join(message["_parts"]))
else:
parts.append(message["content"])
else:
reasoning = message.get("reasoning_content")
parts.append(assistant)
if reasoning:
parts.extend(("<think>", reasoning, "</think>"))
else:
parts.append("</think>")
parts.append(message["content"])
if message.get("tool_calls"):
parts.append(_dsv4_tool_calls(message["tool_calls"]))
# A continued turn is the last message rendered open: no EOS, no cue below.
if add_generation_prompt or m_index != len(merged) - 1:
parts.append(eos)
if add_generation_prompt:
parts.extend((assistant, "<think>" if enable_thinking else "</think>"))
return "".join(parts)
def render_chat_olmoe(messages, enable_thinking=False, reasoning_effort=None, tools=None,
tool_choice=None, add_generation_prompt=True):
"""OLMoE-Instruct's native chat_template (tokenizer_config.json): one
bos_token, then per-message <|system|>/<|user|>/<|assistant|> turns each
closed by a newline, prior assistant turns also closed by eos_token
(bos_token == eos_token == "|||IP_ADDRESS|||", a PII-scrubbing artifact
repurposed as this tokenizer's BOS/EOS marker), and a trailing
"<|assistant|>\\n" generation prompt. No tool-call syntax and no thinking
mode exist in this template, so both parameters are accepted but unused.
add_generation_prompt=False continues a trailing assistant turn. The template closes
even the last assistant turn with eos_token, so the open-turn shape is that turn without
the eos and with no cue -- the same drop-the-terminator move as the ChatML families, with
eos_token as the terminator here."""
if not isinstance(messages, list) or not messages:
raise APIError(400, "`messages` must be a non-empty array.", "messages")
if tool_choice == "none":
tools = None
if (tools or tool_choice not in (None, "none")) and not _TOOL_FALLBACK:
raise APIError(400, "Tool use is not wired up for the OLMoE engine yet. "
"Set COLI_TOOL_FALLBACK=1 to opt into prompt-injected "
"tool translation.", "tools", "unsupported_parameter")
boundary = "|||IP_ADDRESS|||" # bos_token == eos_token in this tokenizer
parts = [boundary]
if tools and _TOOL_FALLBACK:
# Fallback only: without it, `required` is already a 400 above.
preamble = _fallback_tool_preamble(tools)
if tool_choice == "required":
preamble += TOOL_CHOICE_REQUIRED_INSTRUCTION
parts.append(f"<|system|>\n{preamble}\n")
last = len(messages) - 1
for index, message in enumerate(messages):
if not isinstance(message, dict):
raise APIError(400, "Each message must be an object.", f"messages.{index}")
role = message.get("role")
allowed = ("system", "developer", "user", "assistant")
if _TOOL_FALLBACK:
allowed += ("tool",)
if role not in allowed:
raise APIError(400, f"Unsupported role {role!r}.", f"messages.{index}.role")
raw = message.get("content")
text = content_text(raw, f"messages.{index}.content") if raw is not None else ""
if role in ("system", "developer"):
parts.append(f"<|system|>\n{text}\n")
elif role == "user":
parts.append(f"<|user|>\n{text}\n")
elif role == "tool":
# No tool role in this template: the result rides in as a user turn.
parts.append(f"<|user|>\n{_fallback_tool_result(message, index)}\n")
else:
calls = (_fallback_tool_calls(message.get("tool_calls"), index)
if _TOOL_FALLBACK else "")
# A continued turn is the last message rendered open: no eos, no cue.
terminator = "" if (not add_generation_prompt and index == last) else boundary
parts.append(f"<|assistant|>\n{text}{calls}{terminator}")
if index != last:
parts.append("\n")
if add_generation_prompt:
parts.append("<|assistant|>\n")
return "".join(parts)
def render_chat_qwen(messages, enable_thinking=False, reasoning_effort=None, tools=None,
tool_choice=None, add_generation_prompt=True, preserve_thinking=False):
"""Qwen3.6's chat_template, tool calling included: <|im_start|>role\\n ...
<|im_end|>\\n frames, then the generation prompt. The official template
opens a mandatory <think> block after `<|im_start|>assistant\\n` — the
model was never trained on the bare `assistant\\n` state, and greedy
argmax there lands on an EOS special (measured: gen=0). With thinking
disabled the template pre-closes the block instead; both branches are
mirrored here byte for byte.
Tool declarations, assistant tool calls and tool responses follow the checkpoint's
chat_template.jinja: the `# Tools` block opens the one system turn (the client's own
system text goes last in it), calls are the XML-ish <tool_call> form Qwen3.8 shares,
and consecutive `tool` messages share ONE user turn of <tool_response> blocks.
add_generation_prompt=False continues a trailing assistant turn. The template renders an
assistant turn AFTER the last user query with its <think></think> block (an earlier one,
from history, has it stripped) -- so the open-turn shape is that think-form minus the
<|im_end|> terminator and with no cue, not the bare history form the loop emits otherwise.
ChatML's per-turn terminator is why the marker has to be dropped explicitly, as on qwen38.
preserve_thinking is the template's own kwarg of the same name (#1759): every past assistant
turn keeps its <think> block, `reasoning_content` inside it, empty when there is none. Qwen
trained Qwen3.6 on that form. Without it the template strips the block from every turn before
the last user query, so a turn generated after the pre-closed think header comes back without
the header it was fed, the resent history diverges from the engine's state at the first
assistant turn, and prefix reuse -- all or nothing on a recurrent state -- never engages."""
if not isinstance(messages, list) or not messages:
raise APIError(400, "`messages` must be a non-empty array.", "messages")
if tool_choice == "none":
tools = None
if tools is not None and not isinstance(tools, list):
raise APIError(400, "`tools` must be an array.", "tools")
# The template's last_query_index: the last user turn that is a real query and not a
# tool result riding in as one. An assistant turn after it keeps its <think> block
# with or without preserve_thinking. With no query at all the template raises; here
# the index stays where the template starts it, at the last message.
last_query = len(messages) - 1
for index in range(len(messages) - 1, -1, -1):
message = messages[index]
if not isinstance(message, dict) or message.get("role") != "user":
continue
raw = message.get("content")
query = (content_text(raw, f"messages.{index}.content") if raw is not None else "").strip()
if not (query.startswith("<tool_response>") and query.endswith("</tool_response>")):
last_query = index
break
parts = []
# With tools the template builds ONE system turn: the tool block, then the client's
# own system text (trimmed) after a blank line.
first = messages[0]
first_role = first.get("role") if isinstance(first, dict) else None
system_text = ""
start = 0
if first_role in ("system", "developer"):
raw = first.get("content")
system_text = content_text(raw, "messages.0.content").strip() if raw is not None else ""
start = 1
if tools:
block = _qwen_tool_block(tools)
if tool_choice == "required":
block += TOOL_CHOICE_REQUIRED_INSTRUCTION
if system_text:
block += "\n\n" + system_text
parts.append(f"<|im_start|>system\n{block}<|im_end|>\n")
elif start:
parts.append(f"<|im_start|>system\n{system_text}<|im_end|>\n")
for index, message in enumerate(messages[start:], start=start):
if not isinstance(message, dict):
raise APIError(400, "Each message must be an object.", f"messages.{index}")
role = message.get("role")
if role == "developer":
role = "system"
if role not in ("system", "user", "assistant", "tool"):
raise APIError(400, f"Unsupported role {role!r}.", f"messages.{index}.role")
if role == "system":
raise APIError(400, "System message must be at the beginning.",
f"messages.{index}.role")
raw = message.get("content")
text = content_text(raw, f"messages.{index}.content").strip() if raw is not None else ""
if not add_generation_prompt and role == "assistant" and index == len(messages) - 1:
# Continued turn: the template gives a post-query assistant turn a <think></think>
# block, then the model resumes the content. Match it, minus the terminator/cue.
reasoning = message.get("reasoning_content", "")
if not isinstance(reasoning, str):
raise APIError(400, "`reasoning_content` must be a string.",
f"messages.{index}.reasoning_content")
parts.append(f"<|im_start|>assistant\n<think>\n{reasoning.strip()}\n</think>\n\n"
f"{text}")
continue
if role == "tool":
# Consecutive tool results share ONE user turn: the opening tag is written only
# when the previous message was not a tool, the closing one only when the next
# is not.
prev = messages[index - 1].get("role") if index > 0 and isinstance(
messages[index - 1], dict) else None
nxt = messages[index + 1].get("role") if index + 1 < len(messages) and isinstance(
messages[index + 1], dict) else None
if prev != "tool":
parts.append("<|im_start|>user")
parts.append(f"\n<tool_response>\n{text}\n</tool_response>")
if nxt != "tool":
parts.append("<|im_end|>\n")
continue
if role == "assistant":
# The template's reading of a past turn: `reasoning_content` when it is a
# string, otherwise whatever the content carries before a </think> (a client
# echoing the raw reply), which then leaves the content.
reasoning = message.get("reasoning_content")
if reasoning is not None and not isinstance(reasoning, str):
raise APIError(400, "`reasoning_content` must be a string.",
f"messages.{index}.reasoning_content")
if reasoning is None:
reasoning = ""
if "</think>" in text:
reasoning = text.split("</think>")[0].rstrip("\n").split("<think>")[-1].lstrip("\n")
text = text.split("</think>")[-1].lstrip("\n")
if preserve_thinking or index > last_query:
rendered = f"<think>\n{reasoning.strip()}\n</think>\n\n{text}"
else:
rendered = text
calls = message.get("tool_calls")
if calls:
rendered += _qwen_tool_calls(calls, bool(text.strip()), index)
parts.append(f"<|im_start|>assistant\n{rendered}<|im_end|>\n")
continue
parts.append(f"<|im_start|>{role}\n{text}<|im_end|>\n")
if add_generation_prompt:
parts.append("<|im_start|>assistant\n")
parts.append("<think>\n" if enable_thinking else "<think>\n\n</think>\n\n")
return "".join(parts)
# Qwen3.6 and Qwen3.8 declare and emit tool calls using the same XML-ish wire
# format rather than GLM's JSON block or DeepSeek's DSML. Keep the shared
# declaration and call syntax transcribed from chat_template.jinja rather than
# paraphrased: the declaration is what teaches the model the syntax it must emit.
#
# <tool_call>
# <function=NAME>
# <parameter=KEY>
# VALUE
# </parameter>
# </function>
# </tool_call>
QWEN_TOOL_PREAMBLE = ("\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>")
def _qwen_tool_block(tools):
"""The `# Tools` system section, byte-identical to the template's."""
lines = ["# Tools\n\nYou have access to the following functions:\n\n<tools>"]
for tool in tools:
lines.append("\n" + json.dumps(tool, ensure_ascii=False, separators=(", ", ": ")))
lines.append("\n</tools>")
lines.append(QWEN_TOOL_PREAMBLE)
return "".join(lines)
def _qwen_tool_calls(tool_calls, has_content, index):
"""Render assistant tool_calls. The template separates the FIRST call from
preceding content with a blank line only when that content is non-empty, and
every later call with a single newline; getting that wrong changes the prompt
the model is conditioned on."""
if tool_calls is not None and not isinstance(tool_calls, list):
raise APIError(400, "`tool_calls` must be an array.", f"messages.{index}.tool_calls")
out = []
for position, call in enumerate(tool_calls or []):
if not isinstance(call, dict):
raise APIError(400, "Each tool call must be an object.",
f"messages.{index}.tool_calls.{position}")
fn = call.get("function", call)
if not isinstance(fn, dict):
raise APIError(400, "`function` must be an object.",
f"messages.{index}.tool_calls.{position}.function")
name = fn.get("name")
if not isinstance(name, str) or not name:
raise APIError(400, "`function.name` must be a non-empty string.",
f"messages.{index}.tool_calls.{position}.function.name")
lead = ("\n\n" if has_content else "") if position == 0 else "\n"
out.append(f"{lead}<tool_call>\n<function={name}>\n")
args = fn.get("arguments", "")
if isinstance(args, str) and args:
try:
args = json.loads(args)
except (TypeError, ValueError):
raise APIError(400, "`function.arguments` must be a JSON object.",
f"messages.{index}.tool_calls.{position}.function.arguments")
if isinstance(args, dict):
for key, value in args.items():
# The template stringifies a str as-is and tojson's everything
# else, so a string argument must NOT gain quotes here.
rendered = value if isinstance(value, str) else json.dumps(
value, ensure_ascii=False, separators=(", ", ": "))
out.append(f"<parameter={key}>\n{rendered}\n</parameter>\n")
out.append("</function>\n</tool_call>")
return "".join(out)
QWEN3_CODER_SYSTEM = ("You are Qwen, a helpful AI assistant that can interact with a computer "
"to solve tasks.")
QWEN3_CODER_TOOL_RULES = (
"\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:"
"\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\n"
"value_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second "
"parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n"
"<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner "
"<function=...></function> block must be nested within <tool_call></tool_call> XML tags\n"
"- Required parameters MUST be specified\n- You may provide optional reasoning for your "
"function call in natural language BEFORE the function call, but NOT after\n- If there is "
"no function call available, answer the question like normal with your current knowledge "
"and do not tell the user about function calls\n</IMPORTANT>")
def _jinja_string(value):
"""jinja's `string` filter: a str as it is, anything else through str()."""
return value if isinstance(value, str) else str(value)
def _qwen3_coder_extra_keys(fields, handled):
"""The template's render_extra_keys macro: every key it does not name, in order."""
if not isinstance(fields, dict):
return ""
out = []
for key, value in fields.items():
if key in handled:
continue
shown = (json.dumps(value, ensure_ascii=False) if isinstance(value, (dict, list, tuple))
else _jinja_string(value))
out.append(f"\n<{key}>{shown}</{key}>")
return "".join(out)
def _qwen3_coder_tool_block(tools):
"""Qwen3-Coder's `# Tools` section: each function as XML, its parameters one by one."""
out = ["\n\n# Tools\n\nYou have access to the following functions:\n\n<tools>"]
for index, tool in enumerate(tools):
if not isinstance(tool, dict):
raise APIError(400, "Each tool must be an object.", f"tools.{index}")
fn = tool["function"] if isinstance(tool.get("function"), dict) else tool
out.append(f"\n<function>\n<name>{_jinja_string(fn.get('name', ''))}</name>")
if "description" in fn:
out.append(f"\n<description>{_jinja_string(fn['description']).strip()}</description>")
out.append("\n<parameters>")
params = fn.get("parameters")
if isinstance(params, dict) and isinstance(params.get("properties"), dict):
for name, field in params["properties"].items():
out.append(f"\n<parameter>\n<name>{name}</name>")
if isinstance(field, dict) and "type" in field:
out.append(f"\n<type>{_jinja_string(field['type'])}</type>")
if isinstance(field, dict) and "description" in field:
out.append(f"\n<description>{_jinja_string(field['description']).strip()}"
"</description>")
out.append(_qwen3_coder_extra_keys(field, ("name", "type", "description")))
out.append("\n</parameter>")
out.append(_qwen3_coder_extra_keys(params, ("type", "properties")))
out.append("\n</parameters>")
out.append(_qwen3_coder_extra_keys(fn, ("type", "name", "description", "parameters")))
out.append("\n</function>")
out.append("\n</tools>")
out.append(QWEN3_CODER_TOOL_RULES)
return "".join(out)
def render_chat_qwen3_coder(messages, tools=None, tool_choice=None, add_generation_prompt=True):
"""Qwen3-Coder's chat_template (qwen3_moe), byte for byte: ChatML with a newline after
every <|im_end|>, no thinking at all, the system turn carrying the client's system text
and then the XML `# Tools` block (with the template's own system line when the client
sent none), assistant calls as <tool_call><function=...><parameter=...>, and runs of
`tool` messages sharing one user turn of <tool_response> blocks.
Tool-call arguments sent as a JSON string, the OpenAI shape, are read as the object the
template expects. add_generation_prompt=False leaves a trailing assistant turn open."""
if not isinstance(messages, list) or not messages:
raise APIError(400, "`messages` must be a non-empty array.", "messages")
if tool_choice == "none":
tools = None
if tools is not None and not isinstance(tools, list):
raise APIError(400, "`tools` must be an array.", "tools")
parts = []
first = messages[0]
start = 0
system = None
if isinstance(first, dict) and first.get("role") in ("system", "developer"):
raw = first.get("content")
system = content_text(raw, "messages.0.content") if raw is not None else ""
start = 1
if system is not None:
parts.append("<|im_start|>system\n" + system)
elif tools:
parts.append("<|im_start|>system\n" + QWEN3_CODER_SYSTEM)
if tools:
parts.append(_qwen3_coder_tool_block(tools))
if system is not None or tools:
parts.append("<|im_end|>\n")
loop = messages[start:]
for offset, message in enumerate(loop):
index = start + offset
if not isinstance(message, dict):
raise APIError(400, "Each message must be an object.", f"messages.{index}")
role = message.get("role")
if role == "developer":
role = "system"
if role not in ("system", "user", "assistant", "tool"):
raise APIError(400, f"Unsupported role {role!r}.", f"messages.{index}.role")
raw = message.get("content")
text = content_text(raw, f"messages.{index}.content") if raw is not None else ""
calls = message.get("tool_calls") if role == "assistant" else None
if calls is not None and not isinstance(calls, list):
raise APIError(400, "`tool_calls` must be an array.", f"messages.{index}.tool_calls")
if not add_generation_prompt and role == "assistant" and not calls and \
offset == len(loop) - 1:
parts.append(f"<|im_start|>assistant\n{text}") # continued, still open
continue
if calls:
parts.append("<|im_start|>assistant")
if text.strip():
parts.append("\n" + text.strip() + "\n")
for position, call in enumerate(calls):
where = f"messages.{index}.tool_calls.{position}"
fn = call.get("function", call) if isinstance(call, dict) else None
if not isinstance(fn, dict) or not isinstance(fn.get("name"), str) or not fn["name"]:
raise APIError(400, "Each tool call needs a `function.name`.", where)
args = fn.get("arguments", {})
if isinstance(args, str):
try:
args = json.loads(args) if args.strip() else {}
except ValueError:
raise APIError(400, "`function.arguments` must be a JSON object.",
f"{where}.function.arguments")
if not isinstance(args, dict):
raise APIError(400, "`function.arguments` must be a JSON object.",
f"{where}.function.arguments")
parts.append(f"\n<tool_call>\n<function={fn['name']}>\n")
for key, value in args.items():
shown = (json.dumps(value, ensure_ascii=False)
if isinstance(value, (dict, list, tuple)) else _jinja_string(value))
parts.append(f"<parameter={key}>\n{shown}\n</parameter>\n")
parts.append("</function>\n</tool_call>")
parts.append("<|im_end|>\n")
elif role == "tool":
previous = loop[offset - 1] if offset > 0 else None
if isinstance(previous, dict) and previous.get("role") != "tool":
parts.append("<|im_start|>user\n")
parts.append(f"<tool_response>\n{text}\n</tool_response>\n")
following = loop[offset + 1] if offset + 1 < len(loop) else None
if following is None or (isinstance(following, dict) and following.get("role") != "tool"):
parts.append("<|im_end|>\n")
else:
parts.append(f"<|im_start|>{role}\n{text}<|im_end|>\n")
if add_generation_prompt:
parts.append("<|im_start|>assistant\n")
return "".join(parts)
QWEN_CALL_RE = re.compile(
r"<tool_call>\s*<function=([^>\n]+)>\s*(.*?)</function>\s*</tool_call>", re.S)
QWEN_PARAM_RE = re.compile(r"<parameter=([^>\n]+)>\n(.*?)\n</parameter>", re.S)
def parse_qwen_tool_calls(reply, tools=None):
"""Parse Qwen's XML-ish calls back into OpenAI `tool_calls`.
Values are returned as strings, which is what the template feeds in: it
writes a str argument unquoted, so the original type is not recoverable from
the text alone. Where the declared schema says a parameter is not a string we
re-read it as JSON, which restores numbers and booleans without guessing at
anything the schema did not promise."""
schema = {}
for tool in (tools or []):
fn = tool.get("function", tool) if isinstance(tool, dict) else {}
name = fn.get("name")
params = (fn.get("parameters") or {}).get("properties") or {}
if isinstance(name, str) and name:
if isinstance(params, dict):
schema[name] = params
def make_call(name, body):
args = {}
for key, raw in QWEN_PARAM_RE.findall(body):
key = key.strip()
declared = (schema.get(name) or {}).get(key) or {}
kind = declared.get("type") if isinstance(declared, dict) else None
if kind in (None, "string"):
args[key] = raw
elif kind == "boolean" and raw.strip().lower() in ("true", "false"):
# The templates print a bool through jinja's `string`, so the model
# writes `True`; Qwen's own parser reads it case-insensitively.
args[key] = raw.strip().lower() == "true"
else:
try:
args[key] = json.loads(raw)
except (TypeError, ValueError):
args[key] = raw
return {
"id": f"call_{uuid.uuid4().hex[:24]}",
"type": "function",
"function": {
"name": name,
"arguments": json.dumps(args, ensure_ascii=False),
},
}
calls = []
for match in QWEN_CALL_RE.finditer(reply or ""):
name = match.group(1).strip()
calls.append(make_call(name, match.group(2)))
text = QWEN_CALL_RE.sub("", reply or "")
if not calls and tools and (
"<tool_call>" in (reply or "") or "<function=" in (reply or "")
):
sys.stderr.write(
"[api] qwen tool markers present but no call parsed -- "
"possibly truncated or mangled output\n"
)
sys.stderr.flush()
return text.strip(), calls
# Backward compatibility for existing Qwen3.8 callers.
parse_qwen38_tool_calls = parse_qwen_tool_calls
# ---- MiMo-V2.6 (Xiaomi) -------------------------------------------------------------------
# ChatML without a newline after <|im_end|>, every assistant turn carrying its <think> block
# (empty or not), tools declared in their own system turn, calls written INLINE
# (<tool_call><function=N><parameter=K>V</parameter></function></tool_call>, no newlines) and
# tool results as a plain `tool` turn. All of it from the release's chat_template.jinja
# (XiaomiMiMo/MiMo-V2.6-Flash-MOPD); tests/test_mimo_chat_template.py renders that template
# with jinja2 and compares byte for byte.
MIMO_TOOLS_HEAD = "You are provided with the following tools:\n\n<tools>"
def _mimo_tools(tools):
"""render_tools(): each tool object as JSON, key order kept (transformers' tojson)."""
return (MIMO_TOOLS_HEAD
+ "".join("\n" + json.dumps(tool, ensure_ascii=False) for tool in tools)
+ "\n</tools>")
def _mimo_value(value):
"""render_value(): a string as-is, anything else as JSON."""
return value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
def _mimo_tool_calls(calls, index):
out = []
for position, call in enumerate(calls):
where = f"messages.{index}.tool_calls.{position}"
if not isinstance(call, dict):
raise APIError(400, "Each tool call must be an object.", where)
fn = call.get("function", call.get("custom", call))
if not isinstance(fn, dict) or not isinstance(fn.get("name"), str) or not fn["name"]:
raise APIError(400, "A tool call needs a `function.name`.", f"{where}.function")
out.append(f"<tool_call><function={fn['name']}>")
if isinstance(fn.get("input"), str):
out.append(fn["input"])
else:
args = fn.get("arguments")
# An OpenAI client sends the arguments back as a JSON string. The template
# would print that string raw, a shape the model never writes; read as the
# object it is, the call comes back in the <parameter=...> form the model
# produced, which is also what keeps the resent history on the KV prefix.
if isinstance(args, str) and args.strip():
try:
args = json.loads(args)
except (TypeError, ValueError):
raise APIError(400, "`function.arguments` must be a JSON object.",
f"{where}.function.arguments")
if isinstance(args, dict):
for key, value in args.items():
out.append(f"<parameter={key}>{_mimo_value(value)}</parameter>")
elif args not in (None, "", {}):
raise APIError(400, "`function.arguments` must be a JSON object.",
f"{where}.function.arguments")
out.append("</function></tool_call>")
return "".join(out)
def render_chat_mimo(messages, enable_thinking=True, reasoning_effort=None, tools=None,
tool_choice=None, add_generation_prompt=True):
"""MiMo-V2.6's chat template.
One deliberate addition: with thinking on, the generation cue ends in `<think>`. The
template leaves that token to the model, and the model writes it first on every turn
it was trained on (each assistant turn of the template opens with it); putting it in
the prompt is what lets the reasoning split start inside the reasoning, where the
model is. With thinking off the template's own `<think></think>` closes it.
add_generation_prompt=False continues a trailing assistant turn: its past-turn render
minus the closing <|im_end|>, and no cue."""
if not isinstance(messages, list) or not messages:
raise APIError(400, "`messages` must be a non-empty array.", "messages")
if tool_choice in ("none",):
tools = None
if tools is not None and not isinstance(tools, list):
raise APIError(400, "`tools` must be an array.", "tools")
parts = []
if tools:
tools_text = _mimo_tools(tools)
# In the tool turn, after </tools> and before its <|im_end|>: "the functions
# above" has to be true, and MiMo's declaration block is a whole system turn
# of its own, so there is nothing to put the line outside of but the frame.
if tool_choice == "required":
tools_text += TOOL_CHOICE_REQUIRED_INSTRUCTION
parts.append(f"<|im_start|>system\n{tools_text}<|im_end|>")
last = len(messages) - 1
for index, message in enumerate(messages):
if not isinstance(message, dict):
raise APIError(400, "Each message must be an object.", f"messages.{index}")
role = message.get("role")
if role == "developer":
role = "system"
if role not in ("system", "user", "assistant", "tool"):
raise APIError(400, f"Unsupported role {role!r}.", f"messages.{index}.role")
raw = message.get("content")
body = content_text(raw, f"messages.{index}.content") if raw is not None else ""
if role == "assistant":
reasoning = message.get("reasoning_content")
if reasoning is not None and not isinstance(reasoning, str):
raise APIError(400, "`reasoning_content` must be a string.",
f"messages.{index}.reasoning_content")
turn = f"<|im_start|>assistant\n<think>{reasoning or ''}</think>{body}"
calls = message.get("tool_calls")
if calls:
if not isinstance(calls, list):
raise APIError(400, "`tool_calls` must be an array.",
f"messages.{index}.tool_calls")
turn += _mimo_tool_calls(calls, index)
if not add_generation_prompt and index == last:
parts.append(turn) # the open turn: no terminator, no cue
return "".join(parts)
parts.append(turn + "<|im_end|>")
continue
parts.append(f"<|im_start|>{role}\n{body}<|im_end|>")
if add_generation_prompt:
parts.append("<|im_start|>assistant\n")
parts.append("<think>" if enable_thinking else "<think></think>")
return "".join(parts)
MIMO_CALL_RE = re.compile(
r"<tool_call>\s*<function=([^>\n]+)>(.*?)</function>\s*</tool_call>", re.S)
MIMO_PARAM_RE = re.compile(r"<parameter=([^>\n]+)>(.*?)</parameter>", re.S)
def parse_mimo_tool_calls(reply, tools=None):
"""MiMo's calls back into OpenAI `tool_calls`: `(content, tool_calls)`."""
content, calls, _box_map, _content_map = _parse_mimo_tool_calls(reply, tools, False)
return content, calls
def _parse_mimo_tool_calls(reply, tools, track_spans):
"""MiMo's calls back into OpenAI `tool_calls`, with the stage's maps: `(content,
tool_calls, box_map, content_map)`, both maps None unless `track_spans`.
Parameters are inline (`<parameter=K>V</parameter>`); a value that arrives on its own
lines, Qwen-style, loses exactly one newline on each side. Types come from the declared
schema as for Qwen: a string parameter stays text, anything else is read as JSON. A
body with no parameter tags but a JSON object in it is the template's other spelling
(`arguments` as a string) and is taken as the arguments."""
schema = {}
for tool in (tools or []):
fn = tool.get("function", tool) if isinstance(tool, dict) else {}
name = fn.get("name")
params = (fn.get("parameters") or {}).get("properties") or {}
if isinstance(name, str) and name and isinstance(params, dict):
schema[name] = params
def make_call(name, body):
args = {}
found = MIMO_PARAM_RE.findall(body)
for key, raw in found:
key = key.strip()
if raw.startswith("\n"):
raw = raw[1:]
if raw.endswith("\n"):
raw = raw[:-1]
declared = (schema.get(name) or {}).get(key) or {}
kind = declared.get("type") if isinstance(declared, dict) else None
args[key] = _coerce_arg(raw, kind if isinstance(kind, str) else None) \
if kind not in (None, "string") else raw
if not found and body.strip():
try:
parsed = json.loads(body)
if isinstance(parsed, dict):
args = parsed
except (TypeError, ValueError):
pass
return {"id": f"call_{uuid.uuid4().hex[:24]}", "type": "function",
"function": {"name": name, "arguments": json.dumps(args, ensure_ascii=False)}}
text = reply or ""
matches = list(MIMO_CALL_RE.finditer(text))
calls = [make_call(m.group(1).strip(), m.group(2)) for m in matches]
if not calls and tools and ("<tool_call>" in text or "<function=" in text):
sys.stderr.write("[api] mimo tool markers present but no call parsed -- "
"possibly truncated or mangled output\n")
sys.stderr.flush()
if not track_spans:
return MIMO_CALL_RE.sub("", text).strip(), calls, None, None
# The same two deletions as the line above -- every call block, then the surrounding
# whitespace -- found as ranges, so logprobs.content can describe the content returned
# rather than the raw reply with the call syntax in it.
content, box_map = _apply_cuts(text, [(m.start(), m.end()) for m in matches])
head = len(content) - len(content.lstrip())
kept = len(content.strip())
content, step = _apply_cuts(content, [(0, head), (head + kept, len(content))])
return content, calls, box_map, _compose_span_maps(box_map, step)
def render_chat_qwen38(messages, enable_thinking=True, reasoning_effort=None, tools=None,
tool_choice=None, add_generation_prompt=True):
"""Text-only Qwen3.8 chat-template subset with native reasoning hints.
add_generation_prompt=False continues a trailing assistant turn. ChatML closes every
turn with <|im_end|>, so the open-turn shape is the past-turn render of that last message
MINUS its terminator, and no generation cue after it -- the position the model occupies
while writing an assistant turn. (GLM has no per-turn terminator, so there suppressing the
cue is enough; here the terminator has to be dropped too.)"""
if not isinstance(messages, list) or not messages:
raise APIError(400, "`messages` must be a non-empty array.", "messages")
if tool_choice in ("none",):
tools = None # the client forbade them: do not offer any
if tools is not None and not isinstance(tools, list):
raise APIError(400, "`tools` must be an array.", "tools")
instruction = ""
if enable_thinking:
effort = reasoning_effort or "xhigh"
if effort == "high":
effort = "xhigh"
elif effort == "minimal":
effort = "low"
if effort not in ("xhigh", "medium", "low"):
raise APIError(400, "Qwen3.8 reasoning_effort must be high, xhigh, medium, or low.",
"reasoning_effort")
if effort == "xhigh":
instruction = ("Reasoning effort is set to xhigh. Please think carefully through "
"the task, validate key assumptions, consider plausible "
"alternatives, and prioritize correctness, consistency, and "
"clarity in the final answer.")
elif effort == "low":
instruction = ("Reasoning effort is set to low. Keep your thinking brief and "
"focused, moving directly to the conclusion without unnecessary "
"elaboration.")
parts = []
first = messages[0]
first_role = first.get("role") if isinstance(first, dict) else None
if first_role == "developer":
first_role = "system"
system_text = ""
start = 0
if first_role == "system":
raw = first.get("content")
system_text = content_text(raw, "messages.0.content").strip() if raw is not None else ""
start = 1
if tools:
# With tools the template builds ONE system turn in a fixed order:
# reasoning instruction, then the tool block, then the user's own system
# text last -- not the other way round.
head = (instruction + "\n\n") if instruction else ""
block = head + _qwen_tool_block(tools)
if tool_choice == "required":
block += TOOL_CHOICE_REQUIRED_INSTRUCTION
if system_text:
block += "\n\n" + system_text
parts.append(f"<|im_start|>system\n{block}<|im_end|>\n")
elif system_text or instruction:
text = system_text
if instruction:
text = instruction + ("\n\n" + text if text else "")
parts.append(f"<|im_start|>system\n{text}<|im_end|>\n")
for index, message in enumerate(messages[start:], start=start):
if not isinstance(message, dict):
raise APIError(400, "Each message must be an object.", f"messages.{index}")
role = message.get("role")
if role == "developer":
role = "system"
if role not in ("system", "user", "assistant", "tool"):
raise APIError(400, f"Unsupported role {role!r}.", f"messages.{index}.role")
if role == "system" and index != 0:
raise APIError(400, "System message must be at the beginning.",
f"messages.{index}.role")
raw = message.get("content")
text = (content_text(raw, f"messages.{index}.content").strip()
if raw is not None else "")
if role == "tool":
# Consecutive tool results share ONE user turn: the opening tag is
# written only when the previous message was not a tool, and the
# closing one only when the next is not. Emitting a turn per result
# would be a different conversation shape.
prev = messages[index - 1].get("role") if index > 0 and isinstance(
messages[index - 1], dict) else None
nxt = messages[index + 1].get("role") if index + 1 < len(messages) and isinstance(
messages[index + 1], dict) else None
if prev != "tool":
parts.append("<|im_start|>user")
parts.append(f"\n<tool_response>\n{text}\n</tool_response>")
if nxt != "tool":
parts.append("<|im_end|>\n")
continue
if role == "assistant":
reasoning = message.get("reasoning_content", "")
if not isinstance(reasoning, str):
raise APIError(400, "`reasoning_content` must be a string.",
f"messages.{index}.reasoning_content")
calls = message.get("tool_calls")
rendered = f"<think>\n{reasoning.strip()}\n</think>\n\n{text}"
if calls:
rendered += _qwen_tool_calls(calls, bool(text.strip()), index)
# A continued turn is the last message rendered open: no <|im_end|>, no cue.
terminator = "" if (not add_generation_prompt and index == len(messages) - 1) \
else "<|im_end|>\n"
parts.append(f"<|im_start|>assistant\n{rendered}{terminator}")
continue
parts.append(f"<|im_start|>{role}\n{text}<|im_end|>\n")
if add_generation_prompt:
parts.append("<|im_start|>assistant\n")
parts.append("<think>\n" if enable_thinking else "<think>\n\n</think>\n\n")
return "".join(parts)
def render_chat_inkling(messages, enable_thinking=False, reasoning_effort=None, tools=None,
tool_choice=None, audio_out=None, add_generation_prompt=True):
"""Text-only subset of Inkling's chat_template.jinja: role tokens with
<|content_text|> parts and <|end_message|> terminators, an assistant
<|content_model_end_sampling|> after each prior model turn, the
thinking-effort hint appended after the messages (the template's fallback
branch), then <|message_model|> as the generation prompt.
add_generation_prompt=False continues a trailing assistant turn: the last model turn is
rendered open -- without its <|end_message|> and the <|content_model_end_sampling|> that
close it, and with no cue. Those two markers are the terminator to drop here."""
if not isinstance(messages, list) or not messages:
raise APIError(400, "`messages` must be a non-empty array.", "messages")
if tools or (tool_choice not in (None, "none")):
raise APIError(400, "Tool use is not wired up for the Inkling engine yet.",
"tools", "unsupported_parameter")
role_token = {"user": "<|message_user|>", "system": "<|message_system|>",
"developer": "<|message_system|>", "assistant": "<|message_model|>",
"tool": "<|message_tool|>"}
# Thinking effort — template default is 0.9, but at single-machine decode
# speeds unrequested reasoning burns the whole token budget before the answer
# starts, so we default it OFF unless the client asks.
effort_map = {"none": 0.0, "minimal": 0.1, "low": 0.2, "medium": 0.7,
"high": 0.9, "max": 0.99}
if reasoning_effort in effort_map:
eff = effort_map[reasoning_effort]
else:
eff = 0.9 if enable_thinking else 0.0
effort_str = ("<|message_system|><|content_text|>Thinking effort level: "
f"{0 if eff == 0.0 else eff}<|end_message|>")
prompt = []
effort_emitted = False
for index, message in enumerate(messages):
if not isinstance(message, dict):
raise APIError(400, "Each message must be an object.", f"messages.{index}")
role = message.get("role")
rtok = role_token.get(role)
if rtok is None:
raise APIError(400, f"Unsupported role {role!r}.", f"messages.{index}.role")
# the template emits the effort hint inline, right before the first
# non-system message — not at the end. Position matters: it changes the
# exact token sequence the model was trained on.
if not effort_emitted and role not in ("system", "developer"):
prompt.append(effort_str)
effort_emitted = True
open_turn = (not add_generation_prompt and role == "assistant"
and index == len(messages) - 1)
raw = message.get("content")
if audio_out is not None and role == "user" and isinstance(raw, list):
# multipart user content: text runs and audio clips become separate
# TMLv0 messages, in part order (a message carries ONE content type).
# Each DMel frame is one <|audio|> placeholder; the engine replaces
# those embeddings with the frames appended to audio_out.
for kind, val in inkling_content_segments(raw, f"messages.{index}.content", audio_out):
if kind == "text":
prompt.append(f"{rtok}<|content_text|>{val}<|end_message|>")
else:
prompt.append(f"{rtok}<|content_audio_input|>"
+ "<|audio|>" * val + "<|audio_end|><|end_message|>")
else:
text = content_text(raw, f"messages.{index}.content") if raw is not None else ""
# A continued turn is the last model message rendered open: no <|end_message|>.
terminator = "" if open_turn else "<|end_message|>"
prompt.append(f"{rtok}<|content_text|>{text}{terminator}")
if role == "assistant" and not open_turn:
prompt.append("<|content_model_end_sampling|>")
if not effort_emitted: # all-system edge case: fallback
prompt.append(effort_str)
if add_generation_prompt:
prompt.append("<|message_model|>") # generation cue
# Thinking off: prefill the content channel. Without this the model can still
# sample <|content_thinking|> as its first token (the effort hint is only a
# soft signal), open a reasoning block, and burn the whole token budget before
# reaching <|content_text|> — which the splitter then strips to an empty
# answer. Ending the prompt at <|message_model|><|content_text|> forces content
# mode; it is exactly the sequence every non-thinking turn is trained on.
if eff == 0.0:
prompt.append("<|content_text|>")
return "".join(prompt)
def render_chat(messages, enable_thinking=False, reasoning_effort=None, tools=None,
tool_choice=None, add_generation_prompt=True):
"""Render the text-only subset of the official GLM-5.2 chat template.
add_generation_prompt=False continues a trailing assistant turn. GLM has no per-turn
terminator (the next role token ends a turn), so the loop already renders that last message
as a past turn -- <|assistant|><think></think>{content} -- and suppressing the cue leaves
the prompt open on it, exactly as on glm53. Nothing to strip, unlike the ChatML families."""
if not isinstance(messages, list) or not messages:
raise APIError(400, "`messages` must be a non-empty array.", "messages")
prompt = ["[gMASK]<sop>"]
if enable_thinking:
# The endpoint accepts none/minimal/low/medium/high/xhigh, and this used
# to render every one of them except "high" as Max -- so a client asking
# for `minimal` got more reasoning than one asking for `high`, and the
# mapping was not even monotonic (#809). On a single machine that is not
# a cosmetic mismatch: unrequested reasoning spends the token budget
# before the answer starts.
#
# GLM-5.2's template takes a word here, not a number, so the levels map
# onto the ones it understands, in order. `none` cannot appear: it turns
# thinking off upstream and never reaches this branch.
effort = {"minimal": "Low", "low": "Low", "medium": "Medium",
"high": "High", "xhigh": "Max"}.get(reasoning_effort, "High")
prompt.append(f"<|system|>Reasoning Effort: {effort}")
forced = None
if isinstance(tool_choice, dict):
forced = _tool_choice_name(tool_choice)
if forced:
tools = [t for t in (tools or [])
if ((t.get("function", t) if isinstance(t, dict) else {}).get("name") == forced)]
elif tool_choice == "none":
tools = None # the client forbade tools: do not offer them
if tools:
# AUTHORITATIVE GLM-5.2 tool-declaration block (byte-matches chat_template.jinja): the
# `# Tools` + <tools></tools> XML structure is what the model was trained on. A made-up
# preamble makes it hallucinate other frameworks' syntax (e.g. `end_action`).
prompt.append("<|system|>\n# Tools\n\nYou may call one or more functions to assist with the "
"user query.\n\nYou are provided with function signatures within <tools></tools> "
"XML tags:\n<tools>\n")
for tool in tools:
fn = _tool_function(tool)
clean = {k: v for k, v in fn.items() if k not in ("defer_loading", "strict")}
prompt.append(json.dumps(clean, ensure_ascii=False) + "\n")
prompt.append("</tools>\n\nFor each function call, output the function name and arguments "
"within the following XML format:\n<tool_call>{function-name}"
"<arg_key>{arg-key-1}</arg_key><arg_value>{arg-value-1}</arg_value>"
"<arg_key>{arg-key-2}</arg_key><arg_value>{arg-value-2}</arg_value>...</tool_call>")
if forced:
prompt.append(f"\n\nYou must call the function `{forced}`. Do not answer directly.")
elif tool_choice == "required":
prompt.append(TOOL_CHOICE_REQUIRED_INSTRUCTION)
prev_tool = False
for index, message in enumerate(messages):
if not isinstance(message, dict):
raise APIError(400, "Each message must be an object.", f"messages.{index}")
role = message.get("role")
if role in ("system", "developer"):
prompt.append(f"<|system|>{content_text(message.get('content'), f'messages.{index}.content')}")
elif role == "user":
prompt.append(f"<|user|>{content_text(message.get('content'), f'messages.{index}.content')}")
elif role == "assistant":
# content may be null when the message is purely tool_calls
raw = message.get("content")
text = content_text(raw, f"messages.{index}.content") if raw is not None else ""
reasoning = message.get("reasoning_content")
if reasoning is None:
reasoning = ""
elif not isinstance(reasoning, str):
raise APIError(400, "`reasoning_content` must be a string.",
f"messages.{index}.reasoning_content")
prompt.append(f"<|assistant|><think>{reasoning}</think>{text.strip()}")
for tc in _history_tool_calls(message.get("tool_calls"), index):
fn = tc.get("function", tc)
args = fn.get("arguments", "{}")
if isinstance(args, str):
try:
args = json.loads(args)
except (json.JSONDecodeError, TypeError):
args = {}
if not isinstance(args, dict):
# `arguments` that is valid JSON but not an object ("[1,2]",
# "5", a bare list) reached .items() and raised
# AttributeError, which do_POST answers with HTTP 500. The
# same field is already tolerated when it does not parse at
# all, and every sibling renderer renders the call without
# arguments instead of failing; this is the one branch that
# was never completed.
args = {}
prompt.append(BOX_START + (fn.get("name") or ""))
for key, value in args.items():
prompt.append(f"<arg_key>{key}</arg_key><arg_value>"
+ (value if isinstance(value, str)
else json.dumps(value, ensure_ascii=False)) + "</arg_value>")
prompt.append(BOX_END)
elif role == "tool":
if not prev_tool: # one <|observation|> per consecutive tool run
prompt.append("<|observation|>")
prompt.append(TR_OPEN + content_text(message.get("content"), f"messages.{index}.content") + TR_CLOSE)
else:
raise APIError(400, f"Unsupported message role: {role!r}.",
f"messages.{index}.role", "unsupported_role")
prev_tool = (role == "tool")
if add_generation_prompt:
prompt.append("<|assistant|><think>" if enable_thinking else
"<|assistant|><think></think>")
return "".join(prompt)
# ---- immagini per GLM-5.3 ------------------------------------------------------------
# Il modello vede l'immagine come una sequenza di segnaposto <|image|>, uno per
# token che la torre produrra': (griglia_h/2) x (griglia_w/2). Il numero non e'
# negoziabile -- il motore rifiuta se non combacia con gli embedding che riceve --
# quindi si preprocessa PRIMA di rendere il prompt e si espande il segnaposto al
# numero giusto. Cosi' il renderer non deve sapere niente di immagini.
GLM53_IMAGE_OPEN, GLM53_IMAGE, GLM53_IMAGE_CLOSE = (
"<|begin_of_image|>", "<|image|>", "<|end_of_image|>")
def _image_part_url(part, kind):
"""The URL an image content part names. An `image_url` part carries an object,
`{"url": ...}`, and an `input_image` part the URL itself. `.get("url")` on an
`image_url` that was a string or a number raised AttributeError, which do_POST
answers with 500; that is the client's error, so it is a 400 here."""
value = part.get("image_url")
if kind != "image_url":
return value or part.get("url")
if value is None:
return None # _image_bytes_from_url answers the missing url
if not isinstance(value, dict):
raise APIError(400, "`image_url` must be an object with a `url`.", "messages")
return value.get("url")
def _image_bytes_from_url(url):
"""data: URI, file:// o percorso sul disco -> i byte dell'immagine.
A local path is read with the server process's own permissions, and an
inference client is not the operator: with the API key it could read any
file the process can reach ("file:///etc/passwd", or a bare "/etc/passwd").
#1354 refused '..' and confined reads to COLI_IMAGE_ROOT when set, which
left every absolute path readable on the default install (the variable is
unset out of the box). So local paths are now denied unless the operator
sets COLI_IMAGE_ROOT, and then only inside it (resolve() follows symlinks
before the relative_to() check, the same one serve_static uses). The
clients that used to send paths read the file themselves with the user's
own rights and send a data: URI: `coli chat` since this change, `coli web`
always did. Errors stay generic so a reply never confirms a path or its
permissions."""
if not isinstance(url, str) or not url:
raise APIError(400, "image_url.url must be a non-empty string.", "messages")
if url.startswith("data:"):
head, _, payload = url.partition(",")
if "base64" not in head:
raise APIError(400, "only base64 data: URIs are supported.", "messages")
import base64
try:
return base64.b64decode(payload, validate=True)
except Exception:
raise APIError(400, "image_url.url is not valid base64.", "messages")
if url.startswith("http://") or url.startswith("https://"):
# Scaricare da un URL che arriva in una richiesta vorrebbe dire far fare
# al server una chiamata di rete decisa da chi la manda. Non si fa.
raise APIError(400, "remote image URLs are not fetched; send the image "
"as a base64 data: URI or a path on this machine.",
"messages")
raw = url[7:] if url.startswith("file://") else url
image_root = os.environ.get("COLI_IMAGE_ROOT")
if not image_root:
raise APIError(400, "local image paths are disabled on this server: send the image "
"as a base64 data: URI (coli chat and coli web do), or start the "
"server with COLI_IMAGE_ROOT=<dir> to allow files under that "
"directory.", "messages")
if ".." in Path(raw).parts:
raise APIError(400, "image path is not allowed.", "messages")
try:
root = Path(image_root).resolve(strict=True)
if not root.is_dir():
raise ValueError("COLI_IMAGE_ROOT is not a directory")
target = Path(raw).resolve()
target.relative_to(root)
except (ValueError, OSError):
raise APIError(400, "image path is not allowed.", "messages")
try:
# An allowed directory can also contain a FIFO or device. Opening a
# FIFO in ordinary blocking mode would hold an API thread before the
# image decoder can reject it. Inspect the opened descriptor, rather
# than a pre-open path check, and never wait for a special-file writer.
fd = os.open(target, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0))
try:
if not stat.S_ISREG(os.fstat(fd).st_mode):
raise APIError(400, "local image paths must name regular files.", "messages")
with os.fdopen(fd, "rb") as handle:
fd = None # the file object now owns the descriptor
return handle.read()
finally:
if fd is not None:
os.close(fd)
except OSError:
raise APIError(400, "cannot read the requested image.", "messages")
# Qwen3.8 splices images as <|vision_start|> + N x <|image_pad|> + <|vision_end|>,
# and N is not a constant: the resolution is dynamic, so it comes from the grid
# the preprocessor chose. Hardcoding it would put the right vectors in the wrong
# number of slots, which the engine refuses rather than guesses about.
QWEN38_VISION_START = "<|vision_start|>"
QWEN38_IMAGE_PAD = "<|image_pad|>"
QWEN38_VISION_END = "<|vision_end|>"
def _preprocess_qwen38_image(data, model_dir, max_tokens=None):
try:
import sys as _sys
from pathlib import Path as _Path
_sys.path.insert(0, str(_Path(__file__).resolve().parent / "tools"))
from qwen38_image import preprocess
except ImportError as problem:
raise APIError(400, f"image support needs Pillow and numpy ({problem}).",
"messages")
return preprocess(data, model_dir, max_tokens)
def _text_part(part, index, position):
"""A text part's text for the image expanders, which join the parts themselves:
a `text` that is not a string reached "".join as TypeError, which do_POST
answers with 500, where content_text() already answers 400."""
text = part.get("text", "")
if not isinstance(text, str):
raise APIError(400, "Text content parts require a string `text` field.",
f"messages.{index}.content.{position}.text")
return text
def expand_qwen38_images(messages, model_dir, max_tokens=None):
"""Replace image parts with their placeholders and pull out the patches.
Returns (rewritten messages, images). The messages come back as plain text,
so the renderer treats them like any other turn."""
images = []
rewritten = []
for index, message in enumerate(messages):
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
rewritten.append(message)
continue
pieces = []
for position, part in enumerate(content):
if not isinstance(part, dict):
continue
kind = part.get("type")
if kind == "text":
pieces.append(_text_part(part, index, position))
elif kind in ("image_url", "input_image"):
url = _image_part_url(part, kind)
data = _image_bytes_from_url(url)
patches, grid_h, grid_w = _preprocess_qwen38_image(
data, model_dir, max_tokens)
tokens = (grid_h // 2) * (grid_w // 2)
images.append((patches, grid_h, grid_w))
pieces.append(QWEN38_VISION_START + QWEN38_IMAGE_PAD * tokens
+ QWEN38_VISION_END)
else:
raise APIError(400, f"unsupported content part {kind!r}.", "messages")
rewritten.append({**message, "content": "".join(pieces)})
return rewritten, images
def qwen36_has_vision(model_dir):
"""Whether a qwen36 container carries its vision tower (#1757). The converter
writes the tower's shape into qwen36_meta.json only when it copied the weights;
older containers, and text-only checkpoints, have none."""
if not model_dir:
return False
try:
meta = json.loads((Path(model_dir) / "qwen36_meta.json").read_text(encoding="utf-8"))
except (OSError, ValueError):
return False
return isinstance(meta, dict) and isinstance(meta.get("vision"), dict)
def has_image_parts(messages):
"""Whether any message carries a picture part (the shapes expand_*_images() read)."""
for message in messages if isinstance(messages, list) else ():
content = message.get("content") if isinstance(message, dict) else None
if isinstance(content, list) and any(
isinstance(part, dict) and part.get("type") in ("image_url", "input_image")
for part in content):
return True
return False
def expand_glm53_images(messages, model_dir):
"""Sostituisce le parti immagine coi loro segnaposto e ne estrae le patch.
Restituisce (messaggi riscritti, patch). I messaggi tornano con contenuto
testuale puro, quindi il renderer li tratta come qualunque altro turno."""
images = []
rewritten = []
for index, message in enumerate(messages):
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
rewritten.append(message)
continue
pieces = []
for position, part in enumerate(content):
if not isinstance(part, dict):
continue
kind = part.get("type")
if kind == "text":
pieces.append(_text_part(part, index, position))
elif kind in ("image_url", "input_image"):
url = _image_part_url(part, kind)
data = _image_bytes_from_url(url)
patches, grid_h, grid_w = _preprocess_image(data, model_dir)
tokens = (grid_h // 2) * (grid_w // 2)
images.append((patches, grid_h, grid_w))
pieces.append(GLM53_IMAGE_OPEN + GLM53_IMAGE * tokens + GLM53_IMAGE_CLOSE)
else:
raise APIError(400, f"unsupported content part {kind!r}.", "messages")
rewritten.append({**message, "content": "".join(pieces)})
return rewritten, images
# DeepSeek V4.1's image placeholder: every position of an image span carries the same
# id, and what each one MEANS follows from the aligner grid (image_processor.py
# image_token_types): start, then one newline per row of image tokens, then end. The
# engine rebuilds that layout from the grid it gets in the IMAGE frame, so the gateway
# only has to insert the right NUMBER of placeholders -- and getting that number wrong
# is the one failure the engine cannot paper over, which is why it refuses instead.
DSV41_IMAGE_PLACEHOLDER = "<|deepseek_image|>"
def expand_dsv41_images(messages, model_dir, max_tokens=None):
"""Replace image parts with their placeholder span and pull out the patches."""
images, rewritten = [], []
for index, message in enumerate(messages):
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
rewritten.append(message)
continue
pieces = []
for position, part in enumerate(content):
if not isinstance(part, dict):
continue
kind = part.get("type")
if kind == "text":
pieces.append(_text_part(part, index, position))
elif kind in ("image_url", "input_image"):
url = _image_part_url(part, kind)
data = _image_bytes_from_url(url)
patches, grid_h, grid_w, llm_h, llm_w = _preprocess_dsv41_image(
data, model_dir, max_tokens)
span = 1 + (llm_w + 1) * llm_h + 1
images.append((patches, grid_h, grid_w))
pieces.append(DSV41_IMAGE_PLACEHOLDER * span)
else:
raise APIError(400, f"unsupported content part {kind!r}.", "messages")
rewritten.append({**message, "content": "".join(pieces)})
return rewritten, images
def _preprocess_dsv41_image(data, model_dir, max_tokens=None):
try:
import sys as _sys
from pathlib import Path as _Path
_sys.path.insert(0, str(_Path(__file__).resolve().parent / "tools"))
from dsv41_image import preprocess
except ImportError as problem:
raise APIError(400, f"image support needs Pillow and numpy ({problem}).",
"messages")
return preprocess(data, model_dir, max_tokens)
def _preprocess_image(data, model_dir):
"""L'immagine nelle patch che la torre vuole. Il lavoro sta in
tools/glm53_image.py, verificato contro il processore ufficiale."""
try:
import sys as _sys
from pathlib import Path as _Path
_sys.path.insert(0, str(_Path(__file__).resolve().parent / "tools"))
from glm53_image import preprocess
except ImportError as problem:
raise APIError(400, f"image support needs Pillow and numpy ({problem}).",
"messages")
return preprocess(data, model_dir)
GLM53_TOOL_PREAMBLE = (
"<|system|>\n# Tools\n\n"
"You may call one or more functions to assist with the user query.\n\n"
"You are provided with function signatures within <tools></tools> XML tags:\n"
"<tools>\n")
GLM53_TOOL_EPILOGUE = (
"\n</tools>\n\n"
"For each function call, output the function name and arguments within the "
"following XML format:\n"
"<tool_call>{function-name}<arg_key>{arg-key-1}</arg_key>"
"<arg_value>{arg-value-1}</arg_value><arg_key>{arg-key-2}</arg_key>"
"<arg_value>{arg-value-2}</arg_value>...</tool_call>")
def _glm53_tool_json(tool):
"""Una firma di strumento come la serializza il template di GLM-5.3.
Le chiavi restano nell'ordine in cui il client le ha mandate, perche' il
template itera `tool.items()` e non le riordina; `defer_loading` e `strict`
non entrano nel prompt."""
if isinstance(tool, dict) and "function" in tool:
tool = tool["function"]
if not isinstance(tool, dict):
raise APIError(400, "each tool must be an object.", "tools")
parts = [f'"{key}": {json.dumps(value, ensure_ascii=False)}'
for key, value in tool.items()
if key not in ("defer_loading", "strict")]
return "{" + ", ".join(parts) + "}"
def _glm53_tool_block(tools):
"""Il blocco di dichiarazione, spaziatura compresa.
Gli a capo non sono decorativi: sono quelli che escono da chat_template.jinja
e il test li confronta byte a byte contro jinja2, perche' un prompt che
somiglia a quello dell'addestramento non e' quello dell'addestramento."""
body = "".join(f"\n{_glm53_tool_json(tool)}\n\n"
for tool in tools
if not _tool_function(tool).get("defer_loading"))
return GLM53_TOOL_PREAMBLE + body + GLM53_TOOL_EPILOGUE
def _glm53_tool_calls(calls):
"""Le chiamate di un turno assistente passato, nel formato che il modello
stesso produce: <tool_call>nome<arg_key>k</arg_key><arg_value>v</arg_value>.
Le stringhe passano cosi' come sono, il resto come JSON."""
out = []
for call in calls or []:
if isinstance(call, dict) and "function" in call:
call = call["function"]
name = (call or {}).get("name", "")
arguments = (call or {}).get("arguments")
if isinstance(arguments, str):
try:
arguments = json.loads(arguments)
except ValueError:
arguments = {}
if not isinstance(arguments, dict):
arguments = {} # same gap as render_chat above
pieces = [f"<tool_call>{name}"]
for key, value in arguments.items():
rendered = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
pieces.append(f"<arg_key>{key}</arg_key><arg_value>{rendered}</arg_value>")
pieces.append("</tool_call>")
out.append("".join(pieces))
return "\n" + "".join(out) + "\n" if out else ""
def render_chat_glm53(messages, enable_thinking=False, reasoning_effort=None, tools=None,
tool_choice=None, add_generation_prompt=True):
"""Render the text-only subset of the official GLM-5.3-Flash chat template.
Not a variant of the GLM-5.2 renderer above, and the differences are not
cosmetic. GLM-5.3 emits the reasoning-effort system line ALWAYS, because its
template defaults the effort to Max rather than leaving it unset; it accepts
only low, high and max, not the six-level ladder; its generation prompt
OPENS the reasoning block with a bare `<think>` where 5.2 closed it
immediately; and it declares tools with its own preamble and spacing.
What the two share is how a call comes BACK: both models emit
`<tool_call>name<arg_key>k</arg_key><arg_value>v</arg_value></tool_call>`,
so the existing parser needs nothing added for this family.
The whole thing is pinned byte for byte against chat_template.jinja rendered
with jinja2 (tests/glm53_chat_template_harness.py). Getting the prompt nearly
right is the failure mode worth guarding: the model answers either way.
"""
if not isinstance(messages, list) or not messages:
raise APIError(400, "`messages` must be a non-empty array.", "messages")
forced = None
if isinstance(tool_choice, dict):
forced = _tool_choice_name(tool_choice)
if forced:
tools = [t for t in (tools or [])
if ((t.get("function", t) if isinstance(t, dict) else {}).get("name") == forced)]
elif tool_choice == "none":
tools = None # il client li ha vietati: non si offrono
prompt = ["[gMASK]<sop>"]
# La riga di effort esce SEMPRE, come nel template: `effective_reasoning_effort`
# ha un ramo else che vale 'max', quindi non e' mai none. GLM-5.3 non ha un modo
# "non ragionare" -- in questo template `enable_thinking` non esiste proprio, e
# il prompt di generazione APRE sempre <think>.
#
# Quindi enable_thinking=False qui non puo' voler dire "spegni": vuol dire "il
# minimo che il modello supporta", cioe' Low. Il ragionamento avviene comunque;
# a nasconderlo e' il gateway, non il prompt.
#
# La forma che questo sostituisce -- nessuna riga di effort e <think></think>
# chiuso -- non esiste nel template e il modello non l'ha mai vista: e' la causa
# di #1278. Meta' di quella deviazione l'ho aggiunta io in #1282, giustificandola
# con un meccanismo poi misurato falso e ritirato pubblicamente sulla issue.
# Reso il template con jinja2 accanto a questo renderer, l'unica forma che sa
# produrre e':
# [gMASK]<sop><|system|>Reasoning Effort: {Low|High|Max}<|user|>..<|assistant|><think>
#
# low e high passano, tutto il resto e' Max: e' la scala del template, non la
# nostra. `none` non arriva qui, spegne il ragionamento a monte per le famiglie
# che possono davvero spegnerlo.
effort = {"minimal": "Low", "low": "Low", "medium": "High",
"high": "High", "xhigh": "Max"}.get(reasoning_effort,
"Max" if enable_thinking else "Low")
prompt.append(f"<|system|>Reasoning Effort: {effort}")
if tools:
prompt.append(_glm53_tool_block(tools))
# Fuori dal blocco, non dentro: il blocco e' confrontato byte a byte con
# jinja2 (glm53_chat_template_harness.py) e l'istruzione non c'e' nel
# template, quindi va dopo -- come fa render_chat per GLM-5.2.
if tool_choice == "required":
prompt.append(TOOL_CHOICE_REQUIRED_INSTRUCTION)
for index, message in enumerate(messages):
if not isinstance(message, dict):
raise APIError(400, "each message must be an object.", "messages")
role = message.get("role")
content = message.get("content")
if isinstance(content, list): # parti multimodali: solo il testo
texts = []
for position, part in enumerate(content):
if not isinstance(part, dict) or part.get("type") != "text":
continue
text = part.get("text", "")
# a non-string text reached "".join as TypeError, which do_POST answers 500
if not isinstance(text, str):
raise APIError(400, "Text content parts require a string `text` field.",
f"messages.{index}.content.{position}.text")
texts.append(text)
content = "".join(texts)
content = content or ""
if role == "user":
prompt.append(f"<|user|>{content}")
elif role == "system":
prompt.append(f"<|system|>{content}")
elif role == "tool":
prompt.append(f"<|observation|><tool_response>{content}</tool_response>")
elif role == "assistant":
reasoning = message.get("reasoning_content")
if not isinstance(reasoning, str) and "</think>" in content:
reasoning = content.split("</think>")[0].split("<think>")[-1]
content = content.split("</think>")[-1]
opened = f"<think>{reasoning}</think>" if isinstance(reasoning, str) else "<think></think>"
body = content.strip()
calls = _glm53_tool_calls(_history_tool_calls(message.get("tool_calls"), index))
# The template writes "\n<tool_call>". The model, on a turn that is
# nothing but a tool call, writes "</think><tool_call>" with no
# newline between them, and that one token is enough to throw away
# the whole cached prefix: the reuse gate in glm53.c is
# all-or-nothing, so the next turn re-prefills from scratch -- on a
# 3k-token agent history at 2.3 tok/s, twenty minutes (#1576).
#
# A turn that also has text is left exactly as it was. There the
# model's own trailing newline is stripped by .strip() and put back
# by the renderer, so the tokens already line up, and changing that
# case would break the one that works today.
if calls and not body:
calls = calls.lstrip("\n")
prompt.append(f"<|assistant|>{opened}{body}{calls}")
else:
raise APIError(400, f"unsupported message role {role!r}.", "messages")
# Il prompt di generazione apre il blocco, sempre, come il template:
# {%- if add_generation_prompt -%}<|assistant|>{{- '<think>' -}}{%- endif -%}
# Il vecchio commento qui sosteneva che <think></think> chiuso fosse "uno stato
# su cui il modello e' addestrato" perche' il template lo scrive davanti a un
# TURNO PASSATO senza ragionamento. E' vero per un turno passato e falso per il
# prompt di generazione: la posizione da cui il modello scrive non e' mai quella.
#
# add_generation_prompt=False non e' una forma nostra: e' l'altro ramo di questo stesso
# `if` nel template. Il prompt finisce allora sull'ultimo turno assistant reso come
# turno PASSATO -- <think></think> seguito dal contenuto -- e il modello lo prosegue
# invece di aprirne uno nuovo. Chi non chiede la prosecuzione non vede differenza:
# il ramo True e' invariato, byte per byte, ed e' quello che il test confronta.
if add_generation_prompt:
prompt.append("<|assistant|><think>")
return "".join(prompt)
# ---- DeepSeek V4.1 Flash -----------------------------------------------------------------
# The checkpoint ships its chat format as a Python module (encoding/encoding.py), not a
# jinja template, so this is that module's render_message transcribed for the shapes the
# gateway sends. Its pieces, with the vendor's own names:
#
# <|begin▁of▁sentence|> once, at the head of a fresh conversation
# <|System|> leads the conversation when there is a system message or a
# reasoning-effort line; also precedes a mid-conversation one
# Reasoning Effort: N index 0 only, thinking mode only, N in 1..100
# <|User|> / <|Assistant|>
# <think> or </think> the generation cue: open in thinking mode, closed otherwise
# assistant turns end with <|end▁of▁sentence|>
#
# Two details the reference hides in its control flow, both reproduced below. First,
# <think> and </think> straddle a turn boundary: the OPENING token is the transition the
# reference appends after a user turn, the CLOSING one belongs to the assistant turn that
# follows, so an assistant turn always starts with one or the other and there is no such
# thing as a past assistant turn without a </think>. Second, the reference drops the
# reasoning of turns before the last user message -- except when the conversation declares
# tools, where it keeps every one of them (`if any(m.get("tools") ...)`), because a tool
# call is only interpretable next to the reasoning that produced it.
DSV41_BOS = "<|begin▁of▁sentence|>"
DSV41_EOS = "<|end▁of▁sentence|>"
DSV41_SYSTEM = "<|System|>"
DSV41_USER = "<|User|>"
DSV41_ASSISTANT = "<|Assistant|>"
# encoding.py REASONING_EFFORT_MAPPINGS; the default there is "high"
DSV41_EFFORT = {"minimal": 50, "low": 50, "medium": 75, "high": 75, "xhigh": 100}
def _dsv41_tools_block(tools):
"""V4.1 tool-declaration block, rendered by the vendored reference template."""
schemas = []
for tool in (tools or []):
fn = _tool_function(tool)
# Gateway-side scrub: OpenAI clients attach routing hints the model
# schema must not carry.
clean = {k: v for k, v in fn.items() if k not in ("defer_loading", "strict")}
if isinstance(tool, dict) and tool.get("namespace") is not None:
clean = dict(clean, namespace=tool["namespace"])
schemas.append(clean)
return v41_dsml.render_tools(v41_dsml.tools_from_openai_format(schemas))
def _dsv41_merge_turns(messages):
"""encoding.py merge_tool_messages + sort_tool_results_by_call_order.
V4.1 has no standalone tool role: a tool result is a <tool_result> block inside the
NEXT user turn, and consecutive user-side messages collapse into one turn joined by
a blank line. Returns turns of {"role", "parts"/"content", ...}, validating each
original message so field-level errors keep pointing at the client's own indices.
"""
turns = []
for index, message in enumerate(messages):
if not isinstance(message, dict):
raise APIError(400, "Each message must be an object.", f"messages.{index}")
role = message.get("role")
if role == "developer":
role = "system"
if role not in ("system", "user", "assistant", "tool"):
raise APIError(400, f"Unsupported role {role!r}.", f"messages.{index}.role")
raw = message.get("content")
text = content_text(raw, f"messages.{index}.content") if raw is not None else ""
if role == "assistant":
reasoning = message.get("reasoning_content")
if reasoning is not None and not isinstance(reasoning, str):
raise APIError(400, "`reasoning_content` must be a string.",
f"messages.{index}.reasoning_content")
turns.append({"role": "assistant", "content": text,
"reasoning_content": reasoning,
"tool_calls": _history_tool_calls(message.get("tool_calls"), index)})
elif role in ("user", "tool"):
block = ({"kind": "tool_result", "id": message.get("tool_call_id") or "",
"text": v41_dsml.render_tool_result(text)} if role == "tool"
else {"kind": "text", "id": "", "text": text})
if turns and turns[-1]["role"] == "user":
turns[-1]["parts"].append(block)
else:
turns.append({"role": "user", "parts": [block]})
else:
turns.append({"role": "system", "content": text})
# A parallel-call round trip arrives in whatever order the client's tools finished;
# the model reads them positionally, so restore the order it called them in.
order = {}
for turn in turns:
if turn["role"] == "assistant" and turn.get("tool_calls"):
order = {}
for at, call in enumerate(turn["tool_calls"]):
identifier = call.get("id") if isinstance(call, dict) else None
if identifier:
order[identifier] = at
elif turn["role"] == "user" and order:
results = [p for p in turn["parts"] if p["kind"] == "tool_result"]
if len(results) > 1:
results.sort(key=lambda p: order.get(p["id"], 0))
feed = iter(results)
turn["parts"] = [next(feed) if p["kind"] == "tool_result" else p
for p in turn["parts"]]
return turns
def render_chat_dsv41(messages, enable_thinking=False, reasoning_effort=None, tools=None,
tool_choice=None, add_generation_prompt=True):
"""encoding.py _encode_messages_text for one turn.
Tool use follows the checkpoint's own DSML format (encoding/encoding.py, vendored in
v41_dsml.py): schemas are declared at the end of the system message, assistant tool
calls are <|DSML| calls> blocks, and tool results are <tool_result> blocks merged
into the following user turn.
add_generation_prompt=False continues a trailing assistant turn: the last turn is rendered
open -- its <think></think> block and content as a PAST turn, but without the closing
<|end▁of▁sentence|> and with no cue appended. That is the same drop-the-terminator move as
deepseek_v4, whose EOS this shares; the model resumes from the content it was handed.
"""
if not isinstance(messages, list) or not messages:
raise APIError(400, "`messages` must be a non-empty array.", "messages")
forced = None
if isinstance(tool_choice, dict):
forced = _tool_choice_name(tool_choice)
if forced:
tools = [t for t in (tools or [])
if ((t.get("function", t) if isinstance(t, dict) else {}).get("name") == forced)]
elif tool_choice == "none":
tools = None # the client forbade tools: do not offer them
turns = _dsv41_merge_turns(messages)
if tools:
tools_text = _dsv41_tools_block(tools)
if forced:
tools_text += f"\n\nYou must call the function `{forced}`. Do not answer directly."
elif tool_choice == "required":
tools_text += TOOL_CHOICE_REQUIRED_INSTRUCTION
for turn in turns:
if turn["role"] == "system":
turn["content"] += "\n\n" + tools_text
break
else:
# No system message: the reference renders tools on an empty one, so the block
# arrives as <|System|> + "\n\n" + tools. Keep that exact byte layout.
turns.insert(0, {"role": "system", "content": "\n\n" + tools_text})
budget = DSV41_EFFORT.get(reasoning_effort or "high", 75)
effort = (f"Reasoning Effort: {budget} (range 1-100, the higher the value, the more "
f"thorough the reasoning)\n\n") if enable_thinking else ""
# find_last_user_index: a mid-conversation system message counts as a user turn,
# because it is one of the two things the reference puts an assistant header after.
last_user = -1
for index, turn in enumerate(turns):
if turn["role"] == "user" or (turn["role"] == "system" and index > 0):
last_user = index
# With tools on the table the reference keeps every turn's reasoning; without them it
# drops the reasoning of everything before the last user message.
drop_thinking = not tools
prompt = [DSV41_BOS]
for index, turn in enumerate(turns):
role = turn["role"]
if role == "system":
prompt.append(DSV41_SYSTEM)
if index == 0:
prompt.append(effort)
prompt.append(turn["content"])
elif role == "user":
if index == 0 and effort:
prompt.append(DSV41_SYSTEM + effort)
prompt.append(DSV41_USER)
prompt.append("\n\n".join(part["text"] for part in turn["parts"]))
else:
keep = enable_thinking and (not drop_thinking or index > last_user)
prompt.append(DSV41_ASSISTANT)
prompt.append(f"<think>{turn.get('reasoning_content') or ''}</think>"
if keep else "</think>")
prompt.append(turn["content"])
if turn.get("tool_calls"):
prompt.append(v41_dsml.render_tool_calls(turn["tool_calls"]))
# A continued turn is the last message rendered open: no EOS, no cue below.
if add_generation_prompt or index != len(turns) - 1:
prompt.append(DSV41_EOS)
# the generation cue, exactly as render_message appends it after a user turn
if add_generation_prompt:
prompt.append(DSV41_ASSISTANT)
prompt.append("<think>" if enable_thinking and len(turns) - 1 >= last_user else "</think>")
return "".join(prompt)
# ---- continuing an unfinished assistant turn (COLI_CONTINUE_ASSISTANT) ----------------
# A trailing `assistant` message means "continue writing this turn", not "here is a turn I
# already finished". The official template says exactly that, and says it in one place --
# {%- if add_generation_prompt -%}<|assistant|>{{- '<think>' -}}{%- endif -%}
# -- whose False branch every renderer in this file hard-codes to True. With the cue
# suppressed the prompt ends mid-turn, on the shape the template writes in front of a PAST
# assistant turn, which is a position the model saw all through training.
#
# That distinction is what makes this safe on GLM-5.3 specifically. #1327 measured that a
# CLOSED, EMPTY <think></think> at the end of a prompt is out of distribution and the model
# keeps reasoning through it. The position here is a different one: <think></think> followed
# by real content, i.e. the past-turn shape, which is why a continuation must carry text.
#
# llama.cpp needs no switch for this because it runs the checkpoint's jinja at request time,
# so `add_generation_prompt=False` costs it nothing. This gateway renders by hand, on purpose
# and for speed (tests/glm53_chat_template_harness.py says why), and the bill for that choice is
# exactly here: one template flag, one open-turn shape to derive per renderer. Each string
# renderer derives its own, pinned byte-for-byte against the checkpoint's template;
# CONTINUATION_FAMILIES is the set that has done so. Kimi K3 differs in WHERE its shape lives:
# its prompt is framed engine-side (render_chat_kimi hands a K3CHAT1 record to kimi_k3.c, which
# assembles the XTML tokens), so its open turn is a `C` record here plus a branch in that C path,
# pinned by tests/test_k3_chat_tools.c against the tiny tokenizer rather than by a template diff.
# Families whose renderer implements the add_generation_prompt=False (open-turn) branch. A
# trailing assistant turn on a family NOT in this set falls through to the ordinary render
# (the cue is appended, exactly as before this existed) rather than erroring -- continuation is
# on by default, and a family without its open-turn shape yet must not start rejecting requests
# nobody opted into. Each renderer adds itself here in the same commit that derives its shape.
CONTINUATION_FAMILIES = {"glm53", "qwen38", "qwen36", "glm", "olmoe", "deepseek_v4", "inkling",
"kimi", "deepseek_v41", "mimo"}
def resolve_generation_prompt(messages, body):
"""Does this prompt end on a generation cue, or on an assistant turn to continue?
Returns True for the ordinary case (append the cue) and False for a continuation, which is
the template's `add_generation_prompt=False`.
Continuation is ON by default. A message list ending in a non-empty assistant turn already
says "continue me" -- the same contract as Anthropic's API -- and no OpenAI-compatible
client sends a trailing assistant turn by accident. It is deliberately NOT a request field:
a client would have to know colibri specifically to send one, and the clients that most
want this -- anything pointed at an OpenAI- or Anthropic-compatible URL -- send a message
list and nothing else.
COLI_CONTINUE_ASSISTANT=0 is the off-switch, for a deployment that wants the old behaviour
(fold the trailing turn into a completed one and append a fresh cue). It is the only value
that turns this off; anything else, including unset, leaves it on.
A family whose renderer has no open-turn shape yet (ARCH not in CONTINUATION_FAMILIES)
falls through to the ordinary render rather than erroring: continuation defaults on, so a
family added before its open-turn shape must not start rejecting trailing-assistant
requests that worked before. Every shipped family is in the set today, Kimi K3 included --
its open turn is framed in kimi_k3.c (a `C` record), not derived in the renderer here.
"""
continuing = os.environ.get("COLI_CONTINUE_ASSISTANT", "1") != "0"
last = messages[-1] if isinstance(messages, list) and messages else None
if not (isinstance(last, dict) and last.get("role") == "assistant"):
return True
where = f"messages.{len(messages) - 1}"
if not continuing:
return True
if ARCH not in CONTINUATION_FAMILIES:
return True # open-turn shape not derived for this family yet -- render as before
if body.get("tools") or body.get("functions"):
raise APIError(400, "A continued assistant turn cannot be combined with `tools`: "
"the tool-call parsers read an assistant turn from its start, and a "
"continuation can end anywhere -- including inside a <tool_call> "
"block.", "tools", "unsupported_parameter")
if last.get("tool_calls"):
raise APIError(400, "A continued `assistant` message cannot carry `tool_calls`.",
f"{where}.tool_calls", "unsupported_value")
if len(messages) < 2:
raise APIError(400, "A continued `assistant` turn needs a preceding turn to "
"continue from.", "messages")
raw = last.get("content")
if isinstance(raw, list): # multimodal parts: only the text counts
text = "".join(part.get("text", "") for part in raw
if isinstance(part, dict) and part.get("type") == "text")
elif raw is None:
text = ""
elif isinstance(raw, str):
text = raw
else:
raise APIError(400, "Message content must be a string or an array of blocks.",
f"{where}.content")
if not text.strip():
raise APIError(400, "A continued `assistant` turn needs text to continue. An empty "
"one ends the prompt on a closed, empty <think></think> block, which "
"is the out-of-distribution position #1327 removed -- the model "
"reasons straight through it instead of answering.",
f"{where}.content", "invalid_value")
if text != text.rstrip():
raise APIError(400, "A continued `assistant` turn cannot end with whitespace: the "
"template strips it, so the model would resume from different bytes "
"than the ones sent. Put the space at the start of what you expect "
"back instead.", f"{where}.content", "invalid_value")
return False
def render_chat_for_arch(messages, enable_thinking=False, reasoning_effort=None, tools=None,
tool_choice=None, audio_out=None, add_generation_prompt=True,
preserve_thinking=False):
"""Render a chat request with the active engine's native prompt contract.
`add_generation_prompt=False` (a continued assistant turn) is implemented for the families
in CONTINUATION_FAMILIES. resolve_generation_prompt() passes any other family through with
the cue appended, so it never reaches here with the flag False; this stays as the backstop,
because silently appending a cue to a continuation is the exact failure this exists to remove.
"""
if not add_generation_prompt and ARCH not in CONTINUATION_FAMILIES:
raise APIError(400, f"Continuing an assistant turn is not implemented for {ARCH!r}.",
"messages", "unsupported_parameter")
if ARCH == "inkling":
return render_chat_inkling(messages, enable_thinking, reasoning_effort, tools,
tool_choice, audio_out=audio_out,
add_generation_prompt=add_generation_prompt)
if ARCH == "glm53":
return render_chat_glm53(messages, enable_thinking, reasoning_effort, tools,
tool_choice, add_generation_prompt)
if chat_flavor() == "qwen38":
return render_chat_qwen38(messages, enable_thinking, reasoning_effort, tools,
tool_choice, add_generation_prompt)
if chat_flavor() == "qwen3_coder":
return render_chat_qwen3_coder(messages, tools, tool_choice, add_generation_prompt)
if ARCH == "qwen36":
return render_chat_qwen(messages, enable_thinking, reasoning_effort, tools,
tool_choice, add_generation_prompt, preserve_thinking)
if ARCH == "glm":
return render_chat(messages, enable_thinking, reasoning_effort, tools,
tool_choice, add_generation_prompt)
if ARCH == "olmoe":
return render_chat_olmoe(messages, enable_thinking, reasoning_effort, tools,
tool_choice, add_generation_prompt)
if ARCH == "deepseek_v4":
return render_chat_v4(messages, enable_thinking, reasoning_effort, tools,
tool_choice, add_generation_prompt)
if ARCH == "kimi":
return render_chat_kimi(messages, enable_thinking, reasoning_effort, tools,
tool_choice, add_generation_prompt=add_generation_prompt)
if ARCH == "deepseek_v41":
return render_chat_dsv41(messages, enable_thinking, reasoning_effort, tools,
tool_choice, add_generation_prompt)
if ARCH == "mimo":
return render_chat_mimo(messages, enable_thinking, reasoning_effort, tools,
tool_choice, add_generation_prompt)
return render_chat(messages, enable_thinking, reasoning_effort, tools, tool_choice)
# ---- Anthropic Messages API (#343) --------------------------------------------------------
# A translation layer, NOT a second engine path: /v1/messages rewrites an Anthropic-shaped
# request into the exact OpenAI-shaped body the existing path already validates, so prompt
# rendering, scheduling, generation and tool parsing stay single-sourced. Only the request
# translation and the response/SSE shapes are new. Claude Code is the reference client.
ANTHROPIC_LOCAL_SIGNATURE = "colibri-local" # opaque compatibility metadata, not a crypto proof
def starts_in_reasoning(enable_thinking, add_generation_prompt=True):
"""Se l'uscita del modello comincia DENTRO al blocco di ragionamento.
Dipende da come il prompt lo ha lasciato, e ogni famiglia lo lascia come
dice il suo interruttore: acceso apre il blocco e il modello lo chiude da
solo, spento lo chiude gia' il prompt e quello che torna e' risposta pura.
Se le due cose non concordano il ragionamento finisce incollato davanti
alla risposta, che e' il difetto che questa funzione esiste per non avere.
GLM-5.3 e' l'eccezione che rende la regola esplicita: il suo template non
ha un interruttore, render_chat_glm53 apre <think> SEMPRE, e "thinking
spento" vuol dire solo effort Low. L'uscita comincia dentro al blocco in
ogni caso; partire in modalita' testo perche' il client ha detto False e'
esattamente il ragionamento incollato davanti alla risposta di #1278.
Il turno proseguito (add_generation_prompt=False) e' il terzo stato, e non
lo dice l'interruttore: il prompt finisce sull'ultimo turno assistant reso
come turno PASSATO, quindi <think></think> GIA' CHIUSO seguito dal
contenuto, col ragionamento acceso o spento che sia. Il modello riprende in
modalita' testo; se lo splitter parte in modalita' ragionamento aspetta un
</think> che e' gia' passato, e archivia come ragionamento tutta la
risposta -- content vuoto, reasoning_content pieno, stop pulito. Misurato
su glm53 int4, CPU: 10 e 109 caratteri di ragionamento contro
zero di risposta, col prompt corretto sul filo. Vale anche per glm53: la
regola di famiglia sopra dice dove comincia un turno NUOVO, e il turno
proseguito non ne apre nessuno."""
return (enable_thinking or ARCH == "glm53") and add_generation_prompt
class ThinkingStreamSplit:
"""Split GLM's reasoning marker without leaking markers across stream chunks.
`thinking_spans` and `text_spans` are this stage's maps, in its own input's
coordinates. Every character is routed to exactly one bucket or is marker syntax
routed to neither, and which bucket decides which array a token record belongs to.
Like the stop filter's map they are built only when `track_spans` is set, so a request
that never asked for per-token logprobs pays for none of this."""
MARKERS = (THINK_OPEN, THINK_CLOSE)
def __init__(self, on_thinking, on_text, on_thinking_end=None, initial_thinking=True,
track_spans=False):
self.on_thinking = on_thinking
self.on_text = on_text
self.on_thinking_end = on_thinking_end
# #597: GLM emits reasoning only when the prompt opened <think> (thinking on);
# with thinking off the prompt already closed it, so output is pure answer and
# the splitter must start in text mode or it would file the whole answer as reasoning.
self.thinking = initial_thinking
self.buf = ""
self.track_spans = track_spans
self.thinking_spans = []
self.text_spans = []
self.consumed = 0 # input characters resolved, marker or not
def _emit(self, text):
if text:
(self.on_thinking if self.thinking else self.on_text)(text)
if not self.track_spans:
return
if text:
spans = self.thinking_spans if self.thinking else self.text_spans
_append_span(spans, self.consumed, self.consumed + len(text),
spans[-1][2] + (spans[-1][1] - spans[-1][0]) if spans else 0)
# Every emitted run is a prefix of `buf`, so the run's input start is always
# `consumed` and advancing it here keeps that true for the next one.
self.consumed += len(text)
def feed(self, chunk):
self.buf += chunk
while True:
hits = [(offset, marker) for marker in self.MARKERS
if (offset := self.buf.find(marker)) >= 0]
if hits:
offset, marker = min(hits, key=lambda hit: hit[0])
self._emit(self.buf[:offset])
self.buf = self.buf[offset + len(marker):]
if self.track_spans:
self.consumed += len(marker) # the marker itself reaches neither bucket
if marker == THINK_CLOSE and self.thinking:
self.thinking = False
if self.on_thinking_end:
self.on_thinking_end()
continue
hold = 0
for size in range(1, min(len(self.buf), max(map(len, self.MARKERS)) - 1) + 1):
if any(marker.startswith(self.buf[-size:]) for marker in self.MARKERS):
hold = size
flush = len(self.buf) - hold
if flush:
self._emit(self.buf[:flush])
self.buf = self.buf[flush:]
return
def finish(self):
self._emit(self.buf)
self.buf = ""
close = finish # interface parity with InklingStreamSplit in the streaming path
def split_thinking_reply(text, enable_thinking=True, add_generation_prompt=True):
"""Return the marker-free (thinking, answer) portions of one GLM reply."""
thinking, answer, _spans = _split_thinking(text, enable_thinking, False,
add_generation_prompt)
return thinking, answer
def split_thinking_reply_spans(text, enable_thinking=True, add_generation_prompt=True):
"""split_thinking_reply plus the split's maps: `(thinking, answer, (thinking_map,
answer_map))`, both in `text`'s own coordinates."""
return _split_thinking(text, enable_thinking, True, add_generation_prompt)
def _split_thinking(text, enable_thinking, track_spans, add_generation_prompt=True):
"""The split itself. One implementation behind both spellings, so the maps can never
describe a different routing than the text they came with."""
thinking, answer = [], []
split = ThinkingStreamSplit(thinking.append, answer.append,
initial_thinking=starts_in_reasoning(enable_thinking,
add_generation_prompt),
track_spans=track_spans)
split.feed(text)
split.finish()
return "".join(thinking), "".join(answer), (split.thinking_spans, split.text_spans)
def _anthropic_image_part(block, where):
"""An Anthropic `image` block as the OpenAI-shaped part expand_*_images() reads.
A base64 source becomes the same data: URI a client of /v1/chat/completions would
send, so both endpoints hand the expander one picture in one shape. A `url` source
is passed through as-is: the expander already refuses to fetch remote URLs, and
the Anthropic client must meet that refusal, not a second one."""
source = block.get("source")
if not isinstance(source, dict):
raise APIError(400, "`image` blocks require a `source` object.", f"{where}.source")
kind = source.get("type")
if kind == "base64":
media_type, data = source.get("media_type"), source.get("data")
if not isinstance(media_type, str) or not media_type:
raise APIError(400, "`image.source.media_type` is required for base64 sources.",
f"{where}.source.media_type")
if not isinstance(data, str) or not data:
raise APIError(400, "`image.source.data` is required for base64 sources.",
f"{where}.source.data")
return {"type": "image_url", "image_url": {"url": f"data:{media_type};base64,{data}"}}
if kind == "url":
url = source.get("url")
if not isinstance(url, str) or not url:
raise APIError(400, "`image.source.url` is required for url sources.",
f"{where}.source.url")
return {"type": "image_url", "image_url": {"url": url}}
raise APIError(400, "`image.source.type` must be base64 or url.", f"{where}.source.type",
"unsupported_content_type")
def _anthropic_block_text(blocks, param, images=None):
"""Text out of an Anthropic content array (tool_result content is the same shape).
With `images`, a list, `image` blocks are allowed and their OpenAI-shaped parts are
appended to it instead of raising: a tool_result can carry a picture (Claude Code's
Read on an image file does), and the caller decides which turn it rides on."""
if isinstance(blocks, str):
return blocks
if not isinstance(blocks, list):
raise APIError(400, "Content must be a string or an array of blocks.", param)
parts = []
for index, block in enumerate(blocks):
where = f"{param}.{index}"
if isinstance(block, dict) and block.get("type") == "image" and images is not None:
images.append(_anthropic_image_part(block, where))
continue
if not isinstance(block, dict) or block.get("type") != "text":
raise APIError(400, "Colibri currently supports text blocks only here.",
where, "unsupported_content_type")
if not isinstance(block.get("text"), str):
raise APIError(400, "Text blocks require a string `text` field.", f"{where}.text")
parts.append(block["text"])
return "".join(parts)
def anthropic_to_openai(body):
"""Anthropic request -> (messages, tools, tool_choice) in OpenAI shape."""
messages = []
system = body.get("system")
if isinstance(system, str):
if system:
messages.append({"role": "system", "content": system})
elif isinstance(system, list):
text = _anthropic_block_text(system, "system")
if text:
messages.append({"role": "system", "content": text})
elif system is not None:
raise APIError(400, "`system` must be a string or an array of text blocks.", "system")
raw = body.get("messages")
if not isinstance(raw, list) or not raw:
raise APIError(400, "`messages` must be a non-empty array.", "messages")
for index, message in enumerate(raw):
if not isinstance(message, dict):
raise APIError(400, "Each message must be an object.", f"messages.{index}")
role = message.get("role")
if role not in ("user", "assistant"):
raise APIError(400, f"Input message role {role!r} is not supported. Anthropic messages are "
"`user` or `assistant`; a system prompt goes in the top-level `system`.",
f"messages.{index}.role", "unsupported_role")
content = message.get("content")
if isinstance(content, str):
messages.append({"role": role, "content": content})
continue
if not isinstance(content, list):
raise APIError(400, "Message content must be a string or an array of blocks.",
f"messages.{index}.content")
# `parts` keeps the user's text and image blocks in the order the client put
# them; `carried` collects pictures found inside tool_result blocks, which
# ride the user turn that follows the tool turns (the tool role is text).
texts, parts, carried, reasoning, calls, results = [], [], [], [], [], []
for j, block in enumerate(content):
where = f"messages.{index}.content.{j}"
if not isinstance(block, dict):
raise APIError(400, "Each content block must be an object.", where)
kind = block.get("type")
if kind == "text":
if not isinstance(block.get("text"), str):
raise APIError(400, "Text blocks require a string `text` field.", f"{where}.text")
texts.append(block["text"])
parts.append({"type": "text", "text": block["text"]})
elif kind == "image":
if role != "user":
raise APIError(400, "`image` blocks are valid only in user messages.",
where, "unsupported_content_type")
parts.append(_anthropic_image_part(block, where))
elif kind == "thinking":
if role != "assistant":
raise APIError(400, "`thinking` blocks are valid only in assistant messages.",
f"{where}.type", "unsupported_content_type")
if not isinstance(block.get("thinking"), str):
raise APIError(400, "Thinking blocks require a string `thinking` field.",
f"{where}.thinking")
if not isinstance(block.get("signature"), str):
raise APIError(400, "Thinking blocks require a string `signature` field.",
f"{where}.signature")
reasoning.append(block["thinking"])
elif kind == "tool_use":
name = block.get("name")
if not isinstance(name, str) or not name:
raise APIError(400, "`tool_use` blocks require a string `name`.", f"{where}.name")
arguments = block.get("input")
if arguments is None:
arguments = {}
if not isinstance(arguments, dict):
raise APIError(400, "`tool_use.input` must be an object.", f"{where}.input")
calls.append({"id": block.get("id") or ("toolu_" + uuid.uuid4().hex[:24]),
"type": "function",
"function": {"name": name,
"arguments": json.dumps(arguments, ensure_ascii=False)}})
elif kind == "tool_result":
pictures = []
results.append({"role": "tool",
"tool_call_id": block.get("tool_use_id") or "",
"content": _anthropic_block_text(block.get("content", ""),
f"{where}.content", pictures)})
if pictures and role != "user":
raise APIError(400, "`image` blocks are valid only in user messages.",
where, "unsupported_content_type")
carried.extend(pictures)
else:
raise APIError(400, "Colibri supports `text`, `image`, `tool_use` and "
"`tool_result` content blocks only.", f"{where}.type",
"unsupported_content_type")
# tool results precede the user's own text: they answer the previous assistant turn
messages.extend(results)
text = "".join(texts)
if role == "assistant":
if text or reasoning or calls:
entry = {"role": "assistant", "content": text or None}
if reasoning:
entry["reasoning_content"] = "".join(reasoning)
if calls:
entry["tool_calls"] = calls
messages.append(entry)
elif carried or any(part["type"] != "text" for part in parts):
# A picture is in this turn: the content stays a parts list, which is
# what expand_*_images() reads. Pictures out of tool results come first,
# right after the tool turns that produced them.
messages.append({"role": "user", "content": carried + parts})
elif text or not results:
messages.append({"role": "user", "content": text})
return messages
def anthropic_tools(body):
"""Anthropic tools/tool_choice -> OpenAI shape (validated downstream by generation_options)."""
raw = body.get("tools")
if raw is None:
tools = None
elif not isinstance(raw, list):
raise APIError(400, "`tools` must be an array.", "tools")
else:
tools = []
for index, tool in enumerate(raw):
if not isinstance(tool, dict):
raise APIError(400, "Each tool must be an object.", f"tools.{index}")
name = tool.get("name")
if not isinstance(name, str) or not name:
raise APIError(400, "Each tool requires a string `name`.", f"tools.{index}.name")
schema = tool.get("input_schema")
if schema is not None and not isinstance(schema, dict):
raise APIError(400, "`input_schema` must be an object.", f"tools.{index}.input_schema")
function = {"name": name, "parameters": schema or {"type": "object", "properties": {}}}
if isinstance(tool.get("description"), str):
function["description"] = tool["description"]
tools.append({"type": "function", "function": function})
tools = tools or None
choice = body.get("tool_choice")
if choice is None:
return tools, None
if not isinstance(choice, dict):
raise APIError(400, "`tool_choice` must be an object.", "tool_choice")
kind = choice.get("type")
if kind == "auto":
return tools, "auto"
if kind == "any":
return tools, "required"
if kind == "none":
return tools, "none"
if kind == "tool":
name = choice.get("name")
if not isinstance(name, str) or not name:
raise APIError(400, "`tool_choice.name` is required when type is `tool`.",
"tool_choice.name")
return tools, {"type": "function", "function": {"name": name}}
raise APIError(400, "`tool_choice.type` must be auto, any, none, or tool.", "tool_choice.type",
"unsupported_value")
# Generic whitespace-tolerant JSON grammar for response_format {"type": "json_object"}.
# Draft-source semantics: positions with one legal byte draft; jws points just keep
# the walker alive through the model's own spacing (see docs/grammar-draft.md).
GENERIC_JSON_GBNF = (
'root ::= jws jval jws\n'
'jval ::= jobj | jarr | jstr | jnum | "true" | "false" | "null"\n'
'jobj ::= "{" jws ( jstr jws ":" jws jval jws ( "," jws jstr jws ":" jws jval jws )* )? "}"\n'
'jarr ::= "[" jws ( jval jws ( "," jws jval jws )* )? "]"\n'
'jstr ::= "\\"" jchar* "\\""\n'
'jchar ::= [^"\\\\\\x00-\\x1f] | "\\\\" ( ["\\\\/bfnrt] | "u" jhex jhex jhex jhex )\n'
'jhex ::= [0-9a-fA-F]\n'
'jnum ::= "-"? ( "0" | [1-9] [0-9]* ) ( "." [0-9]+ )? ( ( "e" | "E" ) ( "+" | "-" )? [0-9]+ )?\n'
'jws ::= ( " " | "\\t" | "\\n" | "\\r" )*\n'
)
DEFAULT_CHAT_STOP_SEQUENCES = ("<|user|>", "<|observation|>")
# Seconds to wait for the engine to exit on its own after stdin EOF (its
# atexit teardown writes HEAT_FILE). EOF is only observed between turns,
# so an in-flight generation delays exit; override for impatient scripts.
_ENGINE_DRAIN_S = float(os.environ.get("COLI_ENGINE_DRAIN_S", "30"))
def parse_stop_sequences(body):
value = body.get("stop")
if value is None:
return ()
if isinstance(value, str):
sequences = [value]
elif isinstance(value, list):
sequences = value
else:
raise APIError(400, "`stop` must be a string or an array of strings.",
"stop", "invalid_value")
if not 1 <= len(sequences) <= 4:
raise APIError(400, "`stop` must contain between 1 and 4 sequences.",
"stop", "invalid_value")
for index, sequence in enumerate(sequences):
if not isinstance(sequence, str) or not sequence:
raise APIError(400, "Each `stop` sequence must be a non-empty string.",
f"stop.{index}", "invalid_value")
return tuple(sequences)
def conversation_cache_slot(messages, kv_slots):
"""Stable KV slot for a conversation so its turns reuse the same cached prefix.
The chat APIs are stateless: every turn resends the whole history, and the engine
caches each KV slot's prefix. When the client does not pin a `cache_slot`, the
scheduler falls back to `min(free_slots)`, which is blind to which slot already
holds this conversation. Under any interleaving of clients a turn can then land on
another conversation's slot and force a full re-prefill (#634, Defect 1). Hashing a
key that stays constant across a conversation's turns — the leading system messages
plus the first user message, which never change once the conversation has started —
routes every turn of one conversation to the same slot. Distinct conversations
spread across slots; when there are more live conversations than slots, colliding
ones degrade to the old re-prefill behaviour rather than to anything worse.
Returns a slot in [0, kv_slots). Falls back to 0 when there is nothing to key on.
"""
if kv_slots <= 1 or not isinstance(messages, list) or not messages:
return 0
prefix = []
for message in messages:
prefix.append(message)
if isinstance(message, dict) and message.get("role") == "user":
break # first user turn reached: the key is now stable for the whole conversation
try:
key = json.dumps(prefix, sort_keys=True, default=str)
except (TypeError, ValueError):
key = repr(prefix)
digest = hashlib.sha1(key.encode("utf-8", "replace")).digest()
return int.from_bytes(digest[:8], "big") % kv_slots
def stop_policy(body, chat):
sequences = parse_stop_sequences(body)
ignore_leading = body.get("x_colibri_ignore_leading_stop", False)
if not isinstance(ignore_leading, bool):
raise APIError(400, "`x_colibri_ignore_leading_stop` must be a boolean.",
"x_colibri_ignore_leading_stop", "invalid_value")
if chat and ARCH == "glm" and not sequences:
# The GLM chat template owns these role boundaries, so generic OpenAI
# clients should not need model-specific stop knowledge. Inkling has a
# different marker family and receives no implicit GLM stops. Treat an
# occasional leading GLM marker patiently; client-provided stops remain
# strict unless the extension is explicitly requested.
return DEFAULT_CHAT_STOP_SEQUENCES, True
return sequences, ignore_leading
class StopFilter:
"""Stream text without exposing a full or partial stop sequence.
`spans` is this stage's map: the intervals of the raw stream -- the concatenation of
everything ever fed -- that reached `emit`, each with where it landed in the emitted
text. It is built only when `track_spans` is set, and the thinking split and the
tool-call parse are gated the same way, so a request that never asked for per-token
logprobs has no stage compose or retain a map for it."""
def __init__(self, sequences, emit, ignore_leading=False, track_spans=False):
self.sequences = tuple(sequences)
self.emit = emit
self.ignore_leading = ignore_leading
self.pending = ""
self.matched = None
self.useful_content_seen = False
self.leading_matches_ignored = 0
self.track_spans = track_spans
self.spans = []
# Raw offset of self.pending[0], equivalently of feed()'s `text[0]`. Everything the
# filter drops is accounted for by advancing this without emitting.
self.raw_base = 0
self.emitted = 0
def _emit(self, text, raw_start):
if text:
self.emit(text)
if self.track_spans:
_append_span(self.spans, raw_start, raw_start + len(text), self.emitted)
self.emitted += len(text)
if text.strip():
self.useful_content_seen = True
def feed(self, chunk):
if self.matched is not None:
return
text = self.pending + chunk
base = self.raw_base
self.pending = ""
while True:
match = None
for order, sequence in enumerate(self.sequences):
offset = text.find(sequence)
candidate = (offset, order, sequence)
if offset >= 0 and (match is None or candidate[:2] < match[:2]):
match = candidate
if match is None:
break
offset, _order, sequence = match
prefix = text[:offset]
if (self.ignore_leading and not self.useful_content_seen
and not prefix.strip()):
self.leading_matches_ignored += 1
text = text[offset + len(sequence):]
# The ignored marker and the blank prefix in front of it never reach the
# client, so no span covers them and a record inside one resolves to
# "not emitted" instead of derailing the alignment.
base += offset + len(sequence)
if not text:
self.raw_base = base
return
continue
self.matched = sequence
self._emit(prefix, base)
return
hold = 0
maximum = min(len(text), max((len(s) - 1 for s in self.sequences), default=0))
for size in range(1, maximum + 1):
suffix = text[-size:]
if any(sequence.startswith(suffix) for sequence in self.sequences):
hold = size
flush = len(text) - hold
if flush:
self._emit(text[:flush], base)
self.pending = text[flush:]
self.raw_base = base + flush
def finish(self):
if self.matched is None and self.pending:
self._emit(self.pending, self.raw_base)
self.pending = ""
def stopped(self):
return self.matched is not None
class ToolSideband:
"""Request-scoped K3 TOOL frames with the same stop policy as DATA."""
def __init__(self, enabled, sequences, ignore_leading=False):
self.enabled = enabled
self.seen = False
self.parts = []
self.filter = (StopFilter(sequences, self.parts.append, ignore_leading)
if enabled else None)
def feed(self, chunk):
self.seen = True
self.filter.feed(chunk)
def stopped(self):
return bool(self.filter and self.filter.stopped())
def finish(self):
if self.filter:
self.filter.finish()
def reply(self):
return "".join(self.parts) if self.seen else None
def validate_tools(body):
"""Refuse a malformed `tools`/`functions` with a 400 naming the field. The chat renderers
iterate the list and parse_tool_calls reads each schema's `properties` and `required`, so
this has to run before either: chat_completion() calls it ahead of rendering, and
generation_options() calls it again for every other path."""
tools_raw = body.get("tools") or body.get("functions")
if tools_raw is not None:
if not isinstance(tools_raw, list):
raise APIError(400, "`tools` must be a non-empty array.", "tools", "invalid_value")
if not tools_raw:
raise APIError(400, "`tools` must be a non-empty array.", "tools", "invalid_value")
for idx, tool in enumerate(tools_raw):
if not isinstance(tool, dict):
raise APIError(400, f"Each tool must be an object, got {type(tool).__name__} at index {idx}.",
f"tools.{idx}", "invalid_value")
fn = tool.get("function", tool) if isinstance(tool, dict) else {}
if not isinstance(fn, dict):
raise APIError(400, f"Tool function must be an object at index {idx}.",
f"tools.{idx}.function", "invalid_value")
if not fn.get("name"):
raise APIError(400, f"Each tool must have a `name` at index {idx}.",
f"tools.{idx}.function.name", "invalid_value")
if not isinstance(fn["name"], str):
raise APIError(400, f"Tool `name` must be a string at index {idx}.",
f"tools.{idx}.function.name", "invalid_value")
params = fn.get("parameters")
if params is None:
continue
if not isinstance(params, dict):
raise APIError(400, f"Tool `parameters` must be an object at index {idx}.",
f"tools.{idx}.function.parameters", "invalid_value")
if params.get("properties") is not None and not isinstance(params["properties"], dict):
raise APIError(400, f"Tool `parameters.properties` must be an object at index {idx}.",
f"tools.{idx}.function.parameters.properties", "invalid_value")
if params.get("required") is not None and not isinstance(params["required"], list):
raise APIError(400, f"Tool `parameters.required` must be an array at index {idx}.",
f"tools.{idx}.function.parameters.required", "invalid_value")
def generation_options(body, limit):
if body.get("n", 1) != 1:
raise APIError(400, "Colibri currently supports `n=1` only.", "n", "unsupported_value")
best_of = body.get("best_of", 1)
if best_of not in (None, 1):
raise APIError(400, "Colibri currently supports `best_of` equal to 1 only.",
"best_of", "unsupported_value")
logit_bias = body.get("logit_bias")
if logit_bias not in (None, {}):
raise APIError(400, "Colibri does not support a non-empty `logit_bias` yet.",
"logit_bias", "unsupported_value")
if body.get("suffix") is not None:
raise APIError(400, "Colibri does not support `suffix` infill yet.",
"suffix", "unsupported_parameter")
modalities = body.get("modalities")
if modalities is not None:
if (not isinstance(modalities, list) or not modalities or
any(not isinstance(item, str) for item in modalities)):
raise APIError(400, "`modalities` must be a non-empty array of strings.",
"modalities", "invalid_value")
if "audio" in modalities:
raise APIError(400, "Colibri does not support audio output via `modalities`.",
"modalities", "unsupported_value")
if any(item != "text" for item in modalities):
raise APIError(400, "Colibri supports only text output via `modalities`.",
"modalities", "unsupported_value")
# `tools`/`functions` are handled by render_chat (declaration) + parse_tool_calls (output).
validate_tools(body)
choice = body.get("tool_choice")
if choice is not None:
if isinstance(choice, str):
if choice not in ("auto", "none", "required"):
raise APIError(400, "`tool_choice` must be one of \"auto\", \"none\", \"required\", "
"or a function object.", "tool_choice", "unsupported_value")
elif isinstance(choice, dict):
name = _tool_choice_name(choice)
if not name:
raise APIError(400, "`tool_choice` function object must include a name.",
"tool_choice", "invalid_value")
declared = [(t.get("function", t) if isinstance(t, dict) else {}).get("name")
for t in (body.get("tools") or body.get("functions") or [])]
if name not in declared:
raise APIError(400, f"`tool_choice` names {name!r}, which is not in `tools`.",
"tool_choice", "invalid_value")
else:
raise APIError(400, "`tool_choice` must be a string or a function object.",
"tool_choice", "invalid_value")
if choice != "none" and not (body.get("tools") or body.get("functions")):
raise APIError(400, "`tool_choice` requires `tools`.", "tool_choice", "invalid_value")
stop_sequences = parse_stop_sequences(body)
if body.get("frequency_penalty", 0) or body.get("presence_penalty", 0):
raise APIError(400, "Token penalties are not supported yet.", None, "unsupported_parameter")
# `seed` is accepted for request-shape compatibility and silently discarded:
# this server puts no per-request seed on the wire, at any temperature.
# response_format -> optional per-request grammar for the engine's grammar-forced
# draft source (#70/#148). NEVER a sampling constraint: drafts are verified, so a
# schema the engine cannot compile degrades to "no speedup", not to an error and
# not to changed output. json_schema payloads are forwarded as-is (the engine
# compiles them via schema_gbnf.h); {"type": "gbnf"} is a raw-GBNF extension.
grammar = None
response_format = body.get("response_format")
if response_format is not None and response_format != {"type": "text"}:
if not isinstance(response_format, dict) or "type" not in response_format:
raise APIError(400, "`response_format` must be an object with a `type`.",
"response_format", "invalid_value")
ftype = response_format["type"]
if ftype == "json_object":
grammar = GENERIC_JSON_GBNF
elif ftype == "json_schema":
json_schema = response_format.get("json_schema")
schema = json_schema.get("schema") if isinstance(json_schema, dict) else None
if not isinstance(schema, dict):
raise APIError(400, "`response_format.json_schema.schema` must be an object.",
"response_format", "invalid_value")
grammar = json.dumps(schema)
elif ftype == "gbnf":
grammar = response_format.get("grammar")
if not isinstance(grammar, str) or not grammar.strip():
raise APIError(400, "`response_format.grammar` must be a non-empty GBNF string.",
"response_format", "invalid_value")
else:
raise APIError(400, "`response_format.type` must be \"text\", \"json_object\", "
"\"json_schema\" or \"gbnf\".",
"response_format", "unsupported_value")
if grammar is not None and len(grammar.encode("utf-8")) > (1 << 20):
raise APIError(400, "`response_format` grammar/schema exceeds 1 MiB.",
"response_format", "invalid_value")
maximum = body.get("max_completion_tokens")
maximum_param = "max_completion_tokens"
if maximum is None:
maximum = body.get("max_tokens")
maximum_param = "max_tokens"
if maximum is None:
# Client omitted max_tokens: honor the operator's configured budget (--max-tokens /
# --ngen), not an arbitrary 256 — `coli serve --ngen 32768` must mean 32768 (#382).
# Generation still ends at EOS, so this is a cap, not a target.
maximum = limit
temperature = body.get("temperature")
top_p = body.get("top_p")
if temperature is None:
# The launcher publishes --temp through COLI_TEMP (#509, #968). The
# gateway must use that value as its request default or the SERVE frame
# replaces it with 0.7 before any engine can honor the setting.
try:
temperature = float(os.environ.get("COLI_TEMP", "0.7"))
if not math.isfinite(temperature) or not 0 <= temperature <= 2:
temperature = 0.7
except ValueError:
temperature = 0.7
top_p = 0.9 if top_p is None else top_p
if isinstance(maximum, bool) or not isinstance(maximum, int) or maximum < 1:
raise APIError(400, f"`{maximum_param}` must be a positive integer.", maximum_param)
if maximum > limit:
maximum = limit # clamp to the server's --max-tokens cap instead of 400 (#260): OpenAI
# clients (opencode/ai-sdk) default to large max_tokens; rejecting breaks them.
if (isinstance(temperature, bool) or not isinstance(temperature, (int, float)) or
not math.isfinite(temperature) or not 0 <= temperature <= 2):
raise APIError(400, "`temperature` must be between 0 and 2.", "temperature")
if (isinstance(top_p, bool) or not isinstance(top_p, (int, float)) or
not math.isfinite(top_p) or not 0 < top_p <= 1):
raise APIError(400, "`top_p` must be greater than 0 and at most 1.", "top_p")
return maximum, float(temperature), float(top_p), grammar, stop_sequences
# The per-token top-k emission cap the engine's numeric logprobs channel (the SUBMIT
# `logprobs=` key) supports, mirrored in c/decode_batch.h as COLI_SUBMIT_TOPK_MAX. The
# public request range is bound to exactly that interface: 1..32, a named 400 above it.
LOGPROBS_TOP_K_CAP = 32
def logprobs_options(body, chat, engine_supports):
"""Validate the request's logprobs/echo/top_logprobs fields and translate them into
`(engine_k, echo, display_k)`.
- `engine_k` is the SUBMIT `logprobs=` value to send; 0 leaves the per-token numeric
channel off entirely, so no ECHO or extended DATA frames are emitted.
- `echo` is whether the response includes the prompt-echo positions. Chat has no echo
concept, so it is always False there and `echo: true` on chat is a named 400.
- `display_k` is how many `top_logprobs` alternatives the client asked to see, 0 to
LOGPROBS_TOP_K_CAP. It can be lower than `engine_k` when a chat request asks for
logprobs with `top_logprobs: 0`.
Completions' `logprobs` is the legacy integer top-k count; chat's is a boolean gate
plus a separate `top_logprobs` count. Each endpoint takes its own shape and refuses
the other's by name.
`null`, `false` and `0` all normalise to absent and are never errors: on completions
the request then succeeds with `choices[].logprobs: null`, and on chat `true` for
`logprobs` is the only value that opens the gate. `top_logprobs` is type- and
range-checked even when `logprobs` is off, so a malformed value is never silently
ignored because a sibling field made it moot; a valid one with `logprobs` off is a
documented no-op. `top_logprobs` is not read at all on completions, where it is not
part of the request shape.
`echo: true` with no active `logprobs` request is a named 400 rather than a silent
no-op: the prompt echo is built out of the engine's per-token records, so without them
there is nothing to echo, and a 200 that quietly drops a requested field is the shape
this surface exists to stop."""
def _require_logprobs_for_echo(wanted):
if wanted:
raise APIError(400, "`echo` requires `logprobs`.", "echo",
"unsupported_parameter")
echo = body.get("echo", False)
if echo is None:
echo = False # null == absent, same as `logprobs`
if not isinstance(echo, bool):
raise APIError(400, "`echo` must be a boolean.", "echo", "invalid_value")
if chat:
if echo:
raise APIError(400, "`echo` is not supported for chat completions.",
"echo", "unsupported_parameter")
logprobs = body.get("logprobs", False)
if logprobs is None:
logprobs = False # null == absent == no logprobs
if not isinstance(logprobs, bool):
raise APIError(400, "`logprobs` must be a boolean.", "logprobs", "invalid_value")
display_k, param = body.get("top_logprobs", 0), "top_logprobs"
if display_k is None:
display_k = 0 # null == absent, same as `logprobs`
else:
logprobs = body.get("logprobs")
if logprobs is None or logprobs is False:
_require_logprobs_for_echo(echo)
return 0, False, 0
display_k, param = logprobs, "logprobs"
if (isinstance(display_k, bool) or not isinstance(display_k, int) or
not 0 <= display_k <= LOGPROBS_TOP_K_CAP):
raise APIError(400, f"`{param}` must be an integer between 0 and "
f"{LOGPROBS_TOP_K_CAP}.", param, "invalid_value")
if chat and not logprobs:
return 0, False, 0 # top_logprobs validated above; logprobs off is still a no-op
if not chat and display_k == 0:
_require_logprobs_for_echo(echo)
return 0, False, 0 # 0 = no logprobs, documented
if not engine_supports:
# A sender-side capability gate: these endpoints request the numeric per-token
# channel only from an engine this server has pinned for it, and refusing before
# generate() builds the extension header keeps a rejected-SUBMIT payload drain
# unreachable. The message names the gate that refused, not the engine's arch.
raise APIError(400, "Log probabilities are not requested from this engine by "
"these endpoints.", "logprobs", "unsupported_parameter")
return max(1, display_k), echo, display_k
def _json_float(value):
"""A logprob the engine's numeric channel emits as nan/inf/-inf must serialise as JSON
`null`, never the invalid-JSON literals json.dumps would otherwise write and never
clamped to a made-up finite number."""
return value if math.isfinite(value) else None
def _keepalive_choice(chat, visible):
"""The streamed keepalive's single choice, in the endpoint's own chunk shape.
A chat chunk (`chat.completion.chunk`) carries a `delta`; a legacy
`/v1/completions` chunk (`text_completion`) carries `text`, never `delta`.
Emitting a `delta` on the completions stream produces a chunk with no `text`,
which a strict client (the OpenAI SDK models `CompletionChoice.text` as
required) rejects. Chat has a side channel for the diagnostic marker
(reasoning_content, off the visible answer); completions does not, so its
keepalive stays an empty `text` — a "." there would land in the completion —
and still resets the client's idle timer."""
if chat:
return {"index": 0, "delta": {"reasoning_content": "." if visible else ""},
"logprobs": None, "finish_reason": None}
return {"index": 0, "text": "", "logprobs": None, "finish_reason": None}
def _order_echo_records(prompt_records):
"""Place each prompt-echo record at its own wire `pos` index rather than trusting the
order the frames arrived in, and return `(payload bytes, record)` pairs -- the same
shape generated-token records already arrive in.
A duplicate, negative, out-of-range or (by pigeonhole) missing position among the N
records that must fill slots 0..N-1 raises RuntimeError, the class every other
malformed-engine-output path here raises. One shape is told apart from that: a
contiguous run starting above 0, which is what a pin snapshot produces rather than a
malformed engine. That raises EchoPrefixPinned, which the caller answers by name."""
positions = [record["pos"] for record in prompt_records]
if (positions and all(isinstance(p, int) and not isinstance(p, bool)
for p in positions)
and sorted(positions) == list(range(min(positions),
min(positions) + len(positions)))
and min(positions) > 0):
raise EchoPrefixPinned(min(positions))
ordered = [None] * len(prompt_records)
for record in prompt_records:
pos = record["pos"]
if (not isinstance(pos, int) or isinstance(pos, bool)
or not 0 <= pos < len(ordered) or ordered[pos] is not None):
raise RuntimeError(
f"invalid engine ECHO position {pos!r} (expected each of "
f"0..{len(ordered) - 1} exactly once, got {len(prompt_records)} records)")
ordered[pos] = (record["bytes"], record)
return ordered
def _generated_record_texts(generated_records):
"""Each generated record's own decoded text, from one stateful incremental UTF-8
decoder over the generated sequence alone, with the trailing flush landing on the last
record.
This is the decode the engine's `on_text` callback performed while the stop filter was
watching -- same bytes, same fresh decoder, same final flush -- so the running sum of
these lengths is the raw-stream character coordinate the filter's span map is expressed
in. Nothing else may be used to derive record spans."""
decoder = codecs.getincrementaldecoder("utf-8")("replace")
texts = [decoder.decode(data) for data, _record in generated_records]
tail = decoder.decode(b"", final=True)
if tail and texts:
texts[-1] += tail
return texts
def _echo_decoded_texts(prompt_records, generated_records):
"""`(prompt texts, generated texts)` from a single stateful incremental UTF-8 decoder
spanning both halves, prompt-echo positions first.
A codepoint can arrive split across the prompt/generated seam: its leading byte on the
last echo frame, its continuation bytes on the first data frame. One decoder spanning
both resolves that into one character; two independent decodes turn each half into
replacement characters, and `text` and `text_offset` then disagree about how long that
character is. This is therefore not the same decode as _generated_record_texts: the
span map is counted in that function's coordinates, while these texts are what the
echo reconstruction shows."""
decoder = codecs.getincrementaldecoder("utf-8")("replace")
prompt_texts = [decoder.decode(data) for data, _record in _order_echo_records(prompt_records)]
generated_texts = [decoder.decode(data) for data, _record in generated_records]
tail = decoder.decode(b"", final=True)
if tail:
if generated_texts:
generated_texts[-1] += tail
elif prompt_texts:
prompt_texts[-1] += tail
return prompt_texts, generated_texts
def _logprob_positions(prompt_records, generated_records, texts=None):
"""Merge prompt-echo and generated-token records into one ordered list of
`{"text", "raw_lp", "topk", "bytes"}` dicts: echoed positions first, reassembled by
wire `pos`, then generated tokens in emission order.
`texts` supplies each position's text when the caller already decoded them -- the
response path has, because it needs the generated half truncated to what the stop
filter emitted. `bytes` always stays that frame's own raw payload regardless of what
text, if any, it decoded to."""
raw = _order_echo_records(prompt_records)
raw += generated_records
if texts is None:
prompt_texts, generated_texts = _echo_decoded_texts(prompt_records, generated_records)
texts = prompt_texts + generated_texts
return [{"text": text, "raw_lp": record["lp"], "topk": record["topk"], "bytes": data}
for (data, record), text in zip(raw, texts)]
def _own_token_label(entry, topk, idx):
"""The label for one top-k candidate: the position's own text when the candidate is
the chosen token, otherwise `<token_id:N>`.
The wire's top-k table carries raw candidate token ids and no decoded text, and there
is no server-side tokenizer. The one candidate that can be labelled without decoding is
the position's own chosen token: its table entry and its `lp` come from the same
computation, so an exact float match identifies it without comparing token ids.
The engine prints logprobs to six decimal digits, so two candidates can legitimately
share the chosen token's printed value. With no id to break the tie, only the first
table entry in wire order whose value matches is labelled as the chosen token; a later
match is labelled by its raw id like any other unidentified candidate. The table is
unsorted on the wire, so this never assumes the first entry is the argmax."""
tid, tlp = topk[idx]
if tlp != entry["raw_lp"]:
return f"<token_id:{tid}>"
if any(topk[j][1] == entry["raw_lp"] for j in range(idx)):
return f"<token_id:{tid}>"
return entry["text"]
def _refuse_duplicate_candidate(label, seen, requested, param):
"""Refuse a top-k table that repeats a label within one position.
The two response shapes would answer it differently: the legacy completions
`top_logprobs` is a JSON object, so the second entry overwrites the first and the
client silently receives fewer candidates than the engine supplied, while chat's
`content[].top_logprobs` is a list and keeps both. Refused on both surfaces, because a
client cannot tell the two apart from the outside.
`engine_duplicate_logprob_candidate` describes the table the engine sent rather than
this function's reaction to it. `param` is the caller's own parameter -- `logprobs` on
legacy completions, `top_logprobs` on chat -- so a client branching on the code is told
which of its fields is unanswerable."""
if label in seen:
raise APIError(
500, f"The colibri engine sent a top-k table carrying the duplicate "
f"candidate {label!r} at a single position ({requested} "
f"requested).", param, "engine_duplicate_logprob_candidate",
"server_error")
seen.add(label)
def _completions_logprobs_object(prompt_records, generated_records, display_k, texts=None):
"""Build the legacy `/v1/completions` `logprobs` object: `tokens[]`,
`token_logprobs[]`, `top_logprobs[]` (one dict per position, keyed by token text) and
`text_offset[]` (character offsets into the reconstructed text, counted from 0).
The first prompt position's `token_logprobs` entry comes out null because the engine's
own echo position 0 carries a non-finite sentinel -- there is nothing to condition the
first token on. `texts` is each position's text when the caller already has it, and
`text_offset` is counted from those same strings, so the offsets always index the text
this object's own `tokens` join to."""
positions = _logprob_positions(prompt_records, generated_records, texts)
tokens = [p["text"] for p in positions]
token_logprobs = [_json_float(p["raw_lp"]) for p in positions]
top_logprobs = []
for p in positions:
table = {}
displayed = p["topk"][:display_k]
seen = set()
for idx, (tid, tlp) in enumerate(displayed):
label = _own_token_label(p, displayed, idx)
_refuse_duplicate_candidate(label, seen, len(displayed), "logprobs")
table[label] = _json_float(tlp)
top_logprobs.append(table)
text_offset = []
offset = 0
for text in tokens:
text_offset.append(offset)
offset += len(text)
return {"tokens": tokens, "token_logprobs": token_logprobs,
"top_logprobs": top_logprobs, "text_offset": text_offset}
def _chat_logprobs_content(generated_records, display_k, texts=None):
"""Build chat completions' `choices[].logprobs.content[]`: one
`{token, logprob, bytes, top_logprobs}` entry per generated token, each alternative a
`{token, logprob, bytes}`. Chat has no echo concept, so records are used in emission
order.
Every record's text is decoded first, in one pass, including the final flush, before
any entry is built: building entries interleaved with decoding would let the last
entry's `token` gain the flushed text while its own candidate label stayed stale.
`texts` supplies each entry's token text when the caller already has it -- the chat
path passes the text each record contributed to `message.content`, which is a slice of
that record's own decoding wherever a stop, a thinking split or a tool-call parse cut
through the middle of a token."""
if texts is None:
texts = _generated_record_texts(generated_records)
content = []
for (data, record), text in zip(generated_records, texts):
entry = {"text": text, "raw_lp": record["lp"], "topk": record["topk"]}
alternatives = []
displayed = record["topk"][:display_k]
seen = set()
for idx, (tid, tlp) in enumerate(displayed):
label = _own_token_label(entry, displayed, idx)
_refuse_duplicate_candidate(label, seen, len(displayed), "top_logprobs")
alternatives.append({"token": label, "logprob": _json_float(tlp),
"bytes": list(data) if label == text else None})
content.append({"token": text, "logprob": _json_float(record["lp"]),
"bytes": list(data), "top_logprobs": alternatives})
return content
def _records_cover_stream(generated_records, text, span_map):
"""True when the generated records account for the whole raw stream the filter emitted
from -- the question every logprobs-bearing response must answer before its arrays may
claim to describe the text.
Both sides are character counts in raw-stream coordinates. The records' extent is the
running sum _generated_record_texts() produces, which is the coordinate system the span
map is expressed in; the filter's reach is the far end of that map. No decoded string
is compared against another anywhere here, which is what keeps a legitimately
transformed text -- an unflushed decoder, or the echo reconstruction's own seam decode
-- from reading as a gap and turning a working request into a 500.
`text` is read only when the map is empty, where it is the one thing that tells
"nothing was emitted" apart from "nothing was recorded". `span_map` takes no default: a
degenerate "assume everything was emitted" arm here would be unreachable and therefore
untested, which is the worst thing a safety check can contain.
No records at all is a gap like any other whenever text was emitted: the arrays would
describe nothing while the response carries a completion. It is only vacuously covered
when nothing was emitted either."""
extent = sum(len(piece) for piece in _generated_record_texts(generated_records))
if not span_map:
# Not vacuously true: an empty map means either that nothing was emitted -- and
# `text` is then empty too -- or that a filter was built without `track_spans`
# while text was emitted. `text` tells the two apart.
return not text
# Every span, not just the last: a map can carry a hole, and a run that ends inside the
# records while an earlier one runs past them is still a gap.
return all(high <= extent for _low, high, _out in span_map)
def _align_generated_records(generated_records, text, span_map, display_texts=None):
"""Locate every generated-token record in `text` by raw-stream span and return
`(data, record, emitted_text)` for each record that contributed at least one character
to it, in `text` order, which is also record order.
`span_map` is the composed map from raw-stream coordinates to `text`'s: the stop
filter's `spans` alone when `text` is the stop-filtered stream, or that chained with
the thinking split's and the tool-call parse's when `text` is `message.content`. Each
record occupies `[sum(len(decoded_j) for j < k), + len(decoded_k))`, and where that
lands is a lookup rather than a prefix match of decoded bytes against a string the
stage has already rewritten.
A record whose span is fully inside the map is kept whole; fully outside, dropped;
partially inside, kept with the characters that were emitted. A kept record's `data` is
narrowed to the bytes of the characters it reports whenever it was truncated, so
`bytes` can never describe more than `token`.
`display_texts` is the echo reconstruction's own decode, where the generated half is
decoded by the decoder that already consumed the prompt bytes. A record that survives
whole reports that reading directly; a record that is both cut and straddling has the
real character spliced back onto the raw-stream slice, which is exact because the two
decodes differ only over that leading run. A cut falling inside the straddling
codepoint itself has no right answer, and keeps the raw slice.
`span_map` takes no default; None means "every character of `text` was emitted, from
the start of the stream", which a caller with no filter in front of it must ask for
rather than fall into."""
if span_map is None:
span_map = [(0, len(text), 0)] if text else []
texts = _generated_record_texts(generated_records)
aligned, offset = [], 0
for index, (data, record) in enumerate(generated_records):
piece = texts[index]
start, offset = offset, offset + len(piece)
if not piece:
# A frame that only completes a pending codepoint decodes to "" and owns no
# span, so its fate is that of the character its bytes went into. Testing the
# position instead keeps the entry even when that character was deleted.
if _project_span(span_map, start, start + 1):
aligned.append((data, record, ""))
continue
image = _project_span(span_map, start, offset)
if image is None:
continue
kept = image[1] - image[0]
emitted = text[image[0]:image[1]]
alternate = display_texts[index] if display_texts is not None else piece
if alternate != piece:
if kept == len(piece):
# Survived whole: the seam decode is this record's text. Continuation
# bytes that complete nothing give `alternate == ""`, which is right.
emitted = alternate
else:
# Cut and straddling: the raw-stream decode opens with `head` replacement
# characters where the seam decode has one real one, agreeing from there
# on. Splice it back only where that is exact.
head = len(piece) - len(alternate) + 1
if kept >= head and _project_span(span_map, start, start + kept) == image:
emitted = alternate[:1] + emitted[head:]
if emitted != piece:
# `bytes` must never describe more than `token`, so a truncated entry carries
# the bytes it reports; an untruncated one keeps the frame's own payload.
data = emitted.encode("utf-8")
aligned.append((data, record, emitted))
return aligned
def _completion_choice_fields(stats, *, raw_text, text, chat, engine_k, echo, display_k,
prompt_echoes, stop_spans, content_spans, cache_slot):
"""The stop/trim/echo/logprobs tail every non-streaming completion shares, in one
place. Returns `(text, logprobs_obj, finish_reason)`.
Callers differ only in what they do before this point -- the inkling split, the
thinking split, the tool-call parse and the tool sideband. After it they differ in
nothing, so the stop/trim/echo interaction exists here once instead of once per
completion shape.
`raw_text` is the pre-split, pre-tool-parse text the stop filter emitted; `text` is
what the caller intends to return, after whichever splits it applies. They are the same
string when no split fired. `chat` selects the chat `content`/`refusal` shape over the
legacy completions object; `engine_k`, `echo` and `display_k` come from
logprobs_options(). `prompt_echoes` is the echo records the caller collected through
its own `on_echo` sink. `cache_slot` is carried only so a pinned prefix can be named in
the one error that reports it.
`stop_spans` is the stop filter's own map, raw stream to `raw_text`; `content_spans` is
`(target text, composed map)` chaining it onward to the string the arrays must describe
-- `message.content` on the chat path -- or None when no further stage can vouch for
one. Every parameter but `stats` is keyword-only, and the span parameters take no
default: `raw_text` and `text` are same-typed neighbours whose transposition is silent,
and a defaulted span would leave whichever call site forgot to pass one on unaligned
behaviour while the other was fixed.
This holds no state between calls. Every incremental decoder involved is created per
call and the span maps arrive as arguments, so a caller that assembles several
completions in a row cannot carry one's trailing partial codepoint or coordinates into
the next.
`finish_reason` is the engine's own length/stop reason; a caller that can finish for a
reason of its own overrides it."""
finish_reason = "length" if stats["length_limited"] else "stop"
logprobs_obj = None
if engine_k:
channel = stats.get("logprobs") or {"generated": []}
prompt_records = prompt_echoes if echo else []
target, target_spans = ((content_spans[0], content_spans[1]) if content_spans
else (raw_text, stop_spans))
# The arrays describe the WHOLE text on every shape that carries them, not only
# the one that rebuilds `text` out of them: short records, or none at all, would
# otherwise ship a 200 whose arrays describe a prefix and say nothing about it.
if not _records_cover_stream(channel["generated"], target, target_spans):
raise APIError(
500, "The colibri engine sent per-token logprob records that do not "
"cover the generated text, so the logprobs arrays would describe "
"only part of it.", "echo" if echo else "logprobs",
"engine_logprob_records_incomplete", "server_error")
if echo and not prompt_records:
# An engine that sends the generated records and no ECHO frames has answered
# half the opt-in, and the prompt echo is the half `echo` asked for.
raise APIError(
500, "The colibri engine sent no prompt-echo records, so `echo` "
"cannot be answered for this prompt.", "echo",
"engine_logprob_records_incomplete", "server_error")
try:
prompt_texts, seam_texts = _echo_decoded_texts(prompt_records,
channel["generated"])
except EchoPrefixPinned as pinned:
# The engine's own state, not a bad request and not a broken engine: this slot
# holds a pin snapshot covering the first `prefix` tokens, which the prefill
# never reads out, so `echo` cannot be answered in full for this prompt.
raise APIError(
503, f"The colibri engine cannot echo this prompt: KV slot "
f"{cache_slot} holds a pinned prefix of {pinned.prefix} "
f"token(s), which the prefill does not read out, so the "
f"echoed positions would start at {pinned.prefix} rather "
f"than 0.", "echo", "engine_pinned_prefix_not_echoed",
"server_error") from None
aligned = _align_generated_records(channel["generated"], target, target_spans,
display_texts=seam_texts if echo else None)
generated = [(data, record) for data, record, _emitted in aligned]
emitted_texts = [emitted for _data, _record, emitted in aligned]
if chat:
logprobs_obj = {"content": _chat_logprobs_content(generated, display_k,
texts=emitted_texts),
"refusal": None}
else:
logprobs_obj = _completions_logprobs_object(
prompt_records, generated, display_k, texts=prompt_texts + emitted_texts)
if not chat and echo:
# The legacy shape concatenates the prompt and the completion in `text` when
# `echo` is true and `text_offset` indexes that concatenation, so `text` is
# rebuilt from the reconstruction the offsets are counted from rather than
# from two independently decoded halves.
text = "".join(logprobs_obj["tokens"])
return text, logprobs_obj, finish_reason
def _engine_extension_args(engine_k):
"""The `Engine.generate()` keyword arguments that carry this request's SUBMIT
extension, built once from the parsed options and spread at each call site.
An empty mapping when nothing was requested, which is what keeps a plain request's
header byte-identical to one built with no extension at all. When the channel is
requested it also asks for the seventh numeric field, which coli_submit_parse expects
before the first key=value token.
That is this helper's decision for THESE endpoints, not a property of the request
builder. `gbytes_before_ext` is a per-call-site parameter, and the other caller that
sends an extension key -- /v1/systemone's scoring -- does not pass it and keeps the header it has
always sent; changing that endpoint's wire is not this change's to make."""
if not engine_k:
return {}
return {"logprobs": engine_k, "gbytes_before_ext": True}
def engine_exit_status(status):
"""A child's exit status in words: a signal on POSIX (a SIGKILL is most often the
out-of-memory killer), the NTSTATUS on Windows."""
if status < 0:
name = {9: "SIGKILL, most often the out-of-memory killer", 11: "SIGSEGV", 6: "SIGABRT"}.get(-status)
return f"killed by signal {-status}" + (f", {name}" if name else "")
if status >= 0xC0000000:
name = {0xC0000005: "access violation", 0xC0000017: "out of memory", 0xC00000FD: "stack overflow",
0xC0000409: "fail-fast or stack buffer overrun"}.get(status)
return f"exit status 0x{status:08X}" + (f", {name}" if name else "")
return f"exit status {status}"
class EngineLoadError(RuntimeError):
"""The engine said why it could not load, and exited before READY.
`LOAD_FAIL kind=<kind> <detail>` is the last line such an engine writes on the
handshake channel (see docs/serve_protocol.md): `kind` is nomem, io, format or
unsupported, `detail` the text it also wrote to stderr. Raised in place of the
bare "engine exited unexpectedly" so the log names the cause: an out-of-memory
host is not a bad file."""
def __init__(self, kind, detail):
super().__init__(f"colibri engine failed to load (kind={kind}): {detail}")
self.kind = kind
self.detail = detail
def parse_load_fail(data):
"""(kind, detail) from the last LOAD_FAIL line among the bytes an engine wrote
before it should have said READY; None when it wrote no such line."""
found = None
for line in data.decode("utf-8", "replace").splitlines():
fields = line.strip().split(None, 2)
if len(fields) >= 2 and fields[0] == "LOAD_FAIL" and fields[1].startswith("kind="):
found = (fields[1][len("kind="):], fields[2] if len(fields) > 2 else "")
return found
def read_engine_turn(stream, sentinel, on_bytes, caps=None):
pending = b""
while True:
byte = stream.read(1)
if byte == b"":
# The sentinel-length tail was held back in case it began the
# sentinel; at EOF it is the engine's last words (a LOAD_FAIL line
# ends there), so hand it over before giving up.
if pending:
on_bytes(pending)
raise RuntimeError("colibri engine exited unexpectedly")
pending += byte
if pending.endswith(sentinel):
data = pending[:-len(sentinel)]
if data:
on_bytes(data)
break
if len(pending) > len(sentinel):
on_bytes(pending[:-len(sentinel)])
pending = pending[-len(sentinel):]
# CAPS key=value ... between READY and STAT: what the engine loaded (a vision
# tower or not), said BEFORE the status line so a server knows the modalities
# it serves before it takes its first request. Engines that predate the line
# say nothing, and `caps` stays as the caller left it.
while True:
fields = stream.readline().decode("utf-8", "replace").strip().split()
if fields[:1] != ["CAPS"]:
break
if caps is not None:
caps.update(entry.partition("=")[::2] for entry in fields[1:] if "=" in entry)
if len(fields) < 5 or fields[0] != "STAT":
raise RuntimeError(f"invalid engine status: {' '.join(fields)}")
return {
"completion_tokens": int(fields[1]),
"tokens_per_second": float(fields[2]),
"cache_hit_percent": float(fields[3]),
"rss_gb": float(fields[4]),
"prompt_tokens": int(fields[5]) if len(fields) > 5 else 0,
"length_limited": bool(int(fields[6])) if len(fields) > 6 else False,
}
def model_arch(model):
"""Compatibility wrapper over the mandatory family registry."""
return resolve_model(model).descriptor.id
def cap_for_arch(arch, cap, env=None, model=None):
"""Cap-sentinel shim (#379): CURRENT-STATE CALIBRATION, not durable core.
An absent cap (None) means different things across today's engines --
platform-auto in colibri.c (coli_resolve_cap resolves the 0 sentinel
Metal/darwin/SSD-aware), RAM-auto in inkling.c (cap <= 0 fits the expert
LRU to available RAM), while the coli wrapper historically forced 8 on
every engine. This shim INTERNALIZES that external inconsistency at the
one funnel every engine launch passes through: with no explicit cap, a
glm-arch model's engine receives the 0 sentinel to resolve platform-aware
and a non-glm arch receives the legacy 8. An EXPLICIT cap passes through
verbatim to any engine -- including an explicit 0, which for inkling means
upstream's RAM-auto (people who ask for upstream semantics get them).
Keyed on the MODEL's arch (config.json model_type), not the engine
binary's file name: COLI_ENGINE users package the glm engine under
arbitrary names (glm52, colibri-1.2, ...), and basename keying silently
disabled the platform default for exactly them.
MOOTING TRIGGER: upstream unifies cap-sentinel semantics across engines
-> this shim must be removed and re-derived."""
if cap is not None:
return cap
# A measured profile records the exact argv cap used during calibration.
# The launcher passes it privately through the server process because cap
# is not an engine environment knob. It remains below an explicit --cap
# in the precedence chain and is removed before the engine starts.
if env is not None:
try:
measured = int(env.get("COLI_PROFILE_CAP", ""))
except (TypeError, ValueError):
measured = 0
if measured >= 1:
return measured
try:
planned = int(env.get("COLI_PLAN_CAP", ""))
except (TypeError, ValueError):
planned = 0
if planned >= 1:
return planned
if arch in ("deepseek_v41", "mimo") and model is not None:
# V4.1 and MiMo only read their argv cap, not RAM_GB. Without --auto-tier the
# legacy eight slots silently discarded both --ram and RAM_GB (#1666).
from resource_plan import build_plan
settings = env if env is not None else os.environ
ram = settings.get("RAM_GB", "0")
limits = family_by_id(arch).limits
plan = build_plan(model, ram_gb=0 if ram == "auto" else float(ram),
context=int(settings.get(limits.context_env, limits.default_context)),
gpu_indices=[])
slots = plan["tiers"]["ram"]["cache_slots_per_layer"]
if slots < 1:
raise ValueError(f"{family_by_id(arch).display_name} RAM budget cannot hold one "
f"expert slot per layer")
print(f"[{'v41' if arch == 'deepseek_v41' else arch}] RAM plan: {slots} expert cache "
f"slots/layer; --cap overrides", file=sys.stderr)
return slots
return family_by_id(arch).limits.implicit_cap
def decision_head_env(env, model):
"""The dense trunk's width for a checkpoint with a decision head (Clef), when
the operator set none: the head's precise width (f16) if the planner's RAM
budget holds the trunk at that size, the engine's int8 otherwise. Measured on
Clef against its reference in bf16 (docs/clef.md): int8 moves a probability
by up to 0.22, f16 by 0.012, at 2.3x the time. Returns the line it printed,
or None when it had nothing to decide."""
if env.get("COLI_DENSE_BITS"):
return None
from family_registry import decision_head_of, default_context
try:
resolved = resolve_model(model)
except Exception: # not a checkpoint this can read: nothing to decide
return None
head = decision_head_of(resolved)
if not head or not head.precise_dense_bits:
return None
try:
from resource_plan import build_plan
limits = resolved.descriptor.limits
context = int(env.get(limits.context_env) or default_context(resolved))
ram = env.get("RAM_GB", "0")
plan = build_plan(model, ram_gb=0 if ram in ("", "auto") else float(ram), context=context,
gpu_indices=[])
except Exception as error: # the engine's own default stands
line = f"[{head.id}] dense trunk left at the engine's default (no plan: {error})"
print(line, file=sys.stderr)
return line
tier = plan["tiers"]["ram"]
need = (tier["dense_bytes"] + tier["runtime_bytes"] + tier["sequence_state_bytes"]
+ tier["fixed_state_bytes"])
gib = 1 << 30
if need <= tier["budget_bytes"]:
env["COLI_DENSE_BITS"] = str(head.precise_dense_bits)
line = (f"[{head.id}] dense trunk in f16: {need / gib:.1f} GiB fit the {tier['budget_bytes'] / gib:.1f} "
f"GiB budget (COLI_DENSE_BITS=8 for int8, faster and less exact)")
else:
line = (f"[{head.id}] dense trunk in int8: f16 would need {need / gib:.1f} GiB, the budget is "
f"{tier['budget_bytes'] / gib:.1f} GiB (COLI_DENSE_BITS=16 to force it)")
print(line, file=sys.stderr)
return line
def tune_child_env(env, arch):
"""Apply the engine-local defaults that a direct server launch otherwise misses.
``coli chat`` already supplies these values, but users also launch this file
directly. Keep setdefault semantics so every explicit operator setting wins.
"""
if arch != "deepseek_v4":
return env
if not env.get("COLI_NO_OMP_TUNE"):
# The V4 runtime owns OMP_NUM_THREADS: it reserves logical CPUs for its
# expert-loader workers. Supplying a physical-core default here makes
# that runtime policy treat the launcher value as a user override.
env.setdefault("OMP_WAIT_POLICY", "active")
env.setdefault("GOMP_SPINCOUNT", "200000")
env.setdefault("OMP_DYNAMIC", "FALSE")
if sys.platform != "win32":
env.setdefault("OMP_PROC_BIND", "close")
env.setdefault("OMP_PLACES", "cores")
# All speculative paths stay opt-in: partial acceptance requires expensive
# recurrent-attention replay on this engine.
env.setdefault("V4_DRAFT", "0")
env.setdefault("V4_MTP", "0")
env.setdefault("V4_MTP_DRAFT", "3")
env.setdefault("V4_MTP_GB", "0.45")
env.setdefault("V4_MTP_MISS", "96")
env.setdefault("V4_MTP_MIN", "3")
env.setdefault("V4_MTP_CONF", "0.55")
# CUDA-driven MTP drafting stays opt-in (mirrors GLM's COLI_CUDA_MTP): the
# GPU fp4 kernels accumulate fp32 differently from the CPU refs, and a
# speculative draft must match the target bit-for-bit to be accepted.
env.setdefault("V4_MTP_GPU", "0")
return env
def _win_kill_on_close_job(pid):
"""Tie an engine process to this server's lifetime, on Windows.
The engine re-execs itself for OMP tuning, and the re-exec's parent exits
immediately -- so the surviving engine is orphaned at birth. It is not a
descendant of anything the launcher can walk to, and it is not the pid the
server recorded, so neither the pidfile nor a parent-child scan can reach
it. #1049 measured the consequence on Windows 11: 2,617 MB still resident
after a shutdown that reported success, accumulating one ghost per
serve/stop cycle. (Same failure that OOM'd a box on 2026-07-16 with two
17+5 GB ghosts, where `pkill -x glm` matched nothing because the re-exec
renames itself.)
A Job Object with KILL_ON_JOB_CLOSE fixes it at the OS level rather than by
guessing pids: job membership is INHERITED by child processes, so the
re-exec stays inside the job, and when the last handle closes -- normal
exit, TerminateProcess, or a crash of this server -- Windows terminates
everything in it. Returns the handle, which the caller must keep alive for
as long as the engine should live; returns None on any failure, which
simply restores today's behaviour.
"""
if sys.platform != "win32" or not pid:
return None
try:
import ctypes
from ctypes import wintypes
class IO_COUNTERS(ctypes.Structure):
_fields_ = [("ReadOperationCount", ctypes.c_ulonglong),
("WriteOperationCount", ctypes.c_ulonglong),
("OtherOperationCount", ctypes.c_ulonglong),
("ReadTransferCount", ctypes.c_ulonglong),
("WriteTransferCount", ctypes.c_ulonglong),
("OtherTransferCount", ctypes.c_ulonglong)]
class JOBOBJECT_BASIC_LIMIT_INFORMATION(ctypes.Structure):
_fields_ = [("PerProcessUserTimeLimit", ctypes.c_longlong),
("PerJobUserTimeLimit", ctypes.c_longlong),
("LimitFlags", wintypes.DWORD),
("MinimumWorkingSetSize", ctypes.c_size_t),
("MaximumWorkingSetSize", ctypes.c_size_t),
("ActiveProcessLimit", wintypes.DWORD),
("Affinity", ctypes.POINTER(ctypes.c_ulong)),
("PriorityClass", wintypes.DWORD),
("SchedulingClass", wintypes.DWORD)]
class JOBOBJECT_EXTENDED_LIMIT_INFORMATION(ctypes.Structure):
_fields_ = [("BasicLimitInformation", JOBOBJECT_BASIC_LIMIT_INFORMATION),
("IoInfo", IO_COUNTERS),
("ProcessMemoryLimit", ctypes.c_size_t),
("JobMemoryLimit", ctypes.c_size_t),
("PeakProcessMemoryUsed", ctypes.c_size_t),
("PeakJobMemoryUsed", ctypes.c_size_t)]
JobObjectExtendedLimitInformation = 9
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x00002000
PROCESS_SET_QUOTA, PROCESS_TERMINATE = 0x0100, 0x0001
k32 = ctypes.WinDLL("kernel32", use_last_error=True)
k32.CreateJobObjectW.restype = wintypes.HANDLE
k32.OpenProcess.restype = wintypes.HANDLE
job = k32.CreateJobObjectW(None, None)
if not job:
return None
info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION()
info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
if not k32.SetInformationJobObject(job, JobObjectExtendedLimitInformation,
ctypes.byref(info), ctypes.sizeof(info)):
k32.CloseHandle(job); return None
handle = k32.OpenProcess(PROCESS_SET_QUOTA | PROCESS_TERMINATE, False, pid)
if not handle:
k32.CloseHandle(job); return None
ok = k32.AssignProcessToJobObject(job, handle)
k32.CloseHandle(handle)
if not ok:
k32.CloseHandle(job); return None
return job
except Exception:
return None # never let process bookkeeping break starting the engine
def _write_all(stream, data, frame):
"""Write every byte of `data` to `stream`, looping on short writes.
The production engine stdin is a raw, unbuffered pipe (bufsize=0 ->
io.FileIO), whose write() is a single os.write() and may transfer fewer
bytes than it was given (a signal landing mid-write, a full pipe buffer
on a large IMAGE frame). Discarding the return value would leave the
tail of a frame unsent and desynchronize the engine's stdin framing, so
the remainder is re-offered until it is all consumed.
Neither `None` nor 0 is progress. `RawIOBase.write` answers `None` when
the stream is non-blocking and could not take a single byte, and 0 says
the same thing with a count; re-offering the buffer after either would
spin forever, so both fail closed as the named engine-write error a
broken pipe raises."""
written = 0
total = len(data)
view = memoryview(data)
while written < total:
sent = stream.write(view[written:])
# None is RawIOBase's "not one byte went out", not an uncounted
# full write, so it fails closed exactly as a zero count does.
if sent is None or sent <= 0:
raise RuntimeError(
f"failed to write {frame} to the engine "
f"(stdin took {written} of {total} bytes)")
written += sent
# ---------------------------------------------------------------- decision engines
#
# A decision engine (Laya; docs/systemone.md, "Decision engines") does not generate: it
# reads a state and typed questions and returns a probability per option. The
# gateway hands it the request as one DECIDE record and shapes the DECISION it
# gets back into the /v1/systemone reply, the same reply an LLM gives there.
MAX_DECISION_BYTES = 16 << 20
def decision_text(value):
"""A state, an instruction or a criterion as the text a decision model reads.
A string is kept exactly as sent; anything else is JSON, written the way the
reference packages write the same value (`json.dumps(value, ensure_ascii=False)`,
laya's serialize_state and render_criterion), so the model reads the bytes it
was trained on. None stays None: the engine knows the model's own default."""
if value is None or isinstance(value, str):
return value
return json.dumps(value, ensure_ascii=False)
def decision_state_type(value):
"""What the caller sent as the state, by JSON type: a model may read a list (a
conversation, newest turn last) differently from a document."""
if isinstance(value, str):
return "string"
if isinstance(value, dict):
return "object"
if isinstance(value, list):
return "array"
if isinstance(value, bool):
return "boolean"
return "number"
def decision_json_sorted(value):
"""A JSON value the way a raw-form engine's reference writes it (Clef's
render(): compact separators, keys sorted)."""
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
def _raw_decision_record(body):
"""The raw form (docs/systemone.md, "Decision engines"): the caller's own
values, for an engine that renders the request the way its reference does.
Nothing is substituted: instructions are null when none were sent, an empty
text stays empty, a noul side the caller did not describe has no "text" key
(and null when it was described as null), and JSON values are written with
sorted keys and marked "json"."""
def option(label, value, given=True):
out = {"label": label}
if not given:
return out
if value is None or isinstance(value, str):
out["text"] = value
else:
out["text"] = decision_json_sorted(value)
out["json"] = True
return out
questions = []
for qid, question in body["questions"].items():
kind = question["type"]
criteria = question.get("criteria")
instructions = question.get("instructions")
if kind == "choice":
options = [option(str(label), text) for label, text in criteria.items()]
elif kind == "score":
options = [option(str(i), text) for i, text in enumerate(criteria)]
else:
given = criteria if isinstance(criteria, dict) else {}
options = [option(side, given.get(side), side in given) for side in ("false", "true")]
questions.append({"id": qid, "type": kind,
"instructions": (instructions if instructions is None or
isinstance(instructions, str)
else decision_json_sorted(instructions)),
"options": options})
state = body["state"]
return {"record": "raw",
"state": state if isinstance(state, str) else decision_json_sorted(state),
"state_type": decision_state_type(state), "questions": questions}
def systemone_decision_record(body, form=None):
"""The DECIDE record for a /v1/systemone request that passed validation.
Each question keeps its options in the caller's order: a choice's labels with
their descriptions, a score's levels (level 0 first), a noul's false then true
with the optional criteria. Instructions left out get the same default text the
LLM path asks with. form="raw" is the form an engine asks for with
`CAPS decide_record=raw` (_raw_decision_record)."""
if form == "raw":
return _raw_decision_record(body)
if form is not None:
raise ValueError(f"unknown DECIDE record form: {form!r}")
defaults = {"noul": "Is this true?", "choice": "Which of the following applies?",
"score": "Rate this on the scale below."}
questions = []
for qid, question in body["questions"].items():
kind = question["type"]
criteria = question.get("criteria")
instructions = question.get("instructions")
if instructions is None or (isinstance(instructions, str) and not instructions.strip()):
instructions = defaults[kind]
if kind == "choice":
options = [{"label": label,
"text": None if text is None or text == "" else decision_text(text)}
for label, text in criteria.items()]
elif kind == "score":
options = [{"label": str(i), "text": decision_text(text)}
for i, text in enumerate(criteria)]
else:
given = {str(key).lower(): text for key, text in (criteria or {}).items()}
options = [{"label": side,
"text": None if given.get(side) in (None, "") else decision_text(given[side])}
for side in ("false", "true")]
questions.append({"id": qid, "type": kind, "instructions": decision_text(instructions),
"options": options})
state = body["state"]
return {"state": decision_text(state), "state_type": decision_state_type(state),
"questions": questions}
def _decision_texts(record):
yield record["state"]
for question in record["questions"]:
yield question["id"]
yield question["instructions"]
for option in question["options"]:
yield option["label"]
yield option.get("text") # a raw record's undescribed noul side has none
def decision_payload(record):
"""The DECIDE payload bytes: UTF-8 JSON. A NUL would end a C string early and a
lone surrogate has no UTF-8 form, so both are the caller's 422."""
if any(text and "\0" in text for text in _decision_texts(record)):
raise APIError(422, "NUL characters are not supported in a decision request.", "state",
"invalid_value")
try:
return json.dumps(record, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
except UnicodeEncodeError as error:
raise APIError(422, f"The request carries text that is not valid Unicode ({error.reason}).",
"state", "invalid_value") from error
def engine_decides(engine):
"""The engine announced decide=1: /v1/systemone goes to it as a DECIDE record."""
return getattr(engine, "decides", False) is True
def engine_chats(engine):
"""False only for an engine that said chat=0: it has no generating endpoint."""
return getattr(engine, "chats", True) is not False
class _Pending:
"""One in-flight engine request: the queue its frames are delivered on, whether it asked
for the per-token numeric channel, and any fault the dispatcher has recorded against it.
The dispatcher parses a frame's numeric tail only for a request that asked for it, so a
frame carrying fields a request never requested is tolerated exactly as it was before
the channel existed, and a fault in one request's frames can only fail that request.
`failed` is how that stays true without abandoning the engine's turn. The entry is kept
rather than dropped: the request's later frames are routed here and discarded, and the
error is delivered on its terminal frame, so the thread holding the scheduler admission
keeps holding it until the engine says the turn is over. Dropping the entry instead
would free the admission while the engine is still generating, and the next request
would submit into a pipe nobody is reading."""
__slots__ = ("events", "logprob_channel", "failed")
def __init__(self, logprob_channel=False):
self.events = queue.Queue()
self.logprob_channel = logprob_channel
self.failed = None
class Engine:
# cap=None = "not explicitly set": a glm-arch model's engine resolves the
# 0 sentinel (8 historically, 1 on Metal+darwin+fast SSD -- colibri.c
# coli_resolve_cap, #379), non-glm arches get the legacy 8, via
# cap_for_arch above. Same convention as the --cap flags in coli and
# main() below, so programmatic callers that never pass cap get the same
# auto behavior as the CLI; an explicit int (0 included) is verbatim.
def __init__(self, executable, model, cap=None, max_tokens=1024, env=None, kv_slots=1,
family=None):
if family is None:
# Protocol unit tests and embedders may use a synthetic model name.
# Real launcher/server entry points resolve strictly before this
# constructor; never reinterpret an existing invalid config.
config = Path(model) / "config.json"
family = (resolve_model(model).descriptor if config.exists()
else family_by_id(ARCH))
arch = family.id
self.family = family
self.model_dir = str(model)
# The extended SUBMIT namespace, gated once at launch rather than per request: an
# accepted-but-ignored opt-in is a silently wrong answer rather than an error. Two
# flags, not one -- the numeric logprobs/echo channel (`logprobs=`) and
# pre-tokenized token-id intake (`ids=`) are unrelated capabilities with separate
# coli_submit_ext keys. The numeric channel is opened to an engine that keeps the
# whole of its contract: a DATA tail on every generated token, and an ECHO frame
# for EVERY prompt position (" nan 0" at position 0) unless a pin photo covers the
# prefix -- which means never resuming a read-out from a live prefix. colibri.c
# (glm) and mimo.c do; the engines that score /v1/systemone options but resume
# read-outs from a live prefix would answer `echo` with a hole. Token-id intake
# is glm's alone: serve_codec.h, which mimo reads its frames with, has no `ids=`.
self.supports_logprobs_echo = arch in ("glm", "mimo")
self.supports_tok_ids = (arch == "glm")
child_env = dict(env or os.environ, SNAP=str(model), SERVE="1", SERVE_BATCH="1",
NGEN=str(max_tokens), KV_SLOTS=str(kv_slots))
tune_child_env(child_env, arch)
decision_head_env(child_env, model)
resolved_cap = cap_for_arch(arch, cap, child_env, model=model)
child_env.pop("COLI_PROFILE_CAP", None)
child_env.pop("COLI_PLAN_CAP", None)
# Own process group on Windows: a CTRL_BREAK sent to the serve
# process group (the graceful stop, handled as SIGBREAK above) must
# not reach the engine — the C runtime's default would kill it
# before its stdin-EOF teardown (atexit -> HEAT_FILE save) can run.
spawn_flags = subprocess.CREATE_NEW_PROCESS_GROUP if sys.platform == "win32" else 0
self.process = subprocess.Popen(
[str(executable), str(resolved_cap)], env=child_env,
stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=0,
creationflags=spawn_flags,
)
# Keep the job handle on the instance: KILL_ON_JOB_CLOSE fires when the
# LAST handle closes, so this reference is what ties the engine (and the
# OMP re-exec that orphans itself) to the server's lifetime (#1049).
# Guarded so the non-Windows path never touches .pid -- the test suite
# drives this class with a fake process object that has none.
self._win_job = None
if sys.platform == "win32":
self._win_job = _win_kill_on_close_job(getattr(self.process, "pid", None))
self.write_lock = threading.Lock()
self.pending_lock = threading.Lock()
self.pending = {}
self.next_request_id = 1
self.closed = False
self.dispatcher_error = None
self.kv_slots = kv_slots
self.tiers = None
self.hwinfo = None
self.emap = None
self.hits = None
self.hits_seq = 0 # latest "TIERS" snapshot from the engine
self.profile = collections.deque(maxlen=PROFILE_TURNS) # per-turn phase timings
self.profile_seq = 0
self.caps = {} # the engine's CAPS handshake line, key=value
# What the engine wrote before READY, the last 4 KiB of it: an engine that
# exits instead of saying READY leaves its LOAD_FAIL line there.
boot = bytearray()
def keep(data):
boot.extend(data)
del boot[:-4096]
try:
read_engine_turn(self.process.stdout, READY, keep, self.caps)
except RuntimeError:
failure = parse_load_fail(bytes(boot))
if failure is not None:
raise EngineLoadError(*failure) from None
raise
# True/False when the engine said whether it loaded a vision tower; None when
# it said nothing (an engine that predates CAPS, or a family without a tower).
self.vision = {"1": True, "0": False}.get(self.caps.get("vision"))
# A decision engine says decide=1: /v1/systemone then sends it the record
# (DECIDE) instead of scoring options through the logprob channel. chat=0
# means it has nothing else: the generating endpoints answer 400.
self.decides = self.caps.get("decide") == "1"
self.chats = self.caps.get("chat") != "0"
# decide_record=raw: the engine renders the request the way its own reference
# does (Clef), so the record carries the caller's values, not the gateway's
# defaults (systemone_decision_record). None: the default form.
self.decide_record = self.caps.get("decide_record")
self.dispatcher = threading.Thread(target=self._dispatch_stdout,
name="colibri-stdout", daemon=True)
self.dispatcher.start()
@staticmethod
def _stats(fields):
if len(fields) < 5 or fields[0] != "STAT":
raise RuntimeError(f"invalid engine status: {' '.join(fields)}")
return {
"completion_tokens": int(fields[1]),
"tokens_per_second": float(fields[2]),
"cache_hit_percent": float(fields[3]),
"rss_gb": float(fields[4]),
"prompt_tokens": int(fields[5]) if len(fields) > 5 else 0,
"length_limited": bool(int(fields[6])) if len(fields) > 6 else False,
}
def _fail_pending(self, error):
with self.pending_lock:
requests = list(self.pending.values())
self.pending.clear()
for entry in requests:
entry.events.put(("error", error))
def _write_frame(self, request_id, data, frame):
"""Checked server->engine protocol write for CANCEL/STOP: the write
and its flush happen under one write_lock acquisition. Any failure
here -- an OSError from the pipe itself, or _write_all's own
fail-closed RuntimeError on a None/zero-progress write -- drops this
request's pending-map entry: the dispatcher only does that on this
id's own DONE/ERROR frame, and neither arrives when the write that
would have solicited one never reached the engine. An OSError is
additionally re-raised as a named RuntimeError rather than left as
itself: BrokenPipeError is a ConnectionError subclass, so an
unwrapped failure here would fall into do_POST's client-hangup
handler (`except ConnectionError: pass`) and the client would see a
silent connection close instead of the 500 engine_error the failure
actually is. _write_all's own RuntimeError is already the named
error this raises for an OSError, so it is re-raised as-is."""
try:
with self.write_lock:
_write_all(self.process.stdin, data, frame)
self.process.stdin.flush()
except Exception as error:
with self.pending_lock:
self.pending.pop(request_id, None)
if isinstance(error, OSError):
raise RuntimeError(f"failed to write {frame} to the engine ({error})") from error
raise
def _read_exact(self, size):
chunks = []
remaining = size
while remaining:
chunk = self.process.stdout.read(remaining)
if chunk == b"":
raise RuntimeError("truncated engine DATA payload")
chunks.append(chunk)
remaining -= len(chunk)
return b"".join(chunks)
@staticmethod
def _parse_logprob_tail(fields, i):
"""Parse the numeric tail an opted-in DATA/ECHO frame carries:
"<lp> <k> [<tid> <tlp>]*k".
float() reads the engine's numeric wire tokens directly, "nan"/"inf"/"-inf" and any
%g precision alike -- an echo's position 0 has nothing to condition on and carries
"nan 0". The table is unsorted on the wire, so callers must not assume the first
pair is the argmax. Every malformed shape -- a short or over-long field list,
non-numeric fields, an out-of-range k, or a negative token id -- is an APIError
carrying `engine_logprob_tail_malformed`, never a silent partial record.
Fields past the ones the grammar defines are IGNORED, not refused. Tolerating a
longer frame is what the base did for every caller, and this parser now runs for
internal consumers of the channel that read only part of the record."""
if len(fields) < i + 2:
raise _malformed_logprob_tail("missing lp/k")
try:
lp = float(fields[i])
k = int(fields[i + 1])
except ValueError as error:
raise _malformed_logprob_tail(error) from error
if not 0 <= k <= LOGPROBS_TOP_K_CAP:
raise _malformed_logprob_tail(f"k={k} out of range")
if len(fields) < i + 2 + 2 * k:
raise _malformed_logprob_tail("fewer candidate fields than k declares")
topk = []
j = i + 2
for _ in range(k):
try:
tid = int(fields[j])
tlp = float(fields[j + 1])
except ValueError as error:
raise _malformed_logprob_tail(error) from error
if tid < 0:
raise _malformed_logprob_tail(f"negative token id {tid}")
topk.append((tid, tlp))
j += 2
return {"lp": lp, "topk": topk}
def _record_fault(self, entry, error):
"""Record a per-request fault without ending the request here.
The entry stays in the pending map, so the engine's later frames for it still land
somewhere (and are discarded), and `generate()` raises this error when the turn's
terminal frame arrives. The first fault wins: a stream that goes bad usually goes
bad more than once, and the first reading is the one that describes what happened.
The dispatcher itself keeps running, and every other in-flight request is
untouched."""
if entry.failed is None:
entry.failed = error
def _dispatch_stdout(self):
try:
while True:
line = self.process.stdout.readline()
if line == b"":
raise RuntimeError("colibri engine exited unexpectedly")
fields = line.decode("utf-8", "replace").strip().split()
if not fields:
continue
kind = fields[0]
if kind == "DATA" and len(fields) >= 3:
# 3 fields: the legacy frame. More: the U7a per-token
# numeric channel ("DATA <id> <n> <lp> <k> [tid tlp]*k"),
# emitted only for requests that opted in via the SUBMIT
# logprobs field. The payload framing is identical; the
# numeric fields are consumed by the server feature half
# (U7b) -- accepted here so the frame never kills the
# dispatcher (and with it every in-flight request).
request_id = fields[1]
size = int(fields[2])
if not 0 <= size <= 65536:
raise RuntimeError("invalid engine DATA size")
data = self._read_exact(size)
if self._read_exact(1) != b"\n":
raise RuntimeError("invalid engine DATA terminator")
with self.pending_lock:
entry = self.pending.get(request_id)
if entry is None or entry.failed is not None:
continue
record = None
if entry.logprob_channel and len(fields) > 3:
# Parsed only for a request that asked, with the payload already
# drained, so a malformed tail fails that one request rather than
# stopping the dispatcher.
try:
record = self._parse_logprob_tail(fields, 3)
except APIError as error:
self._record_fault(entry, error)
continue
entry.events.put(("data", data if record is None else (data, record)))
elif kind == "TOOL" and len(fields) == 3:
# Opaque, request-scoped structured output. K3 emits an
# initial zero-byte frame before generation so DATA marker
# lookalikes can never be mistaken for engine structure.
request_id = fields[1]
size = int(fields[2])
if not 0 <= size <= 65536:
raise RuntimeError("invalid engine TOOL size")
data = self._read_exact(size)
if self._read_exact(1) != b"\n":
raise RuntimeError("invalid engine TOOL terminator")
with self.pending_lock:
entry = self.pending.get(request_id)
if entry is not None and entry.failed is None:
entry.events.put(("tool", data))
elif kind == "ECHO" and len(fields) >= 6:
# U7a prefill read-out: "ECHO <id> <n> <pos> <lp> <k>
# [tid tlp]*k" plus a DATA-framed payload (n bytes + LF).
# Emitted only for opted-in requests. Questo e' U7b: il
# frame non si butta piu', va alla coda della richiesta che
# l'ha chiesto. Chi non ha chiesto logprobs non ne riceve
# nessuno, quindi il percorso della chat non cambia.
size = int(fields[2])
if not 0 <= size <= 65536:
raise RuntimeError("invalid engine DATA size")
piece = self._read_exact(size)
if self._read_exact(1) != b"\n":
raise RuntimeError("invalid engine DATA terminator")
request_id = fields[1]
with self.pending_lock:
entry = self.pending.get(request_id)
if entry is None or entry.failed is not None:
continue
try:
pos = int(fields[3])
except ValueError:
# Same frame as the numeric tail, so the same containment: this
# request's fault, never a stop for every request in flight.
self._record_fault(entry, _malformed_echo_position(repr(fields[3])))
continue
lp = fields[4]
try:
# `lp` is the numeric tail's own first field, so an unreadable one
# is a malformed tail and carries that name. It is read for every
# ECHO frame, opted in or not, because the closed-set scorer's
# `logprob` key comes from it -- which is why it needs the same
# containment as the position beside it, rather than reaching the
# dispatcher's blanket handler and stopping it for every request.
logprob = None if lp in ("nan", "-nan") else float(lp)
except ValueError:
self._record_fault(entry, _malformed_logprob_tail(
"unreadable prompt-echo log probability %s" % (lp,)))
continue
record = None
if entry.logprob_channel:
try:
record = self._parse_logprob_tail(fields, 4)
except APIError as error:
self._record_fault(entry, error)
continue
entry.events.put(("echo", {
"pos": pos,
# " nan 0" = niente su cui condizionare: la prima
# posizione assoluta non ha un predittore.
"logprob": logprob,
"text": piece.decode("utf-8", "replace"),
# The payload bytes and the numeric tail: the tail is the only
# place the candidate ids exist, and text offsets are rebuilt from
# raw bytes by one decoder spanning the whole sequence. A consumer
# reading only `pos`/`logprob`/`text` is unaffected.
"bytes": piece,
"lp": record["lp"] if record else None,
"topk": record["topk"] if record else [],
}))
elif kind == "DECISION" and len(fields) == 3:
# A decision engine's answer to DECIDE: one JSON payload, then DONE.
request_id = fields[1]
size = int(fields[2])
if not 0 <= size <= MAX_DECISION_BYTES:
raise RuntimeError("invalid engine DECISION size")
data = self._read_exact(size)
if self._read_exact(1) != b"\n":
raise RuntimeError("invalid engine DECISION terminator")
with self.pending_lock:
entry = self.pending.get(request_id)
if entry is not None and entry.failed is None:
entry.events.put(("decision", data))
elif kind == "ACCEPT" and len(fields) >= 3:
# #597: the engine validated the submission (fits context) before prefill.
# Keep it pending — DATA/DONE still follow — and let generate() commit the
# HTTP stream only now, so an earlier CONTEXT_EXCEEDED stays a clean 400.
request_id = fields[1]
with self.pending_lock:
entry = self.pending.get(request_id)
if entry is not None and entry.failed is None:
entry.events.put(("accept", {"prompt_tokens": int(fields[2])}))
elif kind == "DONE" and len(fields) >= 7:
request_id = fields[1]
stats = self._stats(fields[2:])
with self.pending_lock:
entry = self.pending.pop(request_id, None)
if entry is not None:
# The turn is over, so a fault recorded mid-turn is delivered now
# rather than when it was found, which would have freed the
# admission while the engine was still generating.
entry.events.put(("error", entry.failed) if entry.failed
else ("done", stats))
elif kind == "HWINFO" and len(fields) >= 7:
parts = " ".join(fields[6:]).split("|")
self.hwinfo = {"cores": int(fields[1]), "ram_total_gb": float(fields[2]),
"ram_avail_gb": float(fields[3]), "gpus": int(fields[4]),
"vram_total_gb": float(fields[5]),
"cpu": parts[0].strip() if len(parts)>0 else "",
"gpu": parts[1].strip() if len(parts)>1 else ""}
elif kind == "EMAP" and len(fields) == 4:
self.emap = {"rows": int(fields[1]), "cols": int(fields[2]), "map": fields[3]}
elif kind == "HITS" and len(fields) == 4:
self.hits = fields[3]
self.hits_seq += 1
elif kind == "PROF" and len(fields) >= 10:
# per-turn phase timings: where the engine spent this turn's wall time
self.profile.append({
"wall_s": float(fields[1]),
"prompt_tokens": int(fields[2]),
"completion_tokens": int(fields[3]),
"expert_disk_s": float(fields[4]),
"expert_wait_s": float(fields[5]),
"expert_matmul_s": float(fields[6]),
"attention_s": float(fields[7]),
"lm_head_s": float(fields[8]),
"forwards": int(fields[9]),
})
self.profile_seq += 1
elif kind == "TIERS" and len(fields) >= 6:
self.tiers = {"vram": int(fields[1]), "ram": int(fields[2]),
"disk": int(fields[3]), "vram_gb": float(fields[4]),
"ram_gb": float(fields[5])}
elif kind == "ERROR" and len(fields) >= 2:
request_id = fields[1]
message = " ".join(fields[2:]) or "engine request failed"
with self.pending_lock:
entry = self.pending.pop(request_id, None)
if entry is not None:
reported = _engine_error(fields[2:], message)
if entry.failed is not None and not (
isinstance(reported, RuntimeError)
and str(reported) == "CANCELLED"):
# The recorded fault happened first and names the framing
# defect, so it wins -- except against a cancellation, which
# is the client leaving and is answered here exactly as it is
# on a healthy turn.
reported = _with_engine_reason(entry.failed, message)
entry.events.put(("error", reported))
else:
raise RuntimeError(f"invalid engine response: {' '.join(fields)}")
except Exception as error:
if not self.closed:
error = self._dispatcher_cause(error)
self.dispatcher_error = error
print(f"colibri: the engine dispatcher stopped: {error}", file=sys.stderr)
self._fail_pending(error)
def _dispatcher_cause(self, error):
"""The dispatcher's error, with the engine's exit status when the engine is gone:
every later request fails with it, and the log used to say only "dispatcher
stopped" (#1941)."""
try:
gone = str(error) == "colibri engine exited unexpectedly"
status = self.process.wait(timeout=2) if gone else self.process.poll()
except Exception:
status = None
if status is None:
return error
return RuntimeError(f"{error} ({engine_exit_status(status)})")
def decide(self, record, cache_slot=0, cancelled=None):
"""One DECIDE round trip: the record out, the engine's DECISION back.
Returns (decision, stats): the parsed DECISION payload and the DONE line's
statistics. A record the engine refuses raises the APIError its ERROR names
(422 for DECIDE_INVALID); anything else the engine reports is a RuntimeError.
No CANCEL is sent: a decision is one forward pass, and the admission is held
until the engine's terminal frame either way."""
if not self.decides:
raise APIError(400, "This engine does not decide.", "model", "unsupported_endpoint")
if isinstance(cache_slot, bool) or not isinstance(cache_slot, int) or not 0 <= cache_slot < self.kv_slots:
raise APIError(400, "Invalid cache slot.", "cache_slot")
payload = decision_payload(record)
pending = _Pending(False)
with self.pending_lock:
if self.closed:
raise RuntimeError("colibri engine is shutting down")
if self.dispatcher_error is not None:
raise RuntimeError(f"colibri engine dispatcher stopped: {self.dispatcher_error}") \
from self.dispatcher_error
if self.process.poll() is not None:
raise RuntimeError("colibri engine is not running")
request_id = str(self.next_request_id)
self.next_request_id += 1
self.pending[request_id] = pending
try:
with self.write_lock:
if self.process.poll() is not None:
raise RuntimeError("colibri engine is not running")
try:
_write_all(self.process.stdin,
f"DECIDE {request_id} {cache_slot} {len(payload)}\n".encode()
+ payload + b"\n", "DECIDE")
self.process.stdin.flush()
except OSError as error:
raise RuntimeError(f"failed to write DECIDE to the engine ({error})") from error
except Exception:
with self.pending_lock:
self.pending.pop(request_id, None)
raise
decision = None
while True:
kind, value = pending.events.get()
if kind == "decision":
decision = value
elif kind == "done":
if decision is None:
raise RuntimeError("the engine finished a DECIDE without a DECISION")
try:
parsed = json.loads(decision.decode("utf-8"))
except (UnicodeDecodeError, ValueError) as error:
raise RuntimeError(f"the engine sent an unreadable DECISION ({error})") from error
return parsed, value
elif kind == "accept":
continue
else:
raise value
def generate(self, prompt, max_tokens, temperature, top_p, on_text, cache_slot=0,
cancelled=None, grammar=None, stopped=None, on_accept=None, audio=None,
on_tool=None, image=None, logprobs=0, pin=False, on_echo=None,
gbytes_before_ext=False):
if isinstance(cache_slot, bool) or not isinstance(cache_slot, int) or not 0 <= cache_slot < self.kv_slots:
raise APIError(400, "Invalid cache slot.", "cache_slot")
payload = prompt.encode("utf-8")
if b"\0" in payload:
raise APIError(400, "NUL bytes are not supported in prompts.", "messages")
gpayload = grammar.encode("utf-8") if grammar else b""
if b"\0" in gpayload:
raise APIError(400, "NUL bytes are not supported in grammars.", "response_format")
# audio (inkling only): the optional 7th SUBMIT field is grammar bytes
# for glm and DMel bytes for inkling — the two engines never see the
# other's extension, and inkling rejects grammars upstream.
apayload = audio or b""
if gpayload and apayload:
raise APIError(400, "Grammar and audio cannot be combined.", "response_format")
decoder = codecs.getincrementaldecoder("utf-8")("replace")
tool_decoder = codecs.getincrementaldecoder("utf-8")("replace")
def decode(data):
text = decoder.decode(data)
if text:
on_text(text)
def decode_tool(data):
text = tool_decoder.decode(data)
if on_tool is not None:
# Call on zero-byte frames too: that frame declares this
# request's sideband authoritative even when no call follows.
on_tool(text)
pending = _Pending(bool(logprobs))
events = pending.events
with self.pending_lock:
if self.closed:
raise RuntimeError("colibri engine is shutting down")
if self.dispatcher_error is not None:
raise RuntimeError(f"colibri engine dispatcher stopped: {self.dispatcher_error}") \
from self.dispatcher_error
if self.process.poll() is not None:
raise RuntimeError("colibri engine is not running")
request_id = str(self.next_request_id)
self.next_request_id += 1
self.pending[request_id] = pending
xpayload = gpayload or apayload
# DeepSeek V4 prefix hint (optional 8th header field): the byte length of
# the rendered prompt up to the first user/assistant turn marker — the
# stable system prefix. The engine snapshots its attention state at that
# token boundary during the prefill, so the FIRST request of the first
# conversation already seeds the shared-prefix checkpoint that every later
# conversation (opencode session) restores in seconds; without the hint
# the engine only discovers the boundary on the second fresh prompt.
# Older engines parse six or seven fields and ignore the eighth.
prefix_field = ""
if ARCH == "deepseek_v4":
cut = min((i for i in (prompt.find("<\uff5cUser\uff5c>"),
prompt.find("<\uff5cAssistant\uff5c>")) if i > 0),
default=0)
if cut > 0:
prefix_field = f" {len(xpayload)} {len(prompt[:cut].encode('utf-8'))}"
# Chiavi di estensione (decode_batch.h): logprobs=k accende la lettura
# del prefill, pin=1 fotografa lo stato a fine prompt. Una richiesta
# che non le manda produce un header identico a prima, byte per byte.
ext = ""
if logprobs:
ext += f" logprobs={int(logprobs)}"
if pin:
ext += " pin=1"
# `gbytes_before_ext` asks for the 7th numeric field ahead of the extension keys
# even with no grammar or audio payload to describe. It defaults to False, so
# every caller that does not ask keeps the wire it had.
gbytes_field = (f" {len(xpayload)}"
if (xpayload or (ext and gbytes_before_ext)) else "")
header = (f"SUBMIT {request_id} {cache_slot} {len(payload)} {max_tokens} "
f"{temperature:.8g} {top_p:.8g}"
+ (prefix_field if prefix_field else gbytes_field)
+ ext
+ "\n").encode()
try:
with self.write_lock:
if self.process.poll() is not None:
raise RuntimeError("colibri engine is not running")
# Le patch sono binarie e grosse: viaggiano in un frame loro,
# annunciato subito prima del SUBMIT a cui appartengono. Deve
# partire dentro lo stesso lock, o un'altra richiesta potrebbe
# infilarsi in mezzo e prendersi l'immagine di questa.
try:
if image is not None:
patches, grid_h, grid_w = image
blob = patches.tobytes() if hasattr(patches, "tobytes") else patches
try:
_write_all(
self.process.stdin,
f"IMAGE {request_id} {len(blob)} {grid_h} {grid_w}\n".encode()
+ blob + b"\n", "IMAGE")
except OSError as error:
raise RuntimeError(f"failed to write IMAGE to the engine ({error})") from error
_write_all(self.process.stdin, header + payload + xpayload + b"\n", "SUBMIT")
self.process.stdin.flush()
except OSError as error:
raise RuntimeError(f"failed to write SUBMIT to the engine ({error})") from error
except Exception:
with self.pending_lock:
self.pending.pop(request_id, None)
raise
cancel_sent = False
stop_sent = False
accepted = False
# DATA's numeric tail, in emission order. Stays empty when logprobs is 0, because
# the dispatcher never parses a tail for a request that did not ask for one.
generated_logprobs = []
def _accept(info):
# #597: commit exactly once, on the first of ACCEPT / DATA / DONE. A new engine sends
# ACCEPT before any output, so on_accept fires before prefill and a preceding
# CONTEXT_EXCEEDED never reaches here (it propagates as a 400 with nothing committed).
# An older engine that never sends ACCEPT still commits on its first DATA/DONE.
nonlocal accepted
if not accepted:
accepted = True
if on_accept is not None:
on_accept(info)
while True:
try:
kind, value = events.get(timeout=0.05)
except queue.Empty:
# #908: cancelled() is only polled in the "data" branch, so a
# client that disconnects before the engine's first DATA frame
# (it is still prefilling) never cancels: the CANCEL never went
# out, the turn ran to its token limit, and this thread stayed
# blocked until the engine emitted something. Poll the callback
# while idle so a pre-first-frame disconnect cancels too.
#
# Do NOT raise here: this thread holds the scheduler admission,
# and releasing it before the engine confirms the cancel lets
# the next request SUBMIT into a pipe the busy engine is not
# reading — every later request then hangs silently behind the
# orphaned generation. Wait for the engine's ERROR CANCELLED /
# DONE frame; ClientCancelled is raised when it arrives.
if not cancel_sent and not stop_sent and cancelled and cancelled():
cancel_sent = True
self._write_frame(request_id, f"CANCEL {request_id}\n".encode(), "CANCEL")
elif not cancel_sent and not stop_sent and pending.failed is not None:
# A fault was recorded for this request, so nothing it generates from
# here can reach the client. The turn is still the engine's -- the
# admission is held to its terminal frame either way -- but it should
# not run the rest of the budget for a response nobody will get.
#
# The STOP goes out through the checked frame writer (#1721), which
# drops this request's pending-map entry and re-raises an OSError as
# the named engine_error -- exactly the handling this branch carried
# inline before that writer existed. Keeping the entry would be worse
# than losing it: a STOP that never reached the engine is never
# answered with a terminal frame, so the admission would be held for
# the process's lifetime.
stop_sent = True
self._write_frame(request_id, f"STOP {request_id}\n".encode(), "STOP")
continue
if kind == "accept":
if accepted:
raise RuntimeError("engine sent a duplicate ACCEPT frame")
_accept(value)
elif kind == "data":
_accept({"prompt_tokens": None})
# The dispatcher wraps the payload in a tuple only when a logprob record
# rides along; bare bytes is the non-opted-in shape.
data, record = value if isinstance(value, tuple) else (value, None)
if record is not None:
generated_logprobs.append((data, record))
if not cancel_sent and not stop_sent:
decode(data)
if stopped and stopped():
stop_sent = True
self._write_frame(request_id, f"STOP {request_id}\n".encode(), "STOP")
elif cancelled and cancelled():
# Same admission-holding rule as the idle branch above:
# send CANCEL, then keep consuming frames until the
# engine acknowledges with ERROR CANCELLED or DONE.
cancel_sent = True
self._write_frame(request_id, f"CANCEL {request_id}\n".encode(), "CANCEL")
elif kind == "echo":
# Lettura del prefill: arriva PRIMA di ogni DATA e non e' testo
# generato, quindi non passa da decode() e non entra nella
# risposta. La consuma chi ha chiesto il canale.
_accept({"prompt_tokens": None})
if on_echo is not None:
on_echo(value)
elif kind == "tool":
_accept({"prompt_tokens": None})
if not cancel_sent and not stop_sent:
decode_tool(value)
if stopped and stopped():
stop_sent = True
self._write_frame(request_id, f"STOP {request_id}\n".encode(), "STOP")
elif cancelled and cancelled():
cancel_sent = True
self._write_frame(request_id, f"CANCEL {request_id}\n".encode(), "CANCEL")
elif kind == "done":
_accept({"prompt_tokens": None})
if cancel_sent:
# The engine finished the turn before seeing the CANCEL
# (or honored it at a token boundary and still framed a
# DONE). Either way the client is gone: the ack is what
# mattered, the output is not deliverable.
raise ClientCancelled()
tail = decoder.decode(b"", final=True)
if tail:
on_text(tail)
tool_tail = tool_decoder.decode(b"", final=True)
if tool_tail and on_tool is not None:
on_tool(tool_tail)
if logprobs:
value["logprobs"] = {"generated": generated_logprobs}
return value
elif cancel_sent and (value is pending.failed
or (isinstance(value, RuntimeError)
and str(value) == "CANCELLED")):
# The client left, on the two terminal frames that report nothing else: an
# ERROR CANCELLED, which is the engine acknowledging the cancel, and a DONE,
# which hands back the fault recorded while nobody was reading. Raising the
# fault instead books a departed client as a server failure in the
# scheduler's outcome, and tries to write a 500 to a socket that is already
# closed. An ERROR carrying any other text is the engine's own account of
# why the turn ended; that is news, and it is still reported as a failure.
raise ClientCancelled()
else:
raise value
def close(self):
with self.pending_lock:
if self.closed:
return
self.closed = True
self._fail_pending(RuntimeError("colibri engine is shutting down"))
if self.process.poll() is None:
# Graceful drain first: the engine's serve loop reads requests
# from stdin, and EOF there is the one portable path to its
# atexit teardown (qt_shutdown -> HEAT_FILE save). EOF only
# lands between turns, so the drain wait must be generous.
# poll() (not the absence of TimeoutExpired) decides whether
# the hard-stop ladder below still needs to run: wait() may
# simply return None for a process (or test double) that only
# "terminates" when asked.
try:
self.process.stdin.close()
except (OSError, ValueError, AttributeError):
pass
try:
self.process.wait(timeout=_ENGINE_DRAIN_S)
except subprocess.TimeoutExpired:
pass
if self.process.poll() is None:
self.process.terminate()
try:
self.process.wait(timeout=5)
except subprocess.TimeoutExpired:
# A large resident cache (e.g. 111 GB at --memory-gb 126) can
# take longer than the grace period to unmap and free on
# SIGTERM. SIGKILL cannot be caught, so the process is already
# on its way out; a second timeout only means the reap has not
# landed yet. Teardown is best-effort: never raise from here, or
# a completed measurement is lost to a shutdown that succeeded.
self.process.kill()
try:
self.process.wait(timeout=5)
except subprocess.TimeoutExpired:
pass
if self.dispatcher is not threading.current_thread():
self.dispatcher.join(timeout=5)
def model_object(model_id, created):
return {"id": model_id, "object": "model", "created": created, "owned_by": "colibri"}
def is_image_engine(engine):
"""True when the server fronts a text-to-image engine rather than a chat one.
Decided by the engine object, not by the ARCH global, so a server built in a
test (or embedded) with an ImageEngine behaves as one."""
return getattr(engine, "modality", "text") == "image"
# The endpoints that only make sense for a chat model, refused with a pointer
# when the model draws images instead.
TEXT_ENDPOINTS = ("/v1/chat/completions", "/v1/completions", "/v1/messages", "/v1/systemone")
# The endpoints that generate or score text with a language model, refused with a
# pointer to /v1/systemone when the model is a decision engine (chat=0).
GENERATING_ENDPOINTS = ("/v1/chat/completions", "/v1/completions", "/v1/messages")
IMAGE_OPTION_KEYS = ("default_width", "default_height", "default_steps", "min_side",
"max_side", "multiple")
def cors_origin_list(given):
"""The allowed browser origins from --cors-origin values. None keeps the
defaults. A value written `+origin` adds to the list instead of replacing
it: only `+` values extend the defaults, and plain values replace them as
they always have, with any `+` values added after."""
if given is None:
return DEFAULT_CORS_ORIGINS
plain = [origin for origin in given if not origin.startswith("+")]
added = [origin[1:] for origin in given if origin.startswith("+") and origin[1:]]
base = list(DEFAULT_CORS_ORIGINS) if not plain else plain
return tuple(dict.fromkeys(base + added))
def _positive_env(name, default):
try:
value = int(os.environ.get(name, "") or default)
except ValueError:
return default
return value if value > 0 else default
class APIServer(ThreadingHTTPServer):
daemon_threads = True
# SEC: ThreadingHTTPServer spawns one thread per TCP connection with no
# ceiling, and each carries a default 8 MiB stack. Opening connections and
# never completing a request therefore grows thread count -- and memory --
# without bound, before any Host check or auth runs. max_queue bounds the
# inference queue, not the accept loop.
#
# 64 is deliberately small: the engine serves one request at a time
# (kv_slots) behind a queue of 8, so hundreds of concurrent connections buy
# nothing a dashboard plus a handful of clients does not already have. Over
# the cap we close immediately rather than queue, so the cost of a flood is
# paid by the attacker's socket and not by our address space.
MAX_CONNECTIONS = _positive_env("COLI_MAX_CONNECTIONS", 64)
# A global cap alone turns memory exhaustion into connection starvation: one
# attacker holding all 64 slots still locks every real client out. Measured
# exactly that while testing the cap. So also bound what a single source may
# hold, and keep it well under the global cap: a browser opens a handful of
# parallel connections, an SDK fewer, so 8 is generous for any one client and
# leaves 56 slots that one address cannot touch.
MAX_CONNECTIONS_PER_IP = _positive_env("COLI_MAX_CONNECTIONS_PER_IP", 8)
def __init__(self, address, engine, model_id, api_key=None, max_tokens=1024,
cors_origins=DEFAULT_CORS_ORIGINS, max_queue=8, queue_timeout=300,
kv_slots=1, allowed_hosts=()):
super().__init__(address, APIHandler)
self.engine = engine
self.model_id = model_id
self.api_key = api_key
self.max_tokens = max_tokens
self.scheduler = GenerationScheduler(max_queue, queue_timeout, kv_slots)
self.kv_slots = kv_slots
self.cors_origins = tuple(cors_origins)
# Extra Host header values trusted past the DNS-rebinding guard, for a
# reverse proxy / MagicDNS in front of the loopback bind (#597). Explicit
# opt-in only: no wildcard, default stays loopback + bind address.
self.allowed_hosts = tuple(
h.strip().lower() for h in allowed_hosts if h and h.strip())
self.created = int(time.time())
# Hashes of the states /v1/systemone scored lately: a state seen again is
# worth a photo, one seen once is probably a feed (systemone's pin rule).
self._states_seen = collections.OrderedDict()
self._states_lock = threading.Lock()
# slot -> (fixed prefix, its tokens) photographed by /v1/systemone
self.pinned_prefixes = {}
self._conn_lock = threading.Lock()
self._conn_live = 0
self._conn_by_ip = {}
self._conn_owner = {}
def model_entry(self):
"""The /v1/models object. An image model says so, and carries the size
rules its engine announced, so a client can build a valid request
without trial and error."""
entry = model_object(self.model_id, self.created)
entry["input_modalities"] = self.input_modalities()
entry["capabilities"] = self.capabilities()
if is_image_engine(self.engine):
info = getattr(self.engine, "info", None) or {}
entry["image"] = {key: info.get(key) for key in IMAGE_OPTION_KEYS}
return entry
def state_seen(self, state, capacity=256):
"""True if `state` was scored before (and remember it either way)."""
key = hashlib.sha1(state.encode("utf-8", "replace")).digest()
with self._states_lock:
seen = key in self._states_seen
self._states_seen[key] = True
self._states_seen.move_to_end(key)
while len(self._states_seen) > capacity:
self._states_seen.popitem(last=False)
return seen
def capabilities(self):
"""What the served model does, for /v1/models and /health: an image model
draws (image_generation); every text engine answers POST /v1/systemone
(systemone), and a decision engine does only that (decision)."""
if is_image_engine(self.engine):
return ["image_generation"]
if engine_decides(self.engine) and not engine_chats(self.engine):
return ["systemone", "decision"]
return ["chat", "systemone"]
def jev_model_card(self):
"""The served model as Jev's GET /v1/models lists it. colibri does not know a
model's release date; it gives the day this server started."""
try:
name = getattr(self, "display_name", None) or family_by_id(ARCH).display_name
except Exception:
name = self.model_id
return {"name": self.model_id,
"description": f"{name}, served by colibri; POST /v1/systemone answers typed questions.",
"release_date": time.strftime("%Y-%m-%d", time.gmtime(self.created))}
def input_modalities(self):
"""What a request to this server may carry: text, plus image when BOTH the
family has a placeholder expansion and the engine said it loaded its tower.
The family alone is not the truth (a glm53 export can declare vision_config
and ship no model.visual.* tensors, and the engine then serves text), and
the engine alone is not either (a tower the gateway cannot feed is no
modality). An engine that announced nothing is text: the card under-claims
rather than promising a picture nobody checked."""
modalities = ["text"]
if family_by_id(ARCH).capabilities.image and getattr(self.engine, "vision", None) is True:
modalities.append("image")
return modalities
def generate(self, prompt, max_tokens, temperature, top_p, on_text, *args, **kwargs):
started = time.monotonic()
first_output = False
def measured(callback):
def feed(text):
nonlocal first_output
if text and not first_output:
first_output = True
self.scheduler.observe_timing("first_output_seconds", time.monotonic() - started)
return callback(text)
return feed
if kwargs.get("on_tool") is not None:
kwargs["on_tool"] = measured(kwargs["on_tool"])
try:
return self.engine.generate(prompt, max_tokens, temperature, top_p,
measured(on_text), *args, **kwargs)
finally:
self.scheduler.observe_timing("engine_call_seconds", time.monotonic() - started)
def process_request(self, request, client_address):
"""Refuse past the caps instead of spawning an unbounded thread."""
peer = client_address[0] if client_address else "?"
with self._conn_lock:
mine = self._conn_by_ip.get(peer, 0)
if self._conn_live >= self.MAX_CONNECTIONS:
reason = "server cap %d" % self.MAX_CONNECTIONS
elif mine >= self.MAX_CONNECTIONS_PER_IP:
reason = "per-address cap %d" % self.MAX_CONNECTIONS_PER_IP
else:
reason = None
self._conn_live += 1
self._conn_by_ip[peer] = mine + 1
self._conn_owner[id(request)] = peer
if reason:
sys.stderr.write("[api] %s - refused: %s\n" % (peer, reason))
self.shutdown_request(request)
return
try:
super().process_request(request, client_address)
except BaseException:
self._release(request)
raise
def _release(self, request):
with self._conn_lock:
peer = self._conn_owner.pop(id(request), None)
if peer is None:
return # never counted, or already released
if self._conn_live > 0:
self._conn_live -= 1
left = self._conn_by_ip.get(peer, 1) - 1
if left > 0:
self._conn_by_ip[peer] = left
else:
self._conn_by_ip.pop(peer, None) # do not grow a map per peer
def close_request(self, request):
self._release(request)
super().close_request(request)
def _content_length(value):
# int() also accepts signs and underscores, but HTTP lengths are 1*DIGIT.
# Read and early-error drain must agree on that exact framing grammar.
value = value.strip()
if re.fullmatch(r"[0-9]+", value) is None:
raise ValueError("invalid Content-Length")
return int(value)
class _DeadlineReader:
"""rfile wrapper enforcing a CUMULATIVE deadline on reading one request.
SEC: `timeout` below is per socket operation, so it restarts on every byte.
A client dripping one byte every 29 s renews it forever and holds a thread
and a connection slot indefinitely -- the code's own comment claimed the
opposite. The deadline here is absolute: every read shrinks the socket
timeout to the time left, so a drip runs the clock down instead of resetting
it.
It covers the request-read phase only. Generation is not on this clock: a
600-second answer is normal and must not be cut off, so send_response()
hands the socket back to the ordinary timeout once the status line is out.
"""
def __init__(self, raw, sock, per_read, budget):
self._raw, self._sock, self._per_read = raw, sock, per_read
self._expires = time.monotonic() + budget
def _arm(self):
left = self._expires - time.monotonic()
if left <= 0:
raise TimeoutError("request read deadline exceeded")
self._sock.settimeout(min(self._per_read, left))
def readline(self, size=-1):
chunks = []
remaining = size
while remaining != 0:
self._arm()
# peek() does at most one raw socket read. Consume only bytes it
# already buffered, so readline cannot renew one socket timeout
# internally while a peer drips an unfinished header.
buffered = self._raw.peek(1)
if not buffered:
break
newline = buffered.find(b"\n")
take = newline + 1 if newline >= 0 else len(buffered)
if remaining > 0:
take = min(take, remaining)
chunk = self._raw.read1(take)
chunks.append(chunk)
if remaining > 0:
remaining -= len(chunk)
if chunk.endswith(b"\n"):
break
return b"".join(chunks)
def read(self, size=-1):
chunks = []
remaining = size
while remaining != 0:
self._arm()
# BufferedReader.read(size) can perform many recv calls, each
# renewing the same socket timeout. read1() does at most one;
# arm the shrinking absolute budget again before the next.
chunk = self._raw.read1(min(remaining, 65536) if remaining > 0 else 65536)
if not chunk:
break
chunks.append(chunk)
if remaining > 0:
remaining -= len(chunk)
return b"".join(chunks)
def __getattr__(self, name):
return getattr(self._raw, name)
class APIHandler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
# TCP_NODELAY on every connection, set in setup(). A response is the header
# block and the body in two writes; with Nagle on, the body waits for the
# client's delayed ACK of the headers on a kept-alive connection, about 40 ms
# per request (measured: 44 ms median for a /v1/systemone round trip on
# loopback, 1 ms without). That is most of a decision's time on a fast engine.
# Not through disable_nagle_algorithm: macOS refuses the option with EINVAL on
# a socket the client has already reset, and a hangup is not a server error.
disable_nagle_algorithm = False
timeout = 30 # per socket OPERATION. On its own this does not stop a slowloris:
# it restarts on every byte received, so a drip renews it forever.
# READ_DEADLINE below is the cumulative bound that actually does.
READ_DEADLINE = _positive_env("COLI_READ_DEADLINE", 30) # accept -> request read
server_version = "colibri"
_committed = False # status line already on the wire; reset per request below
_body_read = False # request body fully consumed, so nothing is left to drain
def setup(self):
try:
self.request.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, True)
except OSError:
pass # the client already hung up: the request fails as a hangup, below
super().setup()
# Keep the socket-backed reader; handle_one_request re-wraps it with a
# fresh deadline per request rather than wrapping a wrapper each time.
self._raw_rfile = self.rfile
def log_message(self, fmt, *args):
sys.stderr.write("[api] %s - %s\n" % (self.address_string(), fmt % args))
def handle_one_request(self):
"""Per-request bookkeeping for HTTP/1.1 persistence (#597 item 3).
One handler instance serves every request on a keep-alive connection, so both flags
reset here rather than in do_POST. The drain afterwards is the whole fix for the
reported `Bad request syntax ('{...json...}POST /v1/...')`: any early rejection --
403 Host, 401 auth, a bad or oversized Content-Length -- returns before read_json(),
leaving the body in the socket, where the next readline() eats it as a request line.
Draining once at the request boundary covers every such path, present and future,
instead of asking each early return to remember."""
self._committed = False
self._body_read = False
# Fresh budget per request: a keep-alive connection may serve many, and
# each is entitled to its own read window -- but none may drip forever.
self.rfile = _DeadlineReader(self._raw_rfile, self.connection,
self.timeout, self.READ_DEADLINE)
try:
super().handle_one_request()
if not self.close_connection:
self._drain_request_body()
except TimeoutError:
# The read budget ran out. Say so and close; do not answer, because
# we never received a complete request to answer.
sys.stderr.write("[api] %s - request read deadline exceeded\n"
% self.address_string())
self.close_connection = True
return
except ConnectionError:
# ConnectionError, not (BrokenPipeError, ConnectionResetError): those two
# are SIBLINGS of ConnectionAbortedError under it, so the pair caught the
# POSIX spellings and let the Windows one through. #854's log is pages of
# `ConnectionAbortedError: [WinError 10053] An established connection was
# aborted by the software in your host machine` escaping to socketserver,
# from a `coli web` start that was otherwise healthy.
#
# The client hung up mid-response. That is not an error here, it is
# how HTTP clients behave: `coli chat` polls /health while the model
# loads and drops each connection as soon as it has its answer, and
# Ctrl-C during a stream closes the socket by design -- the banner
# tells the user to do exactly that. Without this, socketserver's
# handler prints a full traceback per occurrence, so a normal start
# buried the loading spinner under BrokenPipeError stack traces and
# every cancelled answer looked like a crash.
#
# Caught here rather than in send_json() so it also covers the SSE
# writes in the streaming path, which is where Ctrl-C lands.
self.close_connection = True
return
def send_response(self, code, message=None):
"""Single choke point for "the status line is out". Overriding here rather than
tracking it at each call site means no responder can forget (#597 item 3)."""
self._committed = True
# The request is fully read by the time anything answers, so the read
# deadline has done its job. Restore the plain per-operation timeout:
# generation legitimately takes minutes and must not inherit a clock
# sized for reading a request header.
try:
self.connection.settimeout(self.timeout)
except OSError:
pass
super().send_response(code, message)
def _drain_request_body(self):
"""Consume any unread request body so the next request line is at the head of the
stream. Where the body can't be swallowed safely, close instead: an unreusable
connection is correct, a desynchronised one is not."""
if self._body_read:
return
self._body_read = True
if (self.headers.get_all("Transfer-Encoding")
or len(self.headers.get_all("Content-Length", [])) > 1):
self.close_connection = True # not framed by Content-Length; we don't de-chunk
return
try:
remaining = _content_length(self.headers.get("Content-Length", "0"))
except ValueError:
self.close_connection = True # unparseable framing: the body length is unknown
return
if remaining < 0 or remaining > MAX_BODY:
self.close_connection = True # don't burn bandwidth just to keep a socket warm
return
while remaining > 0:
chunk = self.rfile.read(min(remaining, 65536))
if not chunk:
self.close_connection = True
return
remaining -= len(chunk)
def send_json(self, status, body, request_id=None, headers=None):
data = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode()
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(data)))
if request_id:
self.send_header("x-request-id", request_id)
# what the TypeSafe SDKs read as `request_id` (an error carries it too)
self.send_header("x-typesafe-request-id", request_id)
for name, value in (headers or {}).items():
self.send_header(name, value)
self.send_cors_headers()
self.end_headers()
self.wfile.write(data)
def send_cors_headers(self):
origin = self.headers.get("Origin")
if not origin or ("*" not in self.server.cors_origins and origin not in self.server.cors_origins):
return
self.send_header("Access-Control-Allow-Origin", "*" if "*" in self.server.cors_origins else origin)
self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
self.send_header("Access-Control-Allow-Headers", "Authorization, Content-Type, x-api-key, anthropic-version, "
"X-TypeSafe-SDK, X-TypeSafe-Runtime, X-TypeSafe-Retry-Count")
self.send_header("Access-Control-Expose-Headers",
"x-request-id, x-typesafe-request-id, x-colibri-queue-wait-ms, "
"x-colibri-elapsed-ms, x-colibri-engine-ms, Retry-After")
self.send_header("Access-Control-Max-Age", "600")
if "*" not in self.server.cors_origins:
self.send_header("Vary", "Origin")
LOOPBACK_HOSTS = {"127.0.0.1", "localhost", "::1", ""}
def _is_authed(self):
"""True if no key is configured, or a correct key was presented. Anthropic clients
(Claude Code, the Anthropic SDKs) authenticate with `x-api-key`, not `Bearer` — both
are accepted, and both are compared in constant time."""
if not self.server.api_key:
return True
import hmac
if hmac.compare_digest(self.headers.get("Authorization", ""),
f"Bearer {self.server.api_key}"):
return True
return hmac.compare_digest(self.headers.get("x-api-key", ""), self.server.api_key)
def require_auth(self):
if not self._is_authed():
raise APIError(401, "Invalid or missing API key.", None, "invalid_api_key",
"authentication_error")
def _check_host(self):
"""DNS-rebinding guard: a web page can resolve a hostname to 127.0.0.1 and
drive this local server unless we pin the Host header to loopback / the bind
address. Rejects requests whose Host is anything else. (#SEC-7)"""
host = self.headers.get("Host", "")
if host.startswith("["):
name = host[1:].split("]", 1)[0] # [ipv6]:port
elif host.count(":") == 1:
name = host.rsplit(":", 1)[0] # host:port / ipv4:port
else:
name = host # bare host / bracketless ipv6
name = name.strip().lower()
allowed = set(self.LOOPBACK_HOSTS)
allowed.update(self.server.allowed_hosts) # #597: operator-trusted reverse-proxy names
# A wildcard is an explicit operator opt-out of the guard, for the case
# the guard cannot serve: a container/LAN bind reached by an IP or DNS
# name the server cannot predict (#990 -- Docker port-map, the browser
# sends the host's address, which the container never knows). The guard
# protects a LOOPBACK bind from a malicious page; once bound to 0.0.0.0
# the exposure is already chosen, so `*` adds no risk that bind did not.
if "*" in allowed:
return
try:
allowed.add(str(self.server.server_address[0]).strip("[]").lower())
except Exception:
pass
if name not in allowed:
raise APIError(
403,
"Host header %r not allowed. Add it with --allowed-host %s "
"(or COLI_ALLOWED_HOSTS), or --allowed-host '*' to accept any "
"host when the bind is already public." % (name or "(empty)", name or "<host>"),
None, "forbidden")
def read_json(self):
# No transfer decoder lives here: accepting TE+CL would choose CL even
# though HTTP gives TE precedence. Multiple CL fields are likewise not
# a boundary we should guess. Refuse before engine work and never reuse
# the connection after an ambiguous frame (RFC 9112 section 6.3).
if self.headers.get_all("Transfer-Encoding"):
self.close_connection = True
raise APIError(400, "Transfer-Encoding is not supported; send one Content-Length.")
if len(self.headers.get_all("Content-Length", [])) > 1:
self.close_connection = True
raise APIError(400, "Multiple Content-Length headers are not supported.")
try:
length = _content_length(self.headers.get("Content-Length", "0"))
except ValueError:
self.close_connection = True
raise APIError(400, "Invalid Content-Length header.")
if length < 1 or length > MAX_BODY:
raise APIError(400, f"Request body must be between 1 and {MAX_BODY} bytes.")
raw = self.rfile.read(length)
# Only a full read leaves nothing to drain; a short read means the peer went away
# mid-body, and the drain will notice the EOF and close (#597 item 3).
self._body_read = len(raw) == length
try:
body = json.loads(raw)
except (json.JSONDecodeError, UnicodeDecodeError):
raise APIError(400, "Request body must be valid JSON.")
try:
# An escaped lone surrogate ("\ud83d") parses, but it is the same invalid
# text as the undecodable bytes above: no UTF-8 can carry it, and the
# engine protocol encodes every prompt as UTF-8.
json.dumps(body, ensure_ascii=False).encode("utf-8")
except UnicodeEncodeError:
raise APIError(400, "Request body contains an unpaired UTF-16 surrogate "
"escape; strings must be valid Unicode.")
if not isinstance(body, dict):
raise APIError(400, "Request body must be a JSON object.")
return body
def check_model(self, body):
model = body.get("model")
if model != self.server.model_id:
raise APIError(404, f"The model `{model}` does not exist.", "model", "model_not_found")
# The dashboard ships in two layouts and the old single path only knew one:
# a source checkout puts this file in c/ (so web/dist is one level UP), while
# a release archive and an installed tree put it next to web/dist. Probing for
# index.html rather than the directory keeps an empty leftover web/dist from
# shadowing a real one.
WEB_DIST = next(
(c for c in (Path(__file__).resolve().parent / "web" / "dist",
Path(__file__).resolve().parent.parent / "web" / "dist")
if (c / "index.html").is_file()),
Path(__file__).resolve().parent.parent / "web" / "dist")
def serve_static(self, path):
"""Serve the built web UI (web/dist) so `coli web` is one process.
Read-only, no auth (same trust level as /health), traversal-safe."""
if path.startswith("/v1/") or path == "/health":
return False
base = self.WEB_DIST.resolve()
if not base.is_dir():
return False
rel = unquote(path).lstrip("/") or "index.html"
target = (base / rel).resolve()
try:
target.relative_to(base)
except ValueError:
target = None
if target is None or not target.is_file():
if path == "/" or "." not in rel: # SPA fallback
target = base / "index.html"
if not target.is_file():
return False
else:
return False
ctype = mimetypes.guess_type(str(target))[0] or "application/octet-stream"
data = target.read_bytes()
self.send_response(200)
self.send_header("Content-Type", ctype)
self.send_header("Content-Length", str(len(data)))
self.send_cors_headers()
self.end_headers()
self.wfile.write(data)
return True
def do_GET(self):
request_id = "req_" + uuid.uuid4().hex
try:
self._check_host()
path = urlsplit(self.path).path
if path == "/metrics":
self.require_auth()
data = self.server.scheduler.prometheus().encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
self.send_header("Content-Length", str(len(data)))
self.send_header("Cache-Control", "no-store")
self.end_headers()
self.wfile.write(data)
return
if path == "/health":
# Liveness is always public; hardware/scheduler internals only when a
# request is authed (or no key set), so a configured key isn't leaked
# past a bare 200 to an unauthenticated probe. (#SEC-8)
payload = {"status": "ok"}
if self._is_authed():
payload["arch"] = ARCH # which family answers: coli chat reads it
payload["scheduler"] = self.server.scheduler.snapshot()
payload["kv_slots"] = self.server.kv_slots
payload["input_modalities"] = self.server.input_modalities()
payload["continue_assistant"] = os.environ.get("COLI_CONTINUE_ASSISTANT", "1") != "0" and ARCH in CONTINUATION_FAMILIES
tiers = getattr(self.server.engine, "tiers", None) if self.server.engine else None
if tiers: payload["tiers"] = tiers
hwinfo = getattr(self.server.engine, "hwinfo", None) if self.server.engine else None
if hwinfo: payload["hwinfo"] = hwinfo
payload["capabilities"] = self.server.capabilities()
if is_image_engine(self.server.engine):
payload["image"] = dict(getattr(self.server.engine, "info", None) or {})
self.send_json(200, payload, request_id)
return
if path == "/experts":
payload = {"rows": 0, "cols": 0, "map": "", "hits": "", "seq": 0}
eng = self.server.engine
if self._is_authed() and eng and getattr(eng, "emap", None): # (#SEC-8) hide routing telemetry unless authed
payload.update(eng.emap)
payload["hits"] = eng.hits or ""
payload["seq"] = eng.hits_seq
self.send_json(200, payload, request_id)
return
if path == "/profile":
# (#SEC-8) same gate as /health and /experts above: this endpoint
# is served before require_auth(), so an unauthenticated caller
# reached it even with --api-key set. It carries per-turn
# telemetry -- prompt and completion token counts, per-phase
# timings, up to 120 turns -- which describes what the operator
# is running and how much. The pass that added _is_authed() to
# the two endpoints above did not reach this one.
eng = self.server.engine
payload = {"seq": 0, "turns": []}
if self._is_authed() and eng:
payload["seq"] = getattr(eng, "profile_seq", 0)
payload["turns"] = list(getattr(eng, "profile", ()) or ())
self.send_json(200, payload, request_id)
return
if self.serve_static(path):
return
self.require_auth()
if path == "/v1/models":
# `data` is OpenAI's list; `models` is Jev's (name, description,
# release_date), which the TypeSafe SDKs' models.list() reads.
self.send_json(200, {"object": "list", "data": [self.server.model_entry()],
"models": [self.server.jev_model_card()]},
request_id)
elif path.startswith("/v1/models/") and unquote(path[11:]) == self.server.model_id:
self.send_json(200, self.server.model_entry(), request_id)
else:
raise APIError(404, "Not found.", None, "not_found")
except APIError as error:
self.send_json(error.status, error_object(error), request_id, error.headers)
def do_OPTIONS(self):
try: # (#SEC-7) apply the Host guard uniformly, incl. CORS preflight
self._check_host()
except APIError:
self.send_response(403)
self.send_header("Content-Length", "0")
self.end_headers()
return
self.send_response(204)
self.send_header("Content-Length", "0")
self.send_cors_headers()
self.end_headers()
def do_POST(self):
request_id = "req_" + uuid.uuid4().hex
try:
self._check_host()
self.require_auth()
try:
body = self.read_json()
except TimeoutError:
# Only request intake is on this deadline. A later timeout
# from the engine keeps the existing structured 500 response.
self.close_connection = True
self.log_error("request read deadline exceeded")
return
path = urlsplit(self.path).path
# A client written for Jev sends "jev-latest": on that route the
# served model answers whatever name was asked for.
if path != "/v1/systemone":
self.check_model(body)
if is_image_engine(self.server.engine) and path in TEXT_ENDPOINTS:
raise APIError(400, f"`{self.server.model_id}` is an image generation model and "
"does not chat: POST /v1/images/generations instead.",
"model", "unsupported_endpoint")
if not engine_chats(self.server.engine) and path in GENERATING_ENDPOINTS:
raise APIError(400, f"`{self.server.model_id}` is a decision model: it answers typed "
"questions with calibrated probabilities and does not generate "
"text. POST /v1/systemone instead.",
"model", "unsupported_endpoint")
if path == "/v1/images/generations":
self.image_generation(body, request_id)
elif path == "/v1/chat/completions":
self.chat_completion(body, request_id)
elif path == "/v1/completions":
self.completion(body, request_id)
elif path == "/v1/systemone":
self.systemone(body, request_id)
elif path == "/v1/messages":
self.anthropic_messages(body, request_id)
else:
raise APIError(404, "Not found.", None, "not_found")
except APIError as error:
self._fail(error, request_id)
except ClientCancelled:
pass
except ConnectionError:
pass # same widening as handle_one_request, same reason
except Exception as error:
self.log_error("request failed: %s", error)
try:
self._fail(APIError(500, "The colibri engine failed to process the request.",
None, "engine_error", "server_error"), request_id)
except OSError:
pass
# ---------------------------------------------------------------- il canale di scoring
#
# Il percorso di POST /v1/systemone su un modello linguistico. Il modello non
# genera: si legge il logprob di ogni opzione ammessa e si normalizza sulle
# sole opzioni. Torna una distribuzione, non una stringa. (Un motore di
# decisione, decide=1, risponde da se' con DECIDE: _systemone_decide.)
#
# PERCHE' IL CICLO STA QUI E NON NEL CLIENT. Servono tre cose facili da
# sbagliare: fotografare il prefisso condiviso (pin) cosi ogni opzione paga
# solo i propri token; leggere l'opzione come continuazione della risposta e
# non come coda di un elenco; e normalizzare, perche' somma e media dei
# logprob danno risposte diverse quando le opzioni hanno lunghezze diverse.
# Un client che rifacesse questo ciclo sbaglierebbe una di queste tre, e il
# risultato resterebbe plausibile.
#
# Lo stato viene fotografato una volta e ogni domanda paga solo se stessa:
# e' la forma in cui il canale rende (5,7x misurato contro la chat).
# (Fino alla 1.12.1 questo ciclo aveva anche un endpoint suo, POST /v1/brio,
# con le forme options/questions/schema. /v1/systemone e' ora l'unica API.)
def _score_questions(self, state, questions, normalize="sum", pin_state=True,
fixed=None, cache_slot=None):
"""Score every question's options against `state` through the engine's
logprob channel. `questions` is a list of (question text, options).
Returns (answers, usage, headers): per question the options sorted by
probability, each with `p`, `logprob`, `mean_logprob` and `tokens`, and
the normalised entropy; `prompt_tokens` (the longest prompt) and
`read_tokens` (the option tokens read); the timing headers.
`fixed` is a text read before the state and photographed once per slot
(/v1/systemone's `prefix`); `pin_state` False skips the photo of the
state. The slot defaults to the one derived from the part that does not
change: the fixed text when there is one, else the state."""
# Lo slot si sceglie dallo STATO, non dalla domanda: mille domande
# diverse sullo stesso contesto devono cadere sullo stesso slot, o la
# fotografia del prefisso condiviso non le serve a niente. E' la stessa
# regola di conversation_cache_slot per la chat, con la chiave presa
# dalla parte che non cambia.
if cache_slot is None:
cache_slot = conversation_cache_slot(
[{"role": "system", "content": fixed or state or ""}], self.server.kv_slots)
fixed_prefix = f"{fixed}\n\n" if fixed else ""
state_prefix = fixed_prefix + (f"Context:\n{state}\n\n" if state else "")
started = time.time()
read_total = 0
prompt_max = 0
with self.server.scheduler.admit(self.client_disconnected, cache_slot) as admission:
queue_wait, cache_slot = admission
def score(text, pin):
"""Un giro sul motore: niente generazione, solo la lettura.
max_tokens=0 vale solo con logprobs>0 e vuol dire "leggi il
prompt e fermati". Chiedere un token costerebbe un passo di
decodifica completo per opzione, buttato via."""
echoes = []
accepted = {}
def on_accept(value):
accepted.update(value)
self.server.generate(
text, 0, 0.0, 1.0, lambda _chunk: None, cache_slot,
self.client_disconnected, logprobs=1, pin=pin,
on_echo=echoes.append, on_accept=on_accept)
n = accepted.get("prompt_tokens")
if n is None: # motore senza ACCEPT (olmoe)
n = max((e["pos"] for e in echoes), default=-1) + 1
return n, echoes
def choose(prefix, choices, norm):
"""Fotografa `prefix`, poi un giro per opzione: ognuna paga solo
i propri token. Torna (scored, entropia, token del prefisso)."""
nonlocal read_total, prompt_max
n_prefix, fresh = score(prefix, True)
prompt_max = max(prompt_max, n_prefix)
known = self.server.pinned_prefixes.get(cache_slot)
if known and known[0] == fixed_prefix and len(fresh) > n_prefix - known[1]:
# The engine read the fixed prefix again: its photo was
# evicted. Forget it, so the next request takes it anew.
self.server.pinned_prefixes.pop(cache_slot, None)
scored = []
for option in choices:
# Il cliente se n'e andato: smettere subito invece di
# macinare le opzioni restanti per nessuno. Con una sola
# slot KV un menu lungo abbandonato la terrebbe occupata
# per minuti, e le richieste dietro andrebbero in coda fino
# al timeout -- e' cosi che sono usciti i primi 429.
if self.client_disconnected():
raise ClientCancelled()
_, tail = score(prefix + " " + option, False)
rows = [e for e in tail if e["pos"] >= n_prefix and e["logprob"] is not None]
total = sum(e["logprob"] for e in rows)
count = max(len(rows), 1)
scored.append({"option": option, "logprob": total, "tokens": len(rows),
"mean_logprob": total / count})
if not any(entry["tokens"] for entry in scored):
raise APIError(502, "The engine returned no log probabilities for the "
"options.", None, "engine_error", "server_error")
key = "mean_logprob" if norm == "mean" else "logprob"
if norm == "mean":
token_counts = {entry["tokens"] for entry in scored if entry["tokens"]}
if len(token_counts) > 1:
counts = ", ".join(f"{entry['option']}={entry['tokens']}"
for entry in scored)
print(f"[systemone] WARNING: normalize=mean with unequal option "
f"token counts ({counts}) — per-token averages favor "
f"multi-token options; consider normalize=sum",
file=sys.stderr)
top = max(entry[key] for entry in scored)
weights = [math.exp(entry[key] - top) for entry in scored]
total_weight = sum(weights) or 1.0
for entry, weight in zip(scored, weights):
entry["p"] = weight / total_weight
scored.sort(key=lambda entry: -entry["p"])
entropy = -sum(e["p"] * math.log(max(e["p"], 1e-12)) for e in scored)
entropy /= math.log(max(len(scored), 2))
read_total += sum(e["tokens"] for e in scored)
return scored, round(entropy, 6), n_prefix
# Lo stato da solo, fotografato per primo: e' il livello che tutte
# le domande condividono. Con un livello solo la domanda si rilegge
# una volta per opzione; con due, 176 token invece di 496 su quattro
# item (misurato). Il punto di ritorno sullo stato serve anche TRA
# richieste: la pagina web manda una domanda per richiesta sullo
# stesso documento, e senza la fotografia ognuna rifarebbe il
# prefill di tutto il documento. Quando la fotografia non puo'
# pagare (uno stato visto una volta sola) systemone la spegne.
# The fixed prefix is photographed once per slot, not per request:
# sending it again would read it again (an identical prompt is not a
# strict prefix of itself, so no photo can resume it), which is the
# cost the prefix exists to save. If the photo is evicted, the next
# question shows it (it reads the prefix fresh) and it is retaken.
if fixed_prefix and self.server.pinned_prefixes.get(cache_slot, ("",))[0] != fixed_prefix:
n_fixed, _ = score(fixed_prefix, True)
prompt_max = max(prompt_max, n_fixed)
self.server.pinned_prefixes[cache_slot] = (fixed_prefix, n_fixed)
if state_prefix and state_prefix != fixed_prefix and pin_state:
n_state, _ = score(state_prefix, True)
prompt_max = max(prompt_max, n_state)
answers = []
for text, choices in questions:
prefix = state_prefix + f"Question: {text}\nAnswer:"
scored, entropy, _ = choose(prefix, choices, normalize)
answers.append({"question": text, "answer": scored[0]["option"],
"entropy": entropy, "choices": scored})
headers = {"x-colibri-queue-wait-ms": str(round(queue_wait * 1000)),
"x-colibri-elapsed-ms": str(round((time.time() - started) * 1000))}
return answers, {"prompt_tokens": prompt_max, "read_tokens": read_total}, headers
# ------------------------------------------------------------ Jev-compatible
#
# POST /v1/systemone speaks the request and the reply of TypeSafe's Jev
# API (docs.typesafe.ai/api): a client written for it points at colibri
# and changes the base URL, nothing else. It is colibri's one decision
# API. On a language model the three primitives become closed questions
# on the scoring channel above (_score_questions): the state is
# photographed once and every question pays only its own tokens. A
# decision engine gets the request as it is (DECIDE, _systemone_decide).
#
# noul -> one yes/no question. `noul` is the probability of yes. The
# optional criteria (what true and false mean) go into the
# question text.
# choice -> the labels of `criteria` are the options; their descriptions
# go into the question text, because a label alone ("billing")
# does not say what it means. `confidence` follows their
# documented formula, (n * peak - 1) / (n - 1).
# score -> the levels of `criteria` are the options "1".."n" for the
# model; the reply numbers them from 0 as Jev does. `score` is
# the expected level, `legend` the levels as sent,
# `confidence` as for choice.
#
# What differs, stated rather than hidden: `model` echoes the served
# model, not "jev-latest"; `usage.output_tokens` counts the option tokens
# READ, since this engine generates nothing; validation errors are 422 as
# theirs are, with this server's error envelope. docs/systemone.md has the
# mapping table.
_SYSTEMONE_MAX_QUESTIONS = 64
@staticmethod
def _systemone_text(value, where):
"""Jev's EntryType: a string, or JSON given as an object or an array."""
if value is None:
return None
if isinstance(value, str):
return value.strip() or None
if isinstance(value, (dict, list)):
return json.dumps(value, ensure_ascii=False, indent=2)
raise APIError(422, f"`{where}` must be a string, an object or an array.", where)
@staticmethod
def _systemone_confidence(probabilities):
"""(n * peak - 1) / (n - 1): 1 when all the mass is on one label, 0 when flat."""
values = [float(v) for v in probabilities]
n = len(values)
if n < 2:
return 1.0
return round(max(0.0, (n * max(values) - 1.0) / (n - 1)), 6)
@staticmethod
def _systemone_legend(value, index):
"""A score level as the reply's `legend` gives it back: the criterion the
caller sent (text, object or array), or `level <n>` for one sent as null."""
return value if isinstance(value, (str, dict, list)) else f"level {index}"
def systemone(self, body, request_id):
if body.get("state") is None:
raise APIError(422, "`state` is required: the content the questions are about.", "state")
# Text, an object or an array; an empty text is a state with nothing in it,
# which the Jev schema allows, and the questions are asked without context.
state = self._systemone_text(body.get("state"), "state")
raw = body.get("questions")
if not isinstance(raw, dict) or not raw:
raise APIError(422, "`questions` must be a non-empty object of id: question.", "questions")
if len(raw) > self._SYSTEMONE_MAX_QUESTIONS:
raise APIError(422, f"`questions` accepts at most {self._SYSTEMONE_MAX_QUESTIONS} entries.",
"questions")
plan = [] # (id, kind, text, options, levels)
for qid, question in raw.items():
where = f"questions.{qid}"
if not isinstance(qid, str) or not qid.strip():
raise APIError(422, "Every question id must be a non-empty string.", "questions")
if not isinstance(question, dict):
raise APIError(422, f"`{where}` must be an object.", where)
kind = question.get("type")
instructions = self._systemone_text(question.get("instructions"), f"{where}.instructions")
criteria = question.get("criteria")
if kind == "noul":
if criteria is not None and not isinstance(criteria, dict):
raise APIError(422, f"`{where}.criteria` must be an object with `true` and/or `false`.",
f"{where}.criteria")
yes = self._systemone_text((criteria or {}).get("true"), f"{where}.criteria.true")
no = self._systemone_text((criteria or {}).get("false"), f"{where}.criteria.false")
text = instructions or "Is this true?"
if yes:
text += f"\nyes: {yes}"
if no:
text += f"\nno: {no}"
plan.append((qid, "noul", text + "\nAnswer yes or no.", ["yes", "no"], None))
elif kind == "choice":
if not isinstance(criteria, dict) or not criteria:
raise APIError(422, f"`{where}.criteria` must be a non-empty object of label: description.",
f"{where}.criteria")
if len(criteria) > 255:
raise APIError(422, f"`{where}.criteria` accepts at most 255 labels.", f"{where}.criteria")
labels, lines = [], []
for label, description in criteria.items():
if not isinstance(label, str) or not label.strip():
raise APIError(422, f"Every label of `{where}.criteria` must be a non-empty string.",
f"{where}.criteria")
labels.append(label)
text = self._systemone_text(description, f"{where}.criteria.{label}")
lines.append(f"- {label}: {text}" if text else f"- {label}")
text = (instructions or "Which of the following applies?") + "\nOptions:\n" + "\n".join(lines)
plan.append((qid, "choice", text + "\nAnswer with one of the options.", labels, None))
elif kind == "score":
# Jev numbers the levels from zero, in the order given (its OpenAPI
# schema: "Each description's position determines its score,
# starting at zero"). The model reads them as 1..n, the numbering a
# rubric is usually written in, and the reply maps them back.
if not isinstance(criteria, list) or not 1 <= len(criteria) <= 255:
raise APIError(422, f"`{where}.criteria` must be an array of 1 to 255 level descriptions.",
f"{where}.criteria")
levels = [self._systemone_text(c, f"{where}.criteria[{i}]") or f"level {i}"
for i, c in enumerate(criteria)]
text = (instructions or "Rate this on the scale below.") + "\nScale:\n" + \
"\n".join(f"{i + 1}: {d}" for i, d in enumerate(levels))
plan.append((qid, "score", text + "\nAnswer with the number.",
[str(i + 1) for i in range(len(levels))],
[self._systemone_legend(c, i) for i, c in enumerate(criteria)]))
else:
raise APIError(422, f"`{where}.type` must be \"noul\", \"choice\" or \"score\".", f"{where}.type")
options = self._systemone_options(body)
if engine_decides(self.server.engine):
if options["prefix"]:
raise APIError(422, "`prefix` is a language-model option (a fixed text photographed "
"before a changing state); a decision engine reads the whole "
"request in one pass: put that text in `state` or `instructions`.",
"prefix")
self._systemone_decide(body, plan, request_id, options["cache_slot"])
return
# A question with one option has its answer already: nothing to score.
asked = [entry for entry in plan if len(entry[3]) > 1]
scored, usage, headers = [], {"prompt_tokens": 0, "read_tokens": 0}, None
if asked:
# Photograph the state when it can pay: within this request (two or
# more questions read it), or because it came back from an earlier
# one. A state that changes on every call (a game, a feed) is read
# straight through instead. `pin_state` says it explicitly.
pin_state = options["pin_state"]
seen = self.server.state_seen(state or "")
if pin_state is None:
pin_state = len(asked) > 1 or seen
scored, usage, headers = self._score_questions(
state or "", [(text, choices) for _, _, text, choices, _ in asked],
normalize=options["normalize"], pin_state=pin_state,
fixed=options["prefix"], cache_slot=options["cache_slot"])
scored = iter(scored)
answers = {}
for qid, kind, _, choices, levels in plan:
if len(choices) > 1:
got = next(scored)
p = {c["option"]: c["p"] for c in got["choices"]}
else:
p = {choices[0]: 1.0}
if kind == "noul":
answers[qid] = {"type": "noul", "noul": round(float(p.get("yes", 0.0)), 6)}
elif kind == "choice":
answers[qid] = {"type": "choice", "choice": max(choices, key=lambda o: p[o]),
"probabilities": {o: round(float(p[o]), 6) for o in choices},
"confidence": self._systemone_confidence(p.values())}
else:
values = [float(p[o]) for o in choices]
answers[qid] = {"type": "score",
"score": round(sum(i * v for i, v in enumerate(values)), 6),
"legend": {str(i): legend for i, legend in enumerate(levels)},
"probabilities": {str(i): round(v, 6) for i, v in enumerate(values)},
"confidence": self._systemone_confidence(values)}
self.send_json(200, self._systemone_reply(answers, request_id, usage["prompt_tokens"],
usage["read_tokens"]),
request_id, headers)
def _systemone_options(self, body):
"""colibri's optional fields on /v1/systemone, none of which a Jev client
sends (docs/systemone.md lists them):
normalize "sum" (default) or "mean": how a language model's option
log-probabilities become one score per option
pin_state true / false: photograph the state for the next request;
omitted, the server decides (see systemone)
prefix a fixed text read before the state and always photographed
cache_slot the KV slot, as on the chat endpoints
On a decision engine normalize and pin_state have nothing to act on and
are ignored; prefix is refused."""
normalize = body.get("normalize", "sum")
if normalize not in ("sum", "mean"):
raise APIError(422, "`normalize` must be \"sum\" or \"mean\".", "normalize")
pin_state = body.get("pin_state")
if pin_state is not None and not isinstance(pin_state, bool):
raise APIError(422, "`pin_state` must be true or false.", "pin_state")
prefix = body.get("prefix")
if prefix is not None and (not isinstance(prefix, str) or not prefix.strip()):
raise APIError(422, "`prefix` must be a non-empty string.", "prefix")
cache_slot = body.get("cache_slot")
if cache_slot is not None and (isinstance(cache_slot, bool) or not isinstance(cache_slot, int)
or not 0 <= cache_slot < self.server.kv_slots):
raise APIError(422, f"`cache_slot` must be an integer from 0 to {self.server.kv_slots - 1}.",
"cache_slot")
return {"normalize": normalize, "pin_state": pin_state, "prefix": prefix,
"cache_slot": cache_slot}
def _systemone_reply(self, answers, request_id, input_tokens, output_tokens):
"""The reply both paths send, in the Jev shape: `model` is the served
model, `provider` says who answered, and `usage.cost` is what this server
charges, nothing."""
return {"id": request_id, "model": self.server.model_id, "provider": "colibri",
"answers": answers,
"usage": {"input_tokens": int(input_tokens), "output_tokens": int(output_tokens),
"cost": 0}}
def _systemone_decide(self, body, plan, request_id, cache_slot=None):
"""The native path: the request as one DECIDE record, the engine's
probabilities back, shaped exactly like the LLM path's reply. No prompt is
rendered and no option is scored on its own: one round trip, one forward."""
form = getattr(self.server.engine, "decide_record", None)
if form not in (None, "raw"):
raise APIError(502, f"The decision engine asks for a record form this server does not "
f"write ({form!r}).", None, "engine_error", "server_error")
record = systemone_decision_record(body, form)
if cache_slot is None:
cache_slot = conversation_cache_slot([{"role": "system", "content": record["state"]}],
self.server.kv_slots)
started = time.time()
with self.server.scheduler.admit(self.client_disconnected, cache_slot) as admission:
queue_wait, cache_slot = admission
call = time.monotonic()
try:
decision, _stats = self.server.engine.decide(record, cache_slot,
self.client_disconnected)
finally:
self.server.scheduler.observe_timing("engine_call_seconds", time.monotonic() - call)
got = decision.get("answers") if isinstance(decision, dict) else None
if not isinstance(got, list) or len(got) != len(plan):
raise APIError(502, "The decision engine answered a different number of questions.",
None, "engine_error", "server_error")
answers = {}
for (qid, kind, _, _, levels), question, answer in zip(plan, record["questions"], got):
probs = answer.get("probs") if isinstance(answer, dict) else None
n = len(question["options"])
if (answer.get("id") != qid or not isinstance(probs, list) or len(probs) != n or
not all(isinstance(p, (int, float)) and math.isfinite(p) for p in probs)):
raise APIError(502, f"The decision engine sent no usable probabilities for `{qid}`.",
None, "engine_error", "server_error")
best = max(range(n), key=lambda i: (probs[i], -i))
if kind == "noul":
answers[qid] = {"type": "noul", "noul": round(float(probs[1]), 6)}
elif kind == "choice":
labels = [option["label"] for option in question["options"]]
answers[qid] = {"type": "choice", "choice": labels[best],
"probabilities": {label: round(float(p), 6) for label, p in zip(labels, probs)},
"confidence": self._systemone_confidence(probs)}
else:
answers[qid] = {"type": "score",
"score": round(sum(i * float(p) for i, p in enumerate(probs)), 6),
"legend": {str(i): legend for i, legend in enumerate(levels)},
"probabilities": {str(i): round(float(p), 6) for i, p in enumerate(probs)},
"confidence": self._systemone_confidence(probs)}
reply = self._systemone_reply(answers, request_id, decision.get("input_tokens") or 0, 0)
headers = {"x-colibri-queue-wait-ms": str(round(queue_wait * 1000)),
"x-colibri-elapsed-ms": str(round((time.time() - started) * 1000))}
engine_ms = decision.get("engine_ms")
if isinstance(engine_ms, (int, float)) and math.isfinite(engine_ms):
headers["x-colibri-engine-ms"] = str(round(engine_ms, 1))
self.send_json(200, reply, request_id, headers)
# ------------------------------------------------------------- images
#
# POST /v1/images/generations, the OpenAI shape plus a `colibri` block with
# what was actually drawn (the seed above all: without it an image cannot be
# reproduced). One image at a time through the same scheduler as chat, so
# the queue, its limits and /metrics mean the same thing for both.
def image_generation(self, body, request_id):
engine = self.server.engine
if not is_image_engine(engine):
if not engine_chats(engine):
raise APIError(400, f"`{self.server.model_id}` does not generate images: it is a "
"decision model (POST /v1/systemone).",
"model", "unsupported_endpoint")
raise APIError(400, f"`{self.server.model_id}` does not generate images: it is a "
"chat model (POST /v1/chat/completions).",
"model", "unsupported_endpoint")
try:
request = image_engine.image_request(body, engine.info)
except image_engine.ImageRequestError as error:
raise APIError(400, str(error), error.param, "invalid_value")
stream = body.get("stream", False)
if not isinstance(stream, bool):
raise APIError(400, "`stream` must be a boolean.", "stream")
partial_images = body.get("partial_images")
if partial_images is not None and (isinstance(partial_images, bool) or
not isinstance(partial_images, int) or
not 0 <= partial_images <= 16):
raise APIError(400, "`partial_images` must be an integer between 0 and 16.",
"partial_images")
created = int(time.time())
with contextlib.ExitStack() as stack:
try:
queue_wait, _slot = stack.enter_context(
self.server.scheduler.admit(self.client_disconnected))
except APIError as error:
if error.status == 429:
# The contract's word for "busy past the queue" is 503: an
# image takes minutes, and a client that retries after a
# second, as 429 invites, only refills the queue.
raise APIError(503, error.message, None, error.code, "server_error",
error.headers) from None
raise
if not engine.alive:
raise APIError(503, f"The image engine is not running ({engine.dead or 'exited'}).",
None, "engine_unavailable", "server_error")
queue_headers = {"x-colibri-queue-wait-ms": str(round(queue_wait * 1000))}
if stream:
self._image_stream(engine, request, request_id, created, queue_headers,
partial_images)
return
try:
result = engine.generate(request["prompt"], request["width"], request["height"],
request["steps"], request["seed"], preview=False,
cancelled=self.client_disconnected)
except image_engine.ImageCancelled:
raise ClientCancelled() from None
except image_engine.ImageEngineError as error:
raise APIError(500, f"The image engine failed: {error}", None, "engine_error",
"server_error") from None
self.send_json(200, self._image_payload(result, created), request_id, queue_headers)
@staticmethod
def _image_payload(result, created):
import base64
png = image_engine.encode_png(result["width"], result["height"], result["rgba"], 4)
return {"created": created,
"data": [{"b64_json": base64.b64encode(png).decode("ascii"),
"revised_prompt": None}],
"colibri": {"width": result["width"], "height": result["height"],
"seed": result["seed"], "steps": result["steps"],
"timings": result["timings"]}}
def _image_stream(self, engine, request, request_id, created, queue_headers,
partial_images):
"""SSE: progress, optional partial images, completed, [DONE].
The 200 is committed on the engine's first frame, not before: an engine
that refuses the request outright still answers with a real HTTP error
instead of an error event inside a stream that already said OK."""
import base64
lock = threading.Lock()
state = {"started": False, "connected": True, "last": time.monotonic(), "partials": 0}
def start():
if state["started"]:
return
state["started"] = True
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Cache-Control", "no-cache")
self.send_header("X-Accel-Buffering", "no")
self.send_header("Connection", "close")
self.close_connection = True
self.send_header("x-request-id", request_id)
for name, value in queue_headers.items():
self.send_header(name, value)
self.send_cors_headers()
try:
self.end_headers()
except OSError:
# The client left between admission and the first frame: count
# it as gone, so the generation is cancelled like any hang-up
# instead of this exception abandoning it mid-image.
state["connected"] = False
def write(data):
with lock:
if not state["connected"]:
return
try:
self.wfile.write(data)
self.wfile.flush()
state["last"] = time.monotonic()
except OSError:
state["connected"] = False
def event(name, data):
start()
write(f"event: {name}\ndata: "
f"{json.dumps(data, ensure_ascii=False, separators=(',', ':'))}\n\n".encode())
def cancelled():
return not state["connected"] or self.client_disconnected()
def on_progress(frame):
event("image_generation.progress",
{key: frame.get(key) for key in ("stage", "step", "steps", "elapsed")})
def on_preview(frame, pixels):
limit = partial_images
if limit is not None:
# Spread a requested number of partial images over the run
# instead of spending them on the first steps.
steps = max(1, int(frame.get("steps") or request["steps"]))
due = (state["partials"] + 1) * steps / (limit + 1)
if state["partials"] >= limit or int(frame.get("step") or 0) < due:
return
channels = frame.get("channels", 3)
try:
png = image_engine.encode_png(frame["width"], frame["height"], pixels, channels)
except (KeyError, TypeError, ValueError) as error:
sys.stderr.write(f"[api] {request_id}: unusable PREVIEW frame ({error})\n")
return
event("image_generation.partial_image",
{"b64_json": base64.b64encode(png).decode("ascii"),
"partial_image_index": state["partials"]})
state["partials"] += 1
def on_idle():
# SSE comments keep proxies and idle timers from closing a stream
# that is legitimately silent through a long denoising step.
if state["started"] and time.monotonic() - state["last"] >= 10:
write(b": keepalive\n\n")
try:
result = engine.generate(request["prompt"], request["width"], request["height"],
request["steps"], request["seed"],
preview=partial_images != 0, on_progress=on_progress,
on_preview=on_preview, cancelled=cancelled,
on_idle=on_idle)
except image_engine.ImageCancelled:
if not state["started"]:
raise ClientCancelled() from None
# Once the 200 is out, every ending is an event: a client still
# listening (the web UI) is told the image will not come, then the
# stream closes the way every stream closes.
event("error", {"error": {"message": "cancelled", "type": "cancelled",
"param": None, "code": "cancelled"}})
write(b"data: [DONE]\n\n")
raise ClientCancelled() from None # counted as cancelled, not completed
except image_engine.ImageEngineError as error:
if not state["started"]:
raise APIError(500, f"The image engine failed: {error}", None, "engine_error",
"server_error") from None
event("error", {"error": {"message": f"The image engine failed: {error}",
"type": "server_error", "param": None,
"code": "engine_error"}})
write(b"data: [DONE]\n\n")
return
event("image_generation.completed", self._image_payload(result, created))
write(b"data: [DONE]\n\n")
def _fail(self, error, request_id):
"""Report an error, unless the response is already on the wire. Once a streaming 200
is committed, a second status line would be framed as SSE body -- clients saw a whole
`HTTP/1.1 500` spliced into the event stream. All we can still do is stop talking; the
stream ends at the close, which the 200 already announced (#597 item 3)."""
if self._committed:
self.close_connection = True
return
self.send_json(error.status, self.error_body(error), request_id, error.headers)
def error_body(self, error):
"""Anthropic clients parse a different error envelope; the OpenAI one is unchanged.
A /v1/systemone validation error also carries Jev's `detail` list (FastAPI's
form: loc, msg, type), which a client written for Jev may read."""
path = urlsplit(self.path).path
if path == "/v1/systemone" and error.status == 422:
body = error_object(error)
loc = ["body"]
for part in re.split(r"\.(?![^\[]*\])", error.param or ""):
match = re.fullmatch(r"(.*?)((?:\[\d+\])*)", part)
if match.group(1):
loc.append(match.group(1))
loc.extend(int(i) for i in re.findall(r"\[(\d+)\]", match.group(2)))
body["detail"] = [{"loc": loc, "msg": error.message, "type": "value_error"}]
return body
if path != "/v1/messages":
return error_object(error)
return {"type": "error", "error": {"type": error.error_type, "message": error.message}}
def generation(self, body, prompt, request_id, chat, tools=None, tool_choice=None,
enable_thinking=False, audio=None, image=None,
add_generation_prompt=True):
# COLI_DEBUG tees the engine transaction to stderr: 1 = decoded output stream only,
# 2 = both sides (rendered prompt + output). render_chat already folds prior turns and
# tool results into `prompt`, so level 2 is the full conversation the engine saw.
try:
dbg = int(os.environ.get("COLI_DEBUG", "0"))
except ValueError:
dbg = 0
if dbg >= 2:
sys.stderr.write(f"\n===== PROMPT [{request_id}] =====\n{prompt}\n===== OUTPUT [{request_id}] =====\n")
sys.stderr.flush()
maximum, temperature, top_p, grammar, _requested_stop_sequences = generation_options(
body, self.server.max_tokens)
family = family_by_id(ARCH)
if grammar is not None and not family.capabilities.grammar_payload:
# sibling engines speak the 6-field SUBMIT header only; sending the
# grammar payload extension would desync its stdin framing.
raise APIError(400, f"`response_format` grammars are not supported by the {ARCH} "
"engine yet.", "response_format", "unsupported_parameter")
engine_k, echo, display_k = logprobs_options(
body, chat, getattr(self.server.engine, "supports_logprobs_echo", False))
stop_sequences, ignore_leading_stop = stop_policy(body, chat)
# tools and tool_choice come from chat_completion() already processed/filtered
if chat and tool_choice == "none":
tools = None # client forbade tools: never surface tool_calls
cache_slot = body.get("cache_slot")
if (cache_slot is not None and
(isinstance(cache_slot, bool) or not isinstance(cache_slot, int) or
not 0 <= cache_slot < self.server.kv_slots)):
raise APIError(400, f"`cache_slot` must be an integer between 0 and {self.server.kv_slots - 1}.",
"cache_slot")
if cache_slot is None and self.server.kv_slots > 1:
# #634: pin each conversation to a stable KV slot so multi-turn reuses its
# cached prefix instead of re-prefilling. Only when the request carries a
# conversation; raw /v1/completions keeps the scheduler's free-slot pick.
conversation = body.get("messages")
if isinstance(conversation, list) and conversation:
cache_slot = conversation_cache_slot(conversation, self.server.kv_slots)
stream = body.get("stream", False)
if not isinstance(stream, bool):
raise APIError(400, "`stream` must be a boolean.", "stream")
if engine_k and stream:
# A named 400, not a silent drop of the numeric channel: streamed per-delta
# logprobs are not built, and nothing below this point threads the channel
# into a streaming call.
raise APIError(400, "`logprobs` is not supported together with `stream`.",
"logprobs", "unsupported_parameter")
stream_options = body.get("stream_options") if stream else None
if stream and stream_options is not None and not isinstance(stream_options, dict):
raise APIError(400, "`stream_options` must be an object.", "stream_options")
include_usage = bool((stream_options or {}).get("include_usage"))
object_name = "chat.completion" if chat else "text_completion"
id_prefix = "chatcmpl-" if chat else "cmpl-"
completion_id = id_prefix + uuid.uuid4().hex
created = int(time.time())
with self.server.scheduler.admit(self.client_disconnected, cache_slot) as admission, \
contextlib.ExitStack() as stream_cleanup:
queue_wait, cache_slot = admission
queue_headers = {"x-colibri-queue-wait-ms": str(round(queue_wait * 1000))}
if not stream:
output = []
stop_filter = StopFilter(stop_sequences, output.append, ignore_leading_stop,
track_spans=bool(engine_k))
sideband = ToolSideband(ARCH == "kimi" and chat and bool(tools),
stop_sequences, ignore_leading_stop)
# The engine sends an ECHO frame for every prompt position of every
# opted-in request, so passing a sink is the retention opt-in.
prompt_echoes = []
def generation_stopped():
return stop_filter.stopped() or sideband.stopped()
stats = self.server.generate(
prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,
self.client_disconnected, grammar=grammar, stopped=generation_stopped,
**({"on_tool": sideband.feed} if sideband.enabled else {}),
**({"audio": audio} if audio else {}),
**({"image": image} if image is not None else {}),
**({"on_echo": prompt_echoes.append} if engine_k and echo else {}),
**_engine_extension_args(engine_k))
stop_filter.finish()
sideband.finish()
text = "".join(output)
# What the stop filter emitted, before any split or tool-call parse: the
# base the rest of the pipeline's maps are chained onto.
raw_text = text
reasoning = ""
answer_spans = tool_spans = None
if ARCH == "inkling":
text, reasoning = split_inkling(text)
elif chat:
# #597 item 4: GLM emits reasoning then </think> then the answer. Route the
# reasoning to reasoning_content instead of dumping it (or the raw </think>)
# into the visible answer / tool-call parser.
if engine_k:
reasoning, text, (_thinking_spans, answer_spans) = (
split_thinking_reply_spans(text, enable_thinking,
add_generation_prompt))
else:
reasoning, text = split_thinking_reply(text, enable_thinking,
add_generation_prompt)
# The tool-call parse runs before the logprobs tail because the arrays
# describe the string it produces. It reads nothing the tail writes.
content, calls, content_spans = None, [], None
if chat and tools:
if body.get("strict_tool_calls") and ARCH == "glm":
if stats["length_limited"] and BOX_START in text:
raise APIError(502, "GLM tool call ended at the generation limit.",
code="invalid_model_tool_call", error_type="server_error")
content, calls = parse_glm_tool_calls_strict(text, tools)
elif engine_k:
content, calls, _box_spans, tool_spans = parse_arch_tool_calls_spans(
text, tools, sideband.reply())
else:
content, calls = parse_arch_tool_calls(text, tools, sideband.reply())
# Compose the stages' maps into one raw-stream to returned-string map;
# a stage with no map leaves the chain at the last string it vouches for.
if chat and answer_spans is not None and engine_k:
chain = _compose_span_maps(stop_filter.spans, answer_spans)
target = text
if tools:
if tool_spans is None:
chain = None
else:
chain, target = _compose_span_maps(chain, tool_spans), content
if chain is not None:
content_spans = (target, chain)
text, logprobs_obj, length_finish = _completion_choice_fields(
stats, raw_text=raw_text, text=text, chat=chat, engine_k=engine_k,
echo=echo, display_k=display_k, prompt_echoes=prompt_echoes,
stop_spans=stop_filter.spans, content_spans=content_spans,
cache_slot=cache_slot)
if chat and tools:
message = {"role": "assistant", "content": content or None, "refusal": None}
if reasoning:
message["reasoning_content"] = reasoning
if calls:
message["tool_calls"] = calls
finish = "tool_calls" if calls else length_finish
choice = {"index": 0, "message": message, "logprobs": logprobs_obj,
"finish_reason": finish}
else:
_msg = {"role": "assistant", "content": text, "refusal": None}
if reasoning:
_msg["reasoning_content"] = reasoning
choice = ({"index": 0, "message": _msg,
"logprobs": logprobs_obj, "finish_reason": length_finish} if chat else
{"index": 0, "text": text, "logprobs": logprobs_obj,
"finish_reason": length_finish})
self.send_json(200, {"id": completion_id, "object": object_name, "created": created,
"model": self.server.model_id, "choices": [choice], "usage": self.usage(stats)},
request_id, queue_headers)
return
stream_object = "chat.completion.chunk" if chat else object_name
# #597 item 6: DO NOT commit the 200 yet. The engine validates the prompt against the
# context AFTER we would have sent headers, so an oversized prompt used to be a
# CONTEXT_EXCEEDED discovered too late to send a clean 400. Defer the SSE headers into
# start_stream(), fired on the engine's ACCEPT frame (before prefill); an ERROR that
# arrives before ACCEPT propagates as an APIError with nothing committed -> proper 400.
connected = False
stream_started = [False]
ka_thread = [None]
# KEEPALIVE: engine.generate() blocks SILENTLY during the (minutes-long) cold
# prefill, and the client drops the socket after its idle timeout. A background pump
# emits a keepalive delta whenever no event has been written for KA_GAP seconds. All
# wfile writes share ka_lock so the pump and event() never interleave; last_write
# gates the pump so it stays quiet while real tokens are flowing (e.g. during decode).
ka_lock = threading.Lock()
last_write = [time.time()]
ka_stop = threading.Event()
KA_GAP = 10.0
dbg_echo = dbg >= 1 # tee decoded tokens to stderr (COLI_DEBUG level parsed in generation())
def event(choices, usage_marker=False):
nonlocal connected
if not connected:
return
event_body = {"id": completion_id, "object": stream_object, "created": created,
"model": self.server.model_id, "choices": choices}
if include_usage:
event_body["usage"] = None if not usage_marker else usage_marker
data = json.dumps(event_body, ensure_ascii=False, separators=(",", ":"))
with ka_lock:
try:
self.wfile.write(f"data: {data}\n\n".encode())
self.wfile.flush()
last_write[0] = time.time()
except OSError:
connected = False
def _keepalive():
# #597: an empty delta already resets the client's idle timer without
# painting hundreds of dots in the reasoning panel during a minutes-long
# cold prefill. COLI_VISIBLE_KEEPALIVE=1 restores the old visible "." for
# diagnosing whether keepalives are being delivered at all.
visible = os.environ.get("COLI_VISIBLE_KEEPALIVE") == "1"
ping = [_keepalive_choice(chat, visible)]
while not ka_stop.wait(1.0):
if not connected:
return
if time.time() - last_write[0] >= KA_GAP:
event(ping)
def emit(text):
choice = ({"index": 0, "delta": {"content": text}, "logprobs": None,
"finish_reason": None} if chat else
{"index": 0, "text": text, "logprobs": None, "finish_reason": None})
event([choice])
def emit_reasoning(text): # thinking → reasoning_content deltas (chat only)
event([{"index": 0, "delta": {"reasoning_content": text},
"logprobs": None, "finish_reason": None}])
splitter = (InklingStreamSplit(emit, emit_reasoning if chat else None)
if ARCH == "inkling" else None)
# #597 item 4: GLM (chat) streams reasoning then </think> then the answer. Split the
# reasoning into reasoning_content deltas instead of leaking it — and the raw </think> —
# into visible content or the tool-call buffer.
glm_think = chat and ARCH != "inkling"
def start_stream(_accept_info=None):
# #597 item 6: commit the streaming 200 (and start the keepalive) exactly once,
# only after the engine ACCEPTs the prompt. Idempotent: generate() also calls this
# on the first DATA/DONE so an older engine with no ACCEPT frame still streams.
nonlocal connected
if stream_started[0]:
return
stream_started[0] = True
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Cache-Control", "no-cache")
self.send_header("X-Accel-Buffering", "no")
# An SSE body has neither Content-Length nor chunked framing, so end-of-message
# IS the close -- HTTP/1.1 requires us to say so, or the client waits for a
# length that never comes and then tries to reuse a socket we are about to drop.
# Set close_connection HERE, not after the last event: if generation raises once
# the 200 is out, the connection must still not be offered for reuse (#597 item 3).
self.send_header("Connection", "close")
self.close_connection = True
self.send_header("x-request-id", request_id)
for name, value in queue_headers.items(): self.send_header(name, value)
self.send_cors_headers()
self.end_headers()
connected = True
last_write[0] = time.time()
if chat:
event([{"index": 0, "delta": {"role": "assistant", "content": ""},
"logprobs": None, "finish_reason": None}])
ka_thread[0] = threading.Thread(target=_keepalive, daemon=True)
ka_thread[0].start()
stream_cleanup.callback(ka_thread[0].join, timeout=2)
stream_cleanup.callback(ka_stop.set)
if chat and tools:
# Suppress tool-call markers from the streamed content and parse the authoritative
# calls from the FULL reply after generation. Hold back a marker-length tail so a
# tool-call marker split across engine chunks is still caught.
sp = {"buf": "", "tool": False}
hold = _tool_hold()
raw = []
sideband = ToolSideband(ARCH == "kimi", stop_sequences,
ignore_leading_stop)
def feed_content(chunk): # answer text only (post-</think>)
raw.append(chunk)
if sideband.seen:
emit(chunk)
return
if sp["tool"]:
return
sp["buf"] += chunk
cut = _tool_cut(sp["buf"])
if cut >= 0:
if cut:
emit(sp["buf"][:cut])
sp["buf"] = ""
sp["tool"] = True
return
flush = max(0, len(sp["buf"]) - hold)
if flush:
emit(sp["buf"][:flush])
sp["buf"] = sp["buf"][flush:]
# #597: keep GLM reasoning out of the tool-call buffer — a think splitter sends it
# to reasoning_content and passes only the answer text on to feed_content/parser.
think = (ThinkingStreamSplit(emit_reasoning, feed_content,
initial_thinking=starts_in_reasoning(
enable_thinking, add_generation_prompt))
if glm_think else None)
def emit_tools(chunk):
if dbg_echo:
sys.stderr.write(chunk); sys.stderr.flush()
(think.feed if think else feed_content)(chunk)
stop_filter = StopFilter(stop_sequences, emit_tools, ignore_leading_stop)
def generation_stopped():
return stop_filter.stopped() or sideband.stopped()
stats = self.server.generate(
prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,
self.client_disconnected, grammar=grammar, stopped=generation_stopped,
**({"on_tool": sideband.feed} if sideband.enabled else {}),
on_accept=start_stream, **({"audio": audio} if audio else {}),
**({"image": image} if image is not None else {}),
**_engine_extension_args(engine_k))
stop_filter.finish()
sideband.finish()
if think:
think.finish()
if not sp["tool"] and sp["buf"]:
emit(sp["buf"]) # no tool call happened: flush held tail
_content, calls = parse_arch_tool_calls("".join(raw), tools,
sideband.reply())
for i, tc in enumerate(calls):
event([{"index": 0, "delta": {"tool_calls": [{"index": i, "id": tc["id"],
"type": "function", "function": {"name": tc["function"]["name"],
"arguments": tc["function"]["arguments"]}}]},
"logprobs": None, "finish_reason": None}])
finish = "tool_calls" if calls else ("length" if stats["length_limited"] else "stop")
else:
if splitter is not None: # inkling content/marker splitter
content_split = splitter
elif glm_think: # GLM <think> reasoning → reasoning_content
content_split = ThinkingStreamSplit(
emit_reasoning, emit,
initial_thinking=starts_in_reasoning(enable_thinking,
add_generation_prompt))
else:
content_split = None
def emit_plain(chunk):
if dbg_echo:
sys.stderr.write(chunk); sys.stderr.flush()
(content_split.feed if content_split else emit)(chunk)
stop_filter = StopFilter(stop_sequences, emit_plain, ignore_leading_stop)
stats = self.server.generate(
prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,
self.client_disconnected, grammar=grammar, stopped=stop_filter.stopped,
on_accept=start_stream, **({"audio": audio} if audio else {}),
**({"image": image} if image is not None else {}),
**_engine_extension_args(engine_k))
stop_filter.finish()
if content_split:
content_split.close()
finish = "length" if stats["length_limited"] else "stop"
# generate() returned, so the prompt was ACCEPTed and start_stream() ran; guard anyway.
start_stream()
ka_stop.set() # generation done: stop the keepalive pump
if ka_thread[0] is not None:
ka_thread[0].join(timeout=2)
final_choice = ({"index": 0, "delta": {}, "logprobs": None, "finish_reason": finish}
if chat else {"index": 0, "text": "", "logprobs": None,
"finish_reason": finish})
event([final_choice])
if include_usage:
event([], self.usage(stats))
if connected:
with ka_lock: # (#B9) share the pump's lock so [DONE] can't interleave a keepalive write
try:
self.wfile.write(b"data: [DONE]\n\n")
self.wfile.flush()
except OSError:
pass
# close_connection was already set when the 200 was committed (#597 item 3).
def client_disconnected(self):
try:
readable, _, _ = select.select([self.connection], [], [], 0)
if not readable:
return False
flags = socket.MSG_PEEK | getattr(socket, "MSG_DONTWAIT", 0)
return self.connection.recv(1, flags) == b""
except (OSError, ValueError):
return True
@staticmethod
def usage(stats):
prompt = stats["prompt_tokens"]
completion = stats["completion_tokens"]
return {"prompt_tokens": prompt, "completion_tokens": completion,
"total_tokens": prompt + completion}
def chat_completion(self, body, request_id):
strict_tools = body.get("strict_tool_calls", False)
if not isinstance(strict_tools, bool):
raise APIError(400, "`strict_tool_calls` must be a boolean.", "strict_tool_calls")
if strict_tools and ARCH != "glm":
raise APIError(400, "`strict_tool_calls` currently supports GLM only.",
"strict_tool_calls", "unsupported_parameter")
if strict_tools and body.get("stream") is True:
raise APIError(400, "`strict_tool_calls` requires `stream: false`.",
"stream", "unsupported_parameter")
if strict_tools and body.get("logprobs"):
raise APIError(400, "`strict_tool_calls` does not support `logprobs` yet.",
"logprobs", "unsupported_parameter")
reasoning_effort = body.get("reasoning_effort")
efforts = (None, "none", "minimal", "low", "medium", "high", "xhigh")
if reasoning_effort not in efforts:
raise APIError(400, "`reasoning_effort` must be none, minimal, low, medium, high, or xhigh.",
"reasoning_effort")
# COLI_THINK=1 makes thinking the default when the client sends NEITHER reasoning_effort
# nor enable_thinking (a global switch, like the old server's --think). An explicit
# client value always wins. Default off => exact OpenAI-standard behavior.
if reasoning_effort is None and "enable_thinking" not in body:
# Qwen3.8's official template defaults to enabled xhigh thinking;
# preserve the older opt-in default for the other families.
if chat_flavor() == "qwen38":
reasoning_effort = "xhigh"
elif ARCH == "mimo" or os.environ.get("COLI_THINK", "0") == "1":
# MiMo-V2.6's template thinks unless told not to (enable_thinking false)
reasoning_effort = "high"
enable_thinking = body.get("enable_thinking", reasoning_effort not in (None, "none"))
if not isinstance(enable_thinking, bool):
raise APIError(400, "`enable_thinking` must be a boolean.", "enable_thinking")
if (ARCH == "olmoe" or chat_flavor() == "qwen3_coder") and enable_thinking:
# Qwen3-Coder's template has no thinking mode either (no <think> anywhere).
# OLMoE's template has no thinking mode (render_chat_olmoe: "accepted
# but unused"), so the engine never emits <think>/</think>. Left on,
# the reasoning splitter files the ENTIRE answer as reasoning_content
# and streams an empty `content` -- the drop reported in #984, which
# bit streaming (ThinkingStreamSplit stays in thinking mode forever)
# while non-streaming happened to survive. Make the template's "unused"
# true end-to-end instead of trusting every path to opt out.
enable_thinking = False
# Qwen3.6's preserve_thinking (#1759), under the name Qwen's own API gives it. Off
# by default with thinking on: a standard client does not send the reasoning back,
# and the block would come back empty where the model did think. On by default with
# thinking off: there the block the history gets is the empty one the turn was
# actually generated after, so the resent history is the engine's state byte for
# byte and prefix reuse can engage. Only the qwen36 renderer reads it.
preserve_thinking = body.get("preserve_thinking", not enable_thinking)
if not isinstance(preserve_thinking, bool):
raise APIError(400, "`preserve_thinking` must be a boolean.", "preserve_thinking")
# The request's shape is checked before the image expanders and the renderer read it.
# generation() validates `tools` too, but only after rendering, and a renderer handed a
# `tools` of 5 or a `messages` of null raises TypeError, which do_POST answers with 500.
validate_tools(body)
tools = body.get("tools") or body.get("functions") or None
tool_choice = body.get("tool_choice")
if strict_tools and (not isinstance(tools, list) or not tools or tool_choice == "none"):
raise APIError(400, "`strict_tool_calls` requires active `tools`.",
"strict_tool_calls", "unsupported_parameter")
if strict_tools:
for index, tool in enumerate(tools):
params = _tool_function(tool).get("parameters") or {}
if (not isinstance(params, dict) or
not isinstance(params.get("properties", {}), dict) or
not isinstance(params.get("required", []), list)):
raise APIError(400, "Strict tool schemas need object `parameters`, "
"object `properties` and array `required`.", f"tools.{index}")
audio_clips = [] if ARCH == "inkling" else None
messages = body.get("messages")
if not isinstance(messages, list) or not messages:
raise APIError(400, "`messages` must be a non-empty array.", "messages")
messages, image = self.expand_images(messages)
add_generation_prompt = resolve_generation_prompt(messages, body)
prompt = render_chat_for_arch(messages, enable_thinking, reasoning_effort,
tools, tool_choice, audio_out=audio_clips,
add_generation_prompt=add_generation_prompt,
preserve_thinking=preserve_thinking)
self.generation(body, prompt, request_id, True, tools, tool_choice,
enable_thinking=enable_thinking,
add_generation_prompt=add_generation_prompt,
audio=b"".join(audio_clips) if audio_clips else None,
image=image)
def expand_images(self, messages):
"""(messages with placeholders, the one image or None) for this engine's family.
Le immagini diventano segnaposto PRIMA del rendering: il renderer tratta
poi turni di solo testo, e il conto dei segnaposto e quello degli
embedding non possono divergere. Both chat endpoints come through here,
so a picture is one thing whichever API delivered it."""
model_dir = getattr(self.server.engine, "model_dir", None)
if getattr(self.server.engine, "vision", None) is False and has_image_parts(messages):
# The engine said at its handshake that it loaded no tower. Refuse here,
# by name, before the picture is preprocessed and before the engine
# answers it with a bare BAD_REQUEST (a 500 the client cannot act on).
raise APIError(400, "this engine loaded no vision tower: the checkpoint declares "
"one in its config but its weights are not in the container, "
"so images cannot be served. Reconvert the checkpoint with its "
"vision weights, or send text only.",
"messages", "unsupported_content_type")
if ARCH == "glm53":
messages, images = expand_glm53_images(messages, model_dir)
elif ARCH == "deepseek_v41":
ceiling = os.environ.get("V41_MAX_IMAGE_TOKENS")
messages, images = expand_dsv41_images(messages, model_dir,
int(ceiling) if ceiling else None)
elif ARCH == "mimo":
# MiMo-V2.6's ViT reads the Qwen2-VL processor's patches, with the same
# placeholders: the Qwen expander serves it unchanged.
ceiling = os.environ.get("MIMO_MAX_IMAGE_TOKENS")
messages, images = expand_qwen38_images(messages, model_dir,
int(ceiling) if ceiling else None)
elif ARCH == "qwen38" or (ARCH == "qwen36" and qwen36_has_vision(model_dir)):
# Qwen3.5/3.6/3.8 share the tower and the preprocessor, so qwen36
# checkpoints converted with their tower take the same path (#1757).
ceiling = os.environ.get("Q38_MAX_IMAGE_TOKENS" if ARCH == "qwen38"
else "Q36_MAX_IMAGE_TOKENS")
messages, images = expand_qwen38_images(messages, model_dir,
int(ceiling) if ceiling else None)
else:
return messages, None
if len(images) > 1:
raise APIError(400, "one image per request for now; the engine "
"holds a single pending image.", "messages")
return messages, images[0] if images else None
# ---- Anthropic /v1/messages (#343) ----------------------------------------------------
ANTHROPIC_STOP = {"stop": "end_turn", "length": "max_tokens", "tool_calls": "tool_use"}
def anthropic_messages(self, body, request_id):
for unsupported, why in (("stop_sequences", "custom stop sequences"),
("top_k", "top-k sampling")):
if body.get(unsupported) not in (None, [], ""):
raise APIError(400, f"Colibri does not support `{unsupported}` ({why}) yet.",
unsupported, "unsupported_value")
messages, image = self.expand_images(anthropic_to_openai(body))
tools, tool_choice = anthropic_tools(body)
thinking = body.get("thinking")
if thinking is not None and not isinstance(thinking, dict):
raise APIError(400, "`thinking` must be an object.", "thinking")
enable_thinking = bool(thinking and thinking.get("type") == "enabled")
if not enable_thinking and thinking is None:
if chat_flavor() == "qwen38" or ARCH == "mimo":
enable_thinking = True
elif os.environ.get("COLI_THINK", "0") == "1":
enable_thinking = True
if ARCH == "olmoe" or chat_flavor() == "qwen3_coder":
enable_thinking = False # #984: OLMoE has no thinking mode (see the OpenAI path)
if body.get("max_tokens") is None:
raise APIError(400, "`max_tokens` is required.", "max_tokens")
# Reuse the OpenAI path's own validation by handing it an equivalent body.
translated = {"messages": messages, "max_tokens": body.get("max_tokens"),
"temperature": body.get("temperature"), "top_p": body.get("top_p"),
"stream": body.get("stream", False), "cache_slot": body.get("cache_slot")}
if tools:
translated["tools"] = tools
if tool_choice is not None:
translated["tool_choice"] = tool_choice
if tool_choice == "none":
tools = None
default_effort = "xhigh" if chat_flavor() == "qwen38" and thinking is None else "high"
add_generation_prompt = resolve_generation_prompt(messages, body)
prompt = render_chat_for_arch(messages, enable_thinking,
default_effort if enable_thinking else None,
tools, tool_choice,
add_generation_prompt=add_generation_prompt,
preserve_thinking=not enable_thinking)
self.anthropic_generation(translated, prompt, request_id, tools, enable_thinking,
add_generation_prompt, image)
def anthropic_generation(self, body, prompt, request_id, tools, enable_thinking,
add_generation_prompt=True, image=None):
maximum, temperature, top_p, grammar, _stop_sequences = generation_options(
body, self.server.max_tokens)
# Same policy as /v1/chat/completions: `body` is the translated OpenAI-shaped
# request, and anthropic_messages() has already refused a client `stop_sequences`,
# so this resolves to the implicit GLM role boundaries.
stop_sequences, ignore_leading_stop = stop_policy(body, True)
cache_slot = body.get("cache_slot")
if (cache_slot is not None and
(isinstance(cache_slot, bool) or not isinstance(cache_slot, int) or
not 0 <= cache_slot < self.server.kv_slots)):
raise APIError(400, f"`cache_slot` must be an integer between 0 and {self.server.kv_slots - 1}.",
"cache_slot")
if cache_slot is None and self.server.kv_slots > 1:
# #634: pin each conversation to a stable KV slot so multi-turn reuses its
# cached prefix instead of re-prefilling. Only when the request carries a
# conversation; raw /v1/completions keeps the scheduler's free-slot pick.
conversation = body.get("messages")
if isinstance(conversation, list) and conversation:
cache_slot = conversation_cache_slot(conversation, self.server.kv_slots)
stream = body.get("stream", False)
if not isinstance(stream, bool):
raise APIError(400, "`stream` must be a boolean.", "stream")
message_id = "msg_" + uuid.uuid4().hex[:24]
# GLM-5.3 opens <think> in the prompt even with thinking off (#1278), so its reply
# starts with reasoning either way. Split it off so the answer never carries the
# reasoning or a literal </think>, but only return it as a thinking block when the
# client asked for thinking: the real API never sends one otherwise, and clients
# read the answer from content[0].
split_reasoning = enable_thinking or starts_in_reasoning(enable_thinking,
add_generation_prompt)
def blocks_and_stop(text, stats, tool_reply=None):
"""Split a finished reply into Anthropic content blocks + stop_reason."""
content = []
reasoning = ""
if ARCH == "inkling":
text, reasoning = split_inkling(text)
elif split_reasoning:
reasoning, text = split_thinking_reply(text, enable_thinking,
add_generation_prompt)
if enable_thinking:
content.append({"type": "thinking", "thinking": reasoning,
"signature": ANTHROPIC_LOCAL_SIGNATURE})
calls = []
if tools:
text, calls = parse_arch_tool_calls(text, tools, tool_reply)
if text:
content.append({"type": "text", "text": text})
for call in calls:
function = call["function"]
try:
arguments = json.loads(function["arguments"])
except (json.JSONDecodeError, TypeError):
arguments = {}
content.append({"type": "tool_use", "id": call["id"],
"name": function["name"], "input": arguments})
reason = "tool_calls" if calls else ("length" if stats["length_limited"] else "stop")
return content, self.ANTHROPIC_STOP[reason]
with self.server.scheduler.admit(self.client_disconnected, cache_slot) as admission, \
contextlib.ExitStack() as stream_cleanup:
queue_wait, cache_slot = admission
queue_headers = {"x-colibri-queue-wait-ms": str(round(queue_wait * 1000))}
if not stream:
output = []
stop_filter = StopFilter(stop_sequences, output.append, ignore_leading_stop)
sideband = ToolSideband(ARCH == "kimi" and bool(tools), stop_sequences,
ignore_leading_stop)
def generation_stopped():
return stop_filter.stopped() or sideband.stopped()
stats = self.server.generate(
prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,
self.client_disconnected, grammar=grammar, stopped=generation_stopped,
**({"on_tool": sideband.feed} if sideband.enabled else {}),
**({"image": image} if image is not None else {}))
stop_filter.finish()
sideband.finish()
content, stop_reason = blocks_and_stop("".join(output), stats,
sideband.reply())
self.send_json(200, {
"id": message_id, "type": "message", "role": "assistant",
"model": self.server.model_id, "content": content,
"stop_reason": stop_reason, "stop_sequence": None,
"usage": {"input_tokens": stats["prompt_tokens"],
"output_tokens": stats["completion_tokens"]}},
request_id, queue_headers)
return
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Cache-Control", "no-cache")
self.send_header("X-Accel-Buffering", "no")
self.send_header("Connection", "close") # see the OpenAI path: SSE is close-framed
self.close_connection = True
self.send_header("x-request-id", request_id)
for name, value in queue_headers.items():
self.send_header(name, value)
self.send_cors_headers()
self.end_headers()
connected = [True]
write_lock = threading.Lock()
last_write = [time.time()]
ka_stop = threading.Event()
def send_event(name, payload):
if not connected[0]:
return
data = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
with write_lock:
try:
self.wfile.write(f"event: {name}\ndata: {data}\n\n".encode())
self.wfile.flush()
last_write[0] = time.time()
except OSError:
connected[0] = False
# Anthropic has a first-class keepalive event, so the cold prefill (minutes) does
# not need the OpenAI path's reasoning-delta trick: `ping` is in the protocol.
def keepalive():
while not ka_stop.wait(1.0):
if not connected[0]:
return
if time.time() - last_write[0] >= 10.0:
send_event("ping", {"type": "ping"})
send_event("message_start", {"type": "message_start", "message": {
"id": message_id, "type": "message", "role": "assistant",
"model": self.server.model_id, "content": [], "stop_reason": None,
"stop_sequence": None, "usage": {"input_tokens": 0, "output_tokens": 0}}})
text_index = 1 if enable_thinking else 0
stream_state = {"thinking_closed": not enable_thinking,
"text_started": not enable_thinking}
if enable_thinking:
send_event("content_block_start", {"type": "content_block_start", "index": 0,
"content_block": {"type": "thinking", "thinking": "", "signature": ""}})
else:
send_event("content_block_start", {"type": "content_block_start", "index": 0,
"content_block": {"type": "text", "text": ""}})
ka_thread = threading.Thread(target=keepalive, daemon=True)
ka_thread.start()
stream_cleanup.callback(ka_thread.join, timeout=2)
stream_cleanup.callback(ka_stop.set)
raw = []
sideband = ToolSideband(ARCH == "kimi" and bool(tools), stop_sequences,
ignore_leading_stop)
state = {"buf": "", "in_tool": False}
hold = _tool_hold()
def emit_text(chunk):
if not chunk:
return
if not stream_state["text_started"]:
stream_state["text_started"] = True
send_event("content_block_start", {"type": "content_block_start",
"index": text_index, "content_block": {"type": "text", "text": ""}})
send_event("content_block_delta", {"type": "content_block_delta",
"index": text_index, "delta": {"type": "text_delta", "text": chunk}})
def emit_answer(chunk):
if not tools:
emit_text(chunk)
return
if sideband.seen:
emit_text(chunk)
return
if state["in_tool"]:
return # tool markers never reach the client as text
state["buf"] += chunk
cut = _tool_cut(state["buf"])
if cut >= 0:
if cut:
emit_text(state["buf"][:cut])
state["buf"] = ""
state["in_tool"] = True
return
flush = max(0, len(state["buf"]) - hold)
if flush:
emit_text(state["buf"][:flush])
state["buf"] = state["buf"][flush:]
def emit_thinking(chunk):
if not enable_thinking:
return # reasoning the client did not ask for
send_event("content_block_delta", {"type": "content_block_delta", "index": 0,
"delta": {"type": "thinking_delta", "thinking": chunk}})
def close_thinking():
if stream_state["thinking_closed"]:
return
stream_state["thinking_closed"] = True
send_event("content_block_delta", {"type": "content_block_delta", "index": 0,
"delta": {"type": "signature_delta",
"signature": ANTHROPIC_LOCAL_SIGNATURE}})
send_event("content_block_stop", {"type": "content_block_stop", "index": 0})
if ARCH == "inkling":
split = InklingStreamSplit(emit_answer,
emit_thinking if enable_thinking else None,
close_thinking if enable_thinking else None)
else:
# Anche qui: su GLM-5.3 il blocco e' aperto dal prompt, quindi
# lo splitter serve pure col ragionamento "spento", o il
# pensiero finisce incollato davanti alla risposta.
split = (ThinkingStreamSplit(emit_thinking, emit_answer, close_thinking)
if starts_in_reasoning(enable_thinking, add_generation_prompt)
else None)
def on_text(chunk):
raw.append(chunk)
(split.feed if split else emit_answer)(chunk)
stop_filter = StopFilter(stop_sequences, on_text, ignore_leading_stop)
def generation_stopped():
return stop_filter.stopped() or sideband.stopped()
stats = self.server.generate(
prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,
lambda: not connected[0], grammar=grammar, stopped=generation_stopped,
**({"on_tool": sideband.feed} if sideband.enabled else {}),
**({"image": image} if image is not None else {}))
stop_filter.finish()
sideband.finish()
if split:
split.close()
close_thinking() # budget exhaustion before </think>
if tools and not state["in_tool"] and state["buf"]:
emit_text(state["buf"])
ka_stop.set()
ka_thread.join(timeout=2)
if stream_state["text_started"]:
send_event("content_block_stop", {"type": "content_block_stop",
"index": text_index})
content, stop_reason = blocks_and_stop("".join(raw), stats,
sideband.reply())
index = text_index + 1 if stream_state["text_started"] else 1
for block in content:
if block["type"] != "tool_use":
continue # thinking/text blocks were streamed above
send_event("content_block_start", {"type": "content_block_start", "index": index,
"content_block": {"type": "tool_use", "id": block["id"],
"name": block["name"], "input": {}}})
send_event("content_block_delta", {"type": "content_block_delta", "index": index,
"delta": {"type": "input_json_delta",
"partial_json": json.dumps(block["input"], ensure_ascii=False)}})
send_event("content_block_stop", {"type": "content_block_stop", "index": index})
index += 1
send_event("message_delta", {"type": "message_delta",
"delta": {"stop_reason": stop_reason, "stop_sequence": None},
"usage": {"output_tokens": stats["completion_tokens"]}})
send_event("message_stop", {"type": "message_stop"})
# close_connection was already set when the 200 was committed (#597 item 3).
def completion(self, body, request_id):
prompt = body.get("prompt")
if not isinstance(prompt, str):
raise APIError(400, "Colibri currently requires `prompt` to be a string.", "prompt")
if not prompt:
raise APIError(400, "`prompt` must not be empty.", "prompt")
self.generation(body, prompt, request_id, False)
def serve(model, host="127.0.0.1", port=8000, model_id=None, api_key=None,
cap=None, max_tokens=1024, engine=None, env=None, cors_origins=None,
max_queue=8, queue_timeout=300, kv_slots=1, allowed_hosts=(), family=None):
if not 1 <= max_tokens:
raise ValueError("max_tokens must be positive")
if not 1 <= port <= 65535:
raise ValueError("port must be between 1 and 65535")
if max_queue < 0:
raise ValueError("max_queue cannot be negative")
if queue_timeout <= 0:
raise ValueError("queue_timeout must be positive")
if not 1 <= kv_slots <= 16:
raise ValueError("kv_slots must be between 1 and 16")
pending_family = family
pending_model_id = model_id
pending_engine = engine
if host not in ("127.0.0.1", "localhost", "::1") and not api_key:
# (#SEC-6) Fail closed: an unauthenticated engine on a non-loopback bind exposes
# a compute-heavy API to the network. Refuse unless explicitly overridden.
if os.environ.get("COLI_ALLOW_INSECURE_BIND") == "1":
print("WARNING: binding %s beyond localhost with NO auth (COLI_ALLOW_INSECURE_BIND=1)" % host,
file=sys.stderr)
else:
print("refusing to bind %s beyond localhost without COLI_API_KEY set "
"(set COLI_ALLOW_INSECURE_BIND=1 to override)" % host, file=sys.stderr)
sys.exit(1)
if allowed_hosts and "*" in allowed_hosts:
print("WARNING: --allowed-host '*' accepts ANY Host header "
"(DNS-rebinding guard disabled)", file=sys.stderr)
origins = cors_origin_list(cors_origins)
# Bind before starting the 744B engine. A stale/occupied port must fail in
# milliseconds rather than loading hundreds of GB and leaking a child.
server = APIServer((host, port), None, model_id, api_key, max_tokens, origins,
max_queue, queue_timeout, kv_slots, allowed_hosts=allowed_hosts)
runtime = None
previous_sigterm = signal.getsignal(signal.SIGTERM)
try:
family = pending_family or resolve_model(model).descriptor
global ARCH, CHAT_FLAVOR
ARCH = family.id
CHAT_FLAVOR = detect_chat_flavor(family.id, model)
if CHAT_FLAVOR:
print(f"[gateway] {family.id} engine, {CHAT_FLAVOR} chat template "
"(from the checkpoint's chat_template.jinja)", file=sys.stderr)
engine = pending_engine or default_engine(family)
if not pending_model_id:
try:
pending_model_id = registry_default_model_id(resolve_model(model))
except Exception:
pending_model_id = family.default_model_id
model_id = pending_model_id
server.model_id = model_id
try: # what the checkpoint on disk is called (a Clef, a 27B), for the Jev card
server.display_name = display_for(resolve_model(model))[0]
except Exception:
server.display_name = None
if kv_slots > family.limits.max_kv_slots:
raise ValueError(f"{family.id} engine supports at most "
f"{family.limits.max_kv_slots} KV slot(s)")
if family.modality == "image":
# Its own process class: the image engine speaks a line-and-JSON
# protocol with binary frames, not the text engines' byte stream.
runtime = image_engine.ImageEngine(engine, model, env=env)
# Same lifetime rule as the text engines on Windows: a job object
# takes the engine down with this server, however the server ends.
runtime._win_job = _win_kill_on_close_job(getattr(runtime.process, "pid", None))
print(f"[image] {model_id}: default {runtime.info['default_width']}x"
f"{runtime.info['default_height']}, {runtime.info['default_steps']} steps",
file=sys.stderr)
else:
try:
runtime = Engine(engine,model,cap,max_tokens,env,kv_slots,family)
except EngineLoadError as error:
# The engine said why: one line naming the kind, not a traceback
# that ends in "exited unexpectedly".
print(f"[gateway] engine load failed: kind={error.kind} {error.detail}",
file=sys.stderr)
sys.exit(1)
server.engine = runtime
if family.modality != "image":
# Said once at start-up, so a checkpoint that declares a tower it does
# not carry is visible in the log and not only in a client's 400.
print(f"[gateway] input modalities: {', '.join(server.input_modalities())}"
+ (" (the engine loaded no vision tower; a picture gets a 400)"
if family.capabilities.image and runtime.vision is False else ""),
file=sys.stderr)
print(f"OpenAI-compatible API listening on http://{host}:{port}/v1", file=sys.stderr)
signal.signal(signal.SIGTERM, lambda *_: threading.Thread(target=server.shutdown, daemon=True).start())
# On Windows SIGTERM is never delivered (os.kill is TerminateProcess);
# CTRL_BREAK — the one console signal a controller CAN target at this
# process group — arrives as SIGBREAK. Without this handler it kills
# the serve loop outright, skipping the finally that drains the
# engine (stdin EOF -> atexit -> HEAT_FILE save). The engine child
# runs in its own process group (see Engine.__init__) and does not
# receive this event.
if hasattr(signal, "SIGBREAK"):
signal.signal(signal.SIGBREAK,
lambda *_: threading.Thread(target=server.shutdown, daemon=True).start())
try:
server.serve_forever()
except KeyboardInterrupt:
pass
finally:
signal.signal(signal.SIGTERM, previous_sigterm)
server.scheduler.close()
server.server_close()
if runtime is not None:
runtime.close()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model", default=os.environ.get("COLI_MODEL"), required=not os.environ.get("COLI_MODEL"))
parser.add_argument("--engine")
parser.add_argument("--arch", choices=("auto", *family_ids()), default="auto",
help="chat-template family; auto reads model_type from the model's config.json")
parser.add_argument("--host", default="127.0.0.1")
parser.add_argument("--port", type=int, default=8000)
parser.add_argument("--model-id", default=os.environ.get("COLI_MODEL_ID"))
parser.add_argument("--api-key", default=os.environ.get("COLI_API_KEY"))
parser.add_argument("--cors-origin", action="append", default=None,
help="allowed browser origin; repeat as needed (use '*' for any origin). "
"Plain values replace the default list; +ORIGIN adds to it")
# Absent = not explicitly set: mirrors coli's --cap (see cap_for_arch and issue
# #379 -- glm arch resolves platform-aware, non-glm gets the legacy 8). An
# explicit value, 0 included, reaches the engine verbatim.
parser.add_argument("--cap", type=int, default=None, help="cache slots/layer (default: auto)")
parser.add_argument("--max-tokens", type=int, default=1024)
parser.add_argument("--max-queue", type=int, default=int(os.environ.get("COLI_MAX_QUEUE", "8")))
parser.add_argument("--queue-timeout", type=float,
default=float(os.environ.get("COLI_QUEUE_TIMEOUT", "300")))
parser.add_argument("--kv-slots", type=int, default=int(os.environ.get("COLI_KV_SLOTS", "1")))
parser.add_argument("--allowed-host", action="append",
default=[h.strip() for h in os.environ.get("COLI_ALLOWED_HOSTS", "").split(",") if h.strip()],
help="additional Host header value accepted by the DNS-rebinding guard "
"(reverse proxy / MagicDNS in front of the loopback bind); repeat as needed, "
"or set COLI_ALLOWED_HOSTS as a comma-separated list")
args = parser.parse_args()
try:
resolved = resolve_model(args.model)
except (FamilyConfigError, UnknownFamilyError) as error:
parser.error(str(error))
family = resolved.descriptor
if args.arch != "auto" and args.arch != family.id:
parser.error(f"--arch {args.arch} conflicts with model family {family.id}")
global ARCH
ARCH = family.id
if args.engine is None:
args.engine = str(default_engine(family))
if args.model_id is None:
args.model_id = registry_default_model_id(resolved)
serve(args.model, args.host, args.port, args.model_id, args.api_key,
args.cap,args.max_tokens,args.engine,cors_origins=args.cors_origin,
max_queue=args.max_queue,queue_timeout=args.queue_timeout,kv_slots=args.kv_slots,
allowed_hosts=args.allowed_host,family=family)
if __name__ == "__main__":
main()