Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
import http.client
import io
import json
import math
import os
import socket
import tempfile
import threading
import sys
import time
import struct
import unittest
from unittest.mock import patch
from urllib.error import HTTPError
from urllib.request import Request, urlopen
from pathlib import Path
import openai_server
from family_registry import family_by_id, family_ids
from openai_server import (APIError, engine_exit_status, APIHandler, APIServer, ClientCancelled,
CONTINUATION_FAMILIES, _marker_cuts,
DEFAULT_CHAT_STOP_SEQUENCES, END, GenerationScheduler,
LOGPROBS_TOP_K_CAP, logprobs_options,
READY, Engine, InklingStreamSplit, StopFilter, ThinkingStreamSplit,
_engine_error, _image_bytes_from_url, cap_for_arch,
conversation_cache_slot, model_arch,
generation_options, parse_tool_calls, parse_dsv4_tool_calls,
parse_arch_tool_calls, parse_k3_tool_calls, parse_qwen38_tool_calls,
parse_qwen_tool_calls,
read_engine_turn, render_chat, render_chat_qwen, render_chat_for_arch,
render_chat_glm53, render_chat_inkling, render_chat_kimi,
render_chat_olmoe, render_chat_qwen,
render_chat_qwen38, render_chat_v4, render_chat_dsv41,
render_chat_mimo,
_dsv4_tool_calls, serve,
resolve_generation_prompt, split_thinking_reply,
detect_chat_flavor, qwen36_has_vision,
split_thinking_reply_spans, starts_in_reasoning,
parse_tool_calls_spans, parse_arch_tool_calls_spans,
THINK_OPEN, THINK_CLOSE,
_compose_span_maps, _cut_span_map, _project_span,
_keepalive_choice,
stop_policy, tune_child_env,
TOOL_CHOICE_REQUIRED_INSTRUCTION)
def echo_record(pos, data, lp, topk):
"""One engine ECHO record shaped exactly as Engine._dispatch_stdout builds it:
`pos`/`logprob`/`text`, the three keys the closed-set scorer reads with its own
None-for-nan convention, plus `bytes`/`lp`/`topk`, the payload and numeric tail the
OpenAI logprobs surface reads. Every fixture builds its echo records through this one
helper, so none can drift from the shape the dispatcher emits."""
return {"pos": pos,
"logprob": None if math.isnan(lp) else lp,
"text": data.decode("utf-8", "replace"),
"bytes": data,
"lp": lp,
"topk": topk}
def _spawn_test_server(case, engine, kv_slots=1, max_tokens=16):
"""A throwaway APIServer on an ephemeral port, torn down with the test case."""
server = APIServer(("127.0.0.1", 0), engine, "test-model", "secret", max_tokens,
kv_slots=kv_slots)
thread = threading.Thread(target=server.serve_forever, args=(0.01,), daemon=True)
thread.start()
# LIFO: the join is registered first so it runs last. Joining a still-serving thread
# before shutdown() burns the whole timeout on every test that does it.
case.addCleanup(thread.join, timeout=2)
case.addCleanup(server.server_close)
case.addCleanup(server.shutdown)
case.addCleanup(server.scheduler.close)
return f"http://127.0.0.1:{server.server_port}"
def _post_json(base, path, body):
return urlopen(Request(base + path, data=json.dumps(body).encode(),
headers={"Authorization": "Bearer secret",
"Content-Type": "application/json"}), timeout=5)
def _post_completions(base, body):
return _post_json(base, "/v1/completions", body)
def _post_chat(base, body):
return _post_json(base, "/v1/chat/completions", body)
def _error_body(case, call):
"""The status and the parsed `error` object a refused request puts on the wire. Read
from the RESPONSE BODY, never from an exception object: a named exception that never
reaches the client is the defect, not the fix."""
with case.assertRaises(HTTPError) as caught:
call()
raw = caught.exception.read()
caught.exception.close()
return caught.exception.code, json.loads(raw)["error"]
class ScriptedEngine:
"""An engine double driven entirely by data: the text chunks it feeds to `on_text`, the
per-token numeric records it reports, and the prompt-echo frames it sends.
Every alignment fixture below is one of these with different data, so the boundary a
test describes lives in the test rather than in a subclass of its own."""
def __init__(self, chunks=("ok",), records="derive", echoes=(), echo_base=0,
prompt_tokens=2, length_limited=False, supports_logprobs_echo=True,
supports_tok_ids=True):
self.chunks = tuple(chunks)
# "derive" = one record per chunk, carrying that chunk's own bytes. None = the
# engine sends no numeric channel at all. A list = exactly those (bytes, lp, topk)
# triples, which is how a fixture makes the raw stream and the record bytes
# legitimately disagree.
self.records = records
self.echoes = tuple(echoes)
self.echo_base = echo_base # the first wire `pos`, above 0 for a pin
self.prompt_tokens = prompt_tokens
self.length_limited = length_limited
self.supports_logprobs_echo = supports_logprobs_echo
self.supports_tok_ids = supports_tok_ids
self.calls = []
self.stop_requests = 0
self.last_logprobs = 0
self.last_echo = False
self.last_gbytes = False
def generate(self, prompt, maximum, temperature, top_p, on_text, cache_slot=0,
cancelled=None, grammar=None, stopped=None, on_accept=None, audio=None,
on_tool=None, image=None, logprobs=0, pin=False, on_echo=None,
gbytes_before_ext=False):
self.calls.append((prompt, maximum, temperature, top_p, cache_slot, grammar))
self.last_logprobs = logprobs
self.last_echo = on_echo is not None
self.last_gbytes = gbytes_before_ext
if on_accept is not None:
on_accept({"prompt_tokens": self.prompt_tokens})
if logprobs and on_echo is not None:
for offset, (data, lp, topk) in enumerate(self.echoes):
on_echo(echo_record(self.echo_base + offset, data, lp, topk))
for chunk in self.chunks:
on_text(chunk)
if stopped and stopped():
self.stop_requests += 1
break
stats = {"prompt_tokens": self.prompt_tokens,
"completion_tokens": len(self.chunks),
"length_limited": self.length_limited}
if logprobs and self.records is not None:
triples = ([(chunk.encode("utf-8"), -0.1, [(1, -0.1)]) for chunk in self.chunks]
if self.records == "derive" else self.records)
stats["logprobs"] = {"generated": [(data, {"lp": lp, "topk": topk})
for data, lp, topk in triples]}
return stats
NAN = float("nan")
# Three generated frames whose first is the GLM role marker the default chat stop policy
# swallows. `raw_text` is "Hello world", and the
# marker's record describes characters the client never received while the other two
# describe characters it did.
def _ignored_leading_marker_engine():
return ScriptedEngine(
chunks=("<|user|>", "Hello", " world"),
records=[(b"<|user|>", -0.1, [(0, -0.1)]),
(b"Hello", -0.2, [(1, -0.2)]),
(b" world", -0.30000000000000004, [(2, -0.30000000000000004)])],
echoes=[(b"h", NAN, []), (b"i", -0.1, [(1, -0.1)])])
# One frame decoding to "Hello" against a `stop` of "lo". The filter emits "Hel" and matches, so the frame's record describes five characters
# of which the client received three. The prompt echo is "pr" + "ompt".
def _mid_token_stop_echo_engine():
return ScriptedEngine(
chunks=("Hello",),
records=[(b"Hello", -0.5, [(5, -0.5)])],
echoes=[(b"pr", NAN, []), (b"ompt", -0.1, [(1, -0.1)])])
# A matched stop sequence withholds its own text and everything after it: neither the
# "STOP" record nor the " more" record after it may survive into the response.
def _stop_token_engine():
return ScriptedEngine(
chunks=("ok ", "STOP", " more"),
records=[(b"ok ", -0.1, [(1, -0.1)]),
(b"STOP", -0.2, [(2, -0.2)]),
(b" more", -0.3, [(3, -0.3)])],
echoes=[(b"h", NAN, []), (b"i", -0.1, [(1, -0.1)])])
# The Euro sign split 1+2 across the prompt/generated seam -- lead byte on the last echo
# frame -- with a later frame cut by a matched `stop`. A decoder with no leading-byte
# context reads b"\x82\xac" as two replacement characters, and that is the raw stream the
# stop filter sees.
def _seam_split_stop_engine():
return ScriptedEngine(
chunks=("��", "xSTOPy"),
records=[(b"\x82\xac", -0.2, [(2, -0.2)]),
(b"xSTOPy", -0.3, [(3, -0.3)])],
echoes=[(b"A", NAN, []), (b"\xe2", -0.1, [(1, -0.1)])])
# The seam and the stop on the SAME record: one frame b"\x82\xacHello" whose first two
# bytes complete a Euro sign whose lead byte rode the last prompt frame, cut after "Hel".
def _seam_split_same_record_stop_engine():
return ScriptedEngine(
chunks=("��Hello",),
records=[(b"\x82\xacHello", -0.2, [(2, -0.2)])],
echoes=[(b"A", NAN, []), (b"\xe2", -0.1, [(1, -0.1)])])
# The Euro sign split 1+1+1 across three frames: the first generated frame completes
# nothing, so its own decoding is "" while the raw stream reads it as a replacement
# character.
def _seam_split_across_two_frames_engine():
return ScriptedEngine(
chunks=("�", "�Hello"),
records=[(b"\x82", -0.2, [(2, -0.2)]),
(b"\xacHello", -0.3, [(3, -0.3)])],
echoes=[(b"A", NAN, []), (b"\xe2", -0.1, [(1, -0.1)])])
# A frame that decodes to nothing, sitting exactly on the boundary of the emitted region,
# whose bytes go into the `stop` sequence itself: "ok " occupies raw [0, 3), and the next
# two frames are the halves of "é", which is the stop, so the map ends at 3.
def _suppressed_stop_byte_engine():
return ScriptedEngine(
chunks=("ok ", "é"),
records=[(b"ok ", -0.1, [(1, -0.1)]),
(b"\xc3", -0.3, [(3, -0.3)]),
(b"\xa9", -0.4, [(4, -0.4)])])
# Three generated chunks with a numeric tail for only `covered` of them. The real engine
# gives an opted-in request a tail on every token, but a dropped frame or a different build
# can do this, and the echo path rebuilds `text` from the records.
def _partial_coverage_engine(covered, chunks=("AA", "BB", "CC"), record_bytes=None,
echo_bytes=(b"P", b"Q")):
payloads = record_bytes or tuple(c.encode("utf-8") for c in chunks)
return ScriptedEngine(
chunks=chunks,
records=[(payload, -0.2, [(9, -0.2)]) for payload in payloads[:covered]],
echoes=[(piece, NAN if pos == 0 else -0.1, [] if pos == 0 else [(1, -0.1)])
for pos, piece in enumerate(echo_bytes)])
# One generated chunk whose top-k table repeats a candidate id. `duplicate=False` is the
# control: the same engine, the same shape, distinct ids.
def _duplicate_candidate_engine(duplicate=True):
return ScriptedEngine(
chunks=("cat",),
records=[(b"cat", -0.5,
[(7, -0.1), (7, -0.2)] if duplicate else [(7, -0.1), (8, -0.2)])])
# A pin snapshot covers the first prompt token, so the prefill reads out positions 1..n-1
# and there is nothing to read out before that.
def _pinned_prefix_engine():
return ScriptedEngine(chunks=("ok",), echo_base=1, prompt_tokens=3,
echoes=[(b"b", -0.1, [(1, -0.1)]), (b"c", -0.2, [(2, -0.2)])])
class FakeEngine:
# Both capability gates, on: tests run under the module's default ARCH="glm", where
# Engine.__init__ sets both from the same arch check. The gate tests set them
# explicitly rather than relying on this.
supports_logprobs_echo = True
supports_tok_ids = True
def __init__(self):
self.calls = []
self.stop_requests = 0
self.last_logprobs = 0
self.last_echo = False
self.last_gbytes = False
def generate(self, prompt, maximum, temperature, top_p, on_text, cache_slot=0,
cancelled=None, grammar=None, stopped=None, on_accept=None, logprobs=0,
pin=False, on_echo=None, gbytes_before_ext=False):
self.calls.append((prompt, maximum, temperature, top_p, cache_slot, grammar))
self.last_logprobs = logprobs
self.last_echo = on_echo is not None
self.last_gbytes = gbytes_before_ext
if on_accept is not None: # simulate the engine's ACCEPT frame (#597)
on_accept({"prompt_tokens": 7})
if logprobs and on_echo is not None:
for record in self.echo_records(logprobs):
on_echo(record)
for chunk in ("Hé", "llo"):
on_text(chunk)
if stopped and stopped():
self.stop_requests += 1
break
stats = {"prompt_tokens": 7, "completion_tokens": 2, "length_limited": False}
if logprobs:
stats["logprobs"] = self.logprobs_channel(logprobs)
return stats
def echo_records(self, engine_k):
"""A canned prefill read-out, delivered through `on_echo` exactly as
Engine.generate() delivers the real thing. Position 0 carries the engine's own
"nothing to condition on" sentinel. The tail values are non-dyadic so a fixture
rendered at one precision is distinguishable from one rendered at another."""
k = min(engine_k, 2)
return [echo_record(0, b"H", float("nan"), []),
echo_record(1, b"\xc3\xa9", -0.3, [(72, -0.3), (100, -1.7)][:k])]
def logprobs_channel(self, engine_k):
"""Canned generated-token records, shaped exactly like Engine.generate()'s return
value. Each token's own lp is bit-identical to one of its own top-k entries, which
is the engine's logprob_tail invariant. The records cover, byte for byte, the text
generate() feeds to on_text: the real engine emits a tail for every token of an
opted-in request, so a double that emits text it has no records for would model
nothing the engine does."""
k = min(engine_k, 2)
return {"generated": [
(b"H", {"lp": -0.2, "topk": [(72, -0.2), (200, -2.4)][:k]}),
(b"\xc3\xa9", {"lp": -0.4, "topk": [(101, -0.4), (300, -3.6)][:k]}),
(b"llo", {"lp": -0.6, "topk": [(400, -0.6), (500, -5.2)][:k]})]}
class BlockingEngine(FakeEngine):
def __init__(self):
super().__init__()
self.entered = threading.Event()
self.release = threading.Event()
def generate(self, prompt, maximum, temperature, top_p, on_text, cache_slot=0,
cancelled=None, grammar=None, stopped=None, on_accept=None, logprobs=0,
pin=False, on_echo=None, gbytes_before_ext=False):
self.entered.set()
self.release.wait(2)
return super().generate(prompt, maximum, temperature, top_p, on_text, cache_slot,
cancelled, grammar, stopped, on_accept, logprobs, pin,
on_echo, gbytes_before_ext)
class TemplateTest(unittest.TestCase):
def test_renders_text_subset_of_official_template(self):
prompt = render_chat([
{"role": "system", "content": "System"},
{"role": "developer", "content": "Developer"},
{"role": "user", "content": [{"type": "text", "text": "Hi"}]},
{"role": "assistant", "content": " Hello "},
{"role": "user", "content": "Again"},
])
self.assertEqual(
prompt,
"[gMASK]<sop><|system|>System<|system|>Developer<|user|>Hi"
"<|assistant|><think></think>Hello<|user|>Again"
"<|assistant|><think></think>",
)
def test_rejects_non_text_content(self):
with self.assertRaisesRegex(APIError, "text message content only"):
render_chat([{"role": "user", "content": [
{"type": "image_url", "image_url": {"url": "x"}}
]}])
def test_renders_thinking_prefix(self):
self.assertEqual(
render_chat([{"role": "user", "content": "Hi"}], True, "high"),
"[gMASK]<sop><|system|>Reasoning Effort: High<|user|>Hi<|assistant|><think>",
)
def test_qwen38_defaults_to_xhigh_instruction_and_thinking(self):
prompt = render_chat_qwen38([{"role": "user", "content": "Hi"}])
self.assertEqual(
prompt,
"<|im_start|>system\n"
"Reasoning effort is set to xhigh. Please think carefully through the task, "
"validate key assumptions, consider plausible alternatives, and prioritize "
"correctness, consistency, and clarity in the final answer."
"<|im_end|>\n<|im_start|>user\nHi<|im_end|>\n"
"<|im_start|>assistant\n<think>\n",
)
def test_qwen38_reasoning_efforts_and_disabled_thinking(self):
medium = render_chat_qwen38([{"role": "user", "content": "Hi"}],
reasoning_effort="medium")
self.assertEqual(medium,
"<|im_start|>user\nHi<|im_end|>\n"
"<|im_start|>assistant\n<think>\n")
low = render_chat_qwen38([{"role": "user", "content": "Hi"}],
reasoning_effort="low")
self.assertIn("Reasoning effort is set to low.", low)
high = render_chat_qwen38([{"role": "user", "content": "Hi"}],
reasoning_effort="high")
self.assertIn("Reasoning effort is set to xhigh.", high)
disabled = render_chat_qwen38([{"role": "user", "content": "Hi"}],
enable_thinking=False)
self.assertEqual(disabled,
"<|im_start|>user\nHi<|im_end|>\n"
"<|im_start|>assistant\n<think>\n\n</think>\n\n")
def test_qwen38_still_rejects_non_text_content(self):
# Tools are wired up now; images are not. The engine is text-only, so a
# picture must still be refused rather than silently dropped.
with self.assertRaisesRegex(APIError, "text message content only"):
render_chat_qwen38([{"role": "user", "content": [
{"type": "image_url", "image_url": {"url": "x"}}
]}])
def test_qwen38_renders_and_parses_its_own_tool_format(self):
tool = {"type": "function", "function": {
"name": "weather", "description": "w",
"parameters": {"type": "object",
"properties": {"city": {"type": "string"},
"days": {"type": "integer"}}}}}
prompt = render_chat_qwen38([{"role": "user", "content": "Rome?"}], tools=[tool])
# The declaration teaches the model the syntax it must emit, so the
# preamble is transcribed from chat_template.jinja and not paraphrased.
self.assertIn("# Tools\n\nYou have access to the following functions:\n\n<tools>",
prompt)
self.assertIn("<function=example_function_name>", prompt)
self.assertIn("</tools>", prompt)
# A call with no preceding text attaches directly; one with text is
# separated by a blank line. Getting that wrong changes the prompt.
with_text = render_chat_qwen38([
{"role": "user", "content": "Rome?"},
{"role": "assistant", "content": "Checking.", "tool_calls": [
{"type": "function", "function": {
"name": "weather", "arguments": {"city": "Rome"}}}]},
{"role": "tool", "content": "clear"},
{"role": "user", "content": "thanks"},
], tools=[tool])
self.assertIn("Checking.\n\n<tool_call>\n<function=weather>\n"
"<parameter=city>\nRome\n</parameter>\n</function>\n</tool_call>",
with_text)
# Consecutive tool results share one user turn.
self.assertIn("<|im_start|>user\n<tool_response>\nclear\n</tool_response><|im_end|>",
with_text)
text, calls = parse_qwen_tool_calls(
"Sure.\n\n<tool_call>\n<function=weather>\n<parameter=city>\nRome\n"
"</parameter>\n<parameter=days>\n3\n</parameter>\n</function>\n</tool_call>",
[tool])
self.assertEqual(text, "Sure.")
self.assertEqual(len(calls), 1)
self.assertEqual(calls[0]["function"]["name"], "weather")
# The template writes a string argument unquoted, so the type comes back
# from the declared schema: city stays a string, days becomes an int.
self.assertEqual(json.loads(calls[0]["function"]["arguments"]),
{"city": "Rome", "days": 3})
def test_qwen38_tool_choice_none_suppresses_the_declaration(self):
tool = {"type": "function", "function": {"name": "f", "description": "d"}}
prompt = render_chat_qwen38([{"role": "user", "content": "Hi"}],
tools=[tool], tool_choice="none")
self.assertNotIn("<tools>", prompt)
def test_kimi_payload_preserves_utf8_lengths_and_turns(self):
prompt = render_chat_kimi([
{"role": "system", "content": "Be precise."},
{"role": "user", "content": "你好\nKimi"},
{"role": "assistant", "content": "你好。"},
{"role": "user", "content": "Continue"},
], enable_thinking=True)
self.assertEqual(
prompt,
"K3CHAT1\n"
"M system 11\nBe precise."
"M user 11\n你好\nKimi"
"A 0 9\n你好。"
"M user 8\nContinue"
"G 1\n",
)
def test_qwen36_history_is_what_the_engine_was_fed(self):
"""#1759: prefix reuse on qwen36 is all or nothing, because nothing rewinds the
DeltaNet state, so it engages only if the next turn's prompt begins with exactly
the text the engine read: the previous prompt plus what it generated. The
template's preserve_thinking gives a past turn the <think> block it was generated
after; its default strips the block, and the history diverges at the first
assistant turn of every conversation."""
first = [{"role": "user", "content": "Capital of France?"}]
follow = {"role": "user", "content": "And of Italy?"}
# Thinking off: the generation followed the pre-closed header.
fed = render_chat_qwen(first, enable_thinking=False) + "Paris."
history = first + [{"role": "assistant", "content": "Paris."}, follow]
self.assertTrue(render_chat_qwen(history, enable_thinking=False,
preserve_thinking=True).startswith(fed))
self.assertFalse(render_chat_qwen(history, enable_thinking=False).startswith(fed))
# Thinking on: the history matches only if the client sends the reasoning back,
# as reasoning_content or inside the content, the way the model wrote it.
fed = (render_chat_qwen(first, enable_thinking=True)
+ "Easy one.\n</think>\n\nParis.")
for past in ({"role": "assistant", "content": "Paris.",
"reasoning_content": "Easy one."},
{"role": "assistant",
"content": "<think>\nEasy one.\n</think>\n\nParis."}):
with self.subTest(past=past):
prompt = render_chat_qwen(first + [past, follow], enable_thinking=True,
preserve_thinking=True)
self.assertTrue(prompt.startswith(fed))
# Without preserve_thinking the block leaves the history either way.
self.assertIn("<|im_start|>assistant\nParis.<|im_end|>",
render_chat_qwen(first + [past, follow], enable_thinking=True))
def test_qwen36_images_follow_the_container(self):
"""#1757: a qwen36 container converted with its vision tower says so in
qwen36_meta.json, and only then does the gateway turn image parts into
patches for the engine; an older or text-only container keeps refusing them."""
import tempfile
with tempfile.TemporaryDirectory() as d:
self.assertFalse(qwen36_has_vision(d))
meta = Path(d) / "qwen36_meta.json"
meta.write_text(json.dumps({"num_experts": 0}), encoding="utf-8")
self.assertFalse(qwen36_has_vision(d))
meta.write_text(json.dumps({"vision": {"depth": 27}}), encoding="utf-8")
self.assertTrue(qwen36_has_vision(d))
meta.write_text("not json", encoding="utf-8")
self.assertFalse(qwen36_has_vision(d))
self.assertFalse(qwen36_has_vision(None))
def test_qwen38_images_arrive_as_data_uri_bytes(self):
"""A data: URI reaches the preprocessor as bytes. Image.open read them as a
file name and every image request died with "embedded null byte" (500):
the preprocessor's own test hands it a PIL image, so only the gateway path
broke. Found running Qwen3.8-27B with its tower (#1757)."""
try:
import base64, io, tempfile
import numpy
from PIL import Image
except ImportError as missing:
self.skipTest(f"needs Pillow and numpy ({missing})")
from openai_server import expand_qwen38_images
buffer = io.BytesIO()
Image.fromarray((numpy.arange(64 * 96 * 3) % 251).astype("uint8").reshape(64, 96, 3)).save(buffer, "PNG")
uri = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
with tempfile.TemporaryDirectory() as d:
(Path(d) / "preprocessor_config.json").write_text(json.dumps(
{"patch_size": 16, "merge_size": 2, "temporal_patch_size": 2}), encoding="utf-8")
messages, images = expand_qwen38_images(
[{"role": "user", "content": [{"type": "text", "text": "what is this?"},
{"type": "image_url", "image_url": {"url": uri}}]}], d)
self.assertEqual(len(images), 1)
patches, grid_h, grid_w = images[0]
self.assertEqual(patches.shape[0], grid_h * grid_w)
self.assertIn("what is this?", messages[0]["content"])
def test_an_image_url_that_is_not_an_object_is_a_client_error(self):
"""{"type": "image_url", "image_url": "data:..."} (or a number) made the image
expanders call .get("url") on a string, an AttributeError that do_POST answered
with 500 on GLM-5.3, Qwen3.8 and DeepSeek V4.1. It is the client's error: 400."""
from openai_server import expand_dsv41_images, expand_glm53_images, expand_qwen38_images
for expand in (expand_glm53_images, expand_qwen38_images, expand_dsv41_images):
for value in ("data:image/png;base64,AAAA", 5, ["x"]):
with self.subTest(expand=expand.__name__, value=value):
with self.assertRaises(APIError) as caught:
expand([{"role": "user", "content": [
{"type": "text", "text": "what is this?"},
{"type": "image_url", "image_url": value}]}], None)
self.assertEqual(caught.exception.status, 400)
self.assertIn("`image_url` must be an object", str(caught.exception))
def test_qwen38_template_on_the_qwen36_engine(self):
"""#1757: Qwen3.8-27B is a dense model of Qwen3.5's architecture, so the qwen36
engine runs it, but it ships Qwen3.8's chat_template.jinja. The gateway renders
the template the checkpoint carries, not its engine family's."""
import tempfile
with tempfile.TemporaryDirectory() as d:
template = Path(d) / "chat_template.jinja"
template.write_text("{%- set resolved_reasoning_effort = "
"reasoning_effort|default('xhigh') %}", encoding="utf-8")
self.assertEqual(detect_chat_flavor("qwen36", d), "qwen38")
self.assertIsNone(detect_chat_flavor("glm53", d))
template.write_text("{%- if enable_thinking is defined %}", encoding="utf-8")
self.assertIsNone(detect_chat_flavor("qwen36", d))
template.unlink()
self.assertIsNone(detect_chat_flavor("qwen36", d))
history = [{"role": "user", "content": "Capital of France?"}]
with patch("openai_server.ARCH", "qwen36"), patch("openai_server.CHAT_FLAVOR", "qwen38"):
for thinking in (True, False):
self.assertEqual(render_chat_for_arch(history, thinking, "xhigh"),
render_chat_qwen38(history, thinking, "xhigh"))
with patch("openai_server.ARCH", "qwen36"):
self.assertNotEqual(render_chat_for_arch(history, True, "xhigh"),
render_chat_qwen38(history, True, "xhigh"))
def test_kimi_renders_tool_declaration_and_choice(self):
tools = [{"type": "function", "function": {
"name": "get_weather", "parameters": {"type": "object"}}}]
body = ("# Tools\nHere are the available tools, described in JSONSchema.\n\n"
"```json\n" + json.dumps(tools, ensure_ascii=False, separators=(",", ":"),
sort_keys=True) + "\n```")
prompt = render_chat_kimi([{"role": "user", "content": "Hi"}], tools=tools)
self.assertEqual(prompt, "K3CHAT1\n"
f"Y 12 {len(body.encode('utf-8'))}\ntool-declare{body}"
"M user 2\nHi"
"G 0\n")
# tool_choice=none: the tools are not offered at all
self.assertEqual(render_chat_kimi([{"role": "user", "content": "Hi"}],
tools=tools, tool_choice="none"),
"K3CHAT1\nM user 2\nHiG 0\n")
# tool_choice=required appends the reference's tool-choice system message
prompt = render_chat_kimi([{"role": "user", "content": "Hi"}],
tools=tools, tool_choice="required")
self.assertIn("\ntool-choiceThe system is invoked with `tool_choice=required`.", prompt)
self.assertTrue(prompt.endswith("G 0\n"))
def test_kimi_renders_tool_calls_and_results(self):
messages = [
{"role": "user", "content": "Weather in Rome?"},
{"role": "assistant", "content": "", "tool_calls": [
{"id": "call_1", "type": "function", "function": {
"name": "get_weather",
"arguments": '{"city": "Rome", "days": 1e2, "metric": true}'}}]},
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
]
prompt = render_chat_kimi(messages)
# B: assistant turn carrying the call; V records keep the exact JSON
# literal for non-strings (1e2 stays 1e2) and decode strings.
self.assertIn("B 0 0 0 1\n", prompt)
self.assertIn("F 11 3\nget_weather", prompt)
self.assertIn("V 4 6 4\ncitystringRome", prompt)
self.assertIn("V 4 6 3\ndaysnumber1e2", prompt)
self.assertIn("V 6 7 4\nmetricbooleantrue", prompt)
# O: the result resolves its name through tool_call_id
self.assertIn("O 1 11 5\nget_weathersunny", prompt)
def test_kimi_tool_results_resort_by_call_id(self):
messages = [
{"role": "assistant", "content": "", "tool_calls": [
{"id": "a", "type": "function",
"function": {"name": "first", "arguments": "{}"}},
{"id": "b", "type": "function",
"function": {"name": "second", "arguments": "{}"}}]},
{"role": "tool", "tool_call_id": "b", "content": "B"},
{"role": "tool", "tool_call_id": "a", "content": "A"},
]
prompt = render_chat_kimi(messages)
# Results are re-sorted into tool_calls order, names resolved from ids.
self.assertIn("O 1 5 1\nfirstA", prompt)
self.assertIn("O 2 6 1\nsecondB", prompt)
self.assertLess(prompt.index("O 1 5 1"), prompt.index("O 2 6 1"))
def test_kimi_json_fallback_for_unparseable_arguments(self):
prompt = render_chat_kimi([
{"role": "assistant", "content": "", "tool_calls": [
{"id": "x", "type": "function", "function": {
"name": "fn", "arguments": "not json"}}]}])
self.assertIn("J 2 8\nfnnot json", prompt)
def test_glm_renders_a_tool_call_whose_arguments_are_not_an_object(self):
"""`arguments` that parses but is not an object must not kill the request.
Both GLM renderers already tolerate `arguments` that does not parse at
all -- the except branch sets {} -- and every sibling renderer (Kimi
above, Qwen3.8, DeepSeek V4/V4.1) renders the call without arguments
rather than failing. Only the "parsed, but not an object" case reached
.items(), raised AttributeError, and came back as HTTP 500 "The colibri
engine failed to process the request." on a request the engine never
saw.
"""
import openai_server as srv
for arguments in ('[1, 2]', '"text"', '5', [1, 2], 7):
with self.subTest(arguments=arguments):
messages = [
{"role": "user", "content": "run it"},
{"role": "assistant", "content": "", "tool_calls": [
{"id": "x", "type": "function",
"function": {"name": "fn", "arguments": arguments}}]},
{"role": "tool", "content": "done"},
]
glm = render_chat(list(messages))
self.assertIn("fn", glm)
self.assertNotIn("<arg_key>", glm)
glm53 = srv.render_chat_glm53(list(messages))
self.assertIn("<tool_call>fn", glm53)
self.assertNotIn("<arg_key>", glm53)
# An object still renders its arguments, on both renderers.
renders = [{"role": "assistant", "content": "", "tool_calls": [
{"id": "x", "type": "function",
"function": {"name": "fn", "arguments": '{"city": "Rome"}'}}]}]
self.assertIn("<arg_key>city</arg_key><arg_value>Rome</arg_value>",
render_chat(list(renders)))
self.assertIn("<arg_key>city</arg_key><arg_value>Rome</arg_value>",
srv.render_chat_glm53(list(renders)))
def test_kimi_still_rejects_unknown_roles(self):
with self.assertRaisesRegex(APIError, "Unsupported role"):
render_chat_kimi([{"role": "critic", "content": "hm"}])
with self.assertRaisesRegex(APIError, "resolvable tool name"):
render_chat_kimi([{"role": "tool", "content": "orphan result"}])
def test_kimi_parses_generated_tool_calls(self):
reply = ('Sure.<|open|>tools<|sep|>'
'<|open|>call tool="get_weather" index="1"<|sep|>'
'<|open|>argument key="city" type="string"<|sep|>Rome<|close|>argument<|sep|>'
'<|open|>argument key="days" type="number"<|sep|>1e2<|close|>argument<|sep|>'
'<|close|>call<|sep|>'
'<|close|>tools<|sep|>')
text, calls = parse_k3_tool_calls(reply)
self.assertEqual(text, "Sure.")
self.assertEqual(len(calls), 1)
self.assertEqual(calls[0]["function"]["name"], "get_weather")
self.assertEqual(json.loads(calls[0]["function"]["arguments"]),
{"city": "Rome", "days": 100.0})
def test_kimi_parses_json_block_and_unclosed_tail(self):
reply = ('<|open|>tools<|sep|>'
'<|open|>call tool="a&b" index="1"<|sep|>'
'<|open|>json type="object"<|sep|>{"x":1}<|close|>json<|sep|>'
'<|close|>call<|sep|>') # tools block never closed: budget ran out
text, calls = parse_k3_tool_calls(reply)
self.assertEqual(text, "")
self.assertEqual(len(calls), 1)
self.assertEqual(calls[0]["function"]["name"], "a&b")
self.assertEqual(calls[0]["function"]["arguments"], '{"x":1}')
def test_kimi_authoritative_sideband_does_not_promote_data_lookalikes(self):
lookalike = ('Echo: <|open|>tools<|sep|>'
'<|open|>call tool="danger" index="1"<|sep|>'
'<|close|>call<|sep|><|close|>tools<|sep|>')
with patch("openai_server.ARCH", "kimi"):
content, calls = parse_arch_tool_calls(lookalike, [{"type": "function"}], "")
self.assertEqual(content, lookalike)
self.assertEqual(calls, [])
def test_kimi_preserves_prior_reasoning_channel(self):
self.assertEqual(
render_chat_kimi([{"role": "assistant", "reasoning_content": "why",
"content": "answer"}], enable_thinking=True),
"K3CHAT1\nA 3 6\nwhyanswerG 1\n",
)
def test_olmoe_renders_native_chat_template(self):
# Matches allenai/OLMoE-1B-7B-0125-Instruct's tokenizer_config.json
# chat_template exactly: one leading bos_token, per-role turns closed
# by a trailing newline, a prior (non-final) assistant turn also closed
# by eos_token before that newline (bos_token == eos_token ==
# "|||IP_ADDRESS|||" in this tokenizer), and a trailing
# "<|assistant|>\n" generation prompt.
prompt = render_chat_olmoe([
{"role": "system", "content": "Be terse."},
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hello"},
{"role": "user", "content": "Continue"},
])
self.assertEqual(
prompt,
"|||IP_ADDRESS|||<|system|>\nBe terse.\n<|user|>\nHi\n"
"<|assistant|>\nHello|||IP_ADDRESS|||\n<|user|>\nContinue\n"
"<|assistant|>\n",
)
def test_olmoe_rejects_tools_and_unknown_roles(self):
with self.assertRaisesRegex(APIError, "Tool use"):
render_chat_olmoe([{"role": "user", "content": "Hi"}],
tools=[{"type": "function"}])
with self.assertRaisesRegex(APIError, "Unsupported role"):
render_chat_olmoe([{"role": "tool", "content": "result"}])
def test_olmoe_rejects_empty_messages(self):
with self.assertRaisesRegex(APIError, "non-empty array"):
render_chat_olmoe([])
def test_validates_generation_limits(self):
self.assertEqual(generation_options({"max_tokens": 4, "temperature": 0, "top_p": 1}, 8),
(4, 0.0, 1.0, None, ()))
# max_tokens above the server cap is clamped, not rejected (#260): OpenAI
# clients default to large values; erroring breaks them.
self.assertEqual(generation_options({"max_tokens": 9, "temperature": 0, "top_p": 1}, 8),
(8, 0.0, 1.0, None, ()))
# non-positive / non-int max_tokens is still a hard error
with self.assertRaises(APIError):
generation_options({"max_tokens": 0}, 8)
with self.assertRaises(APIError):
generation_options({"temperature": math.nan}, 8)
with self.assertRaises(APIError):
generation_options({"top_p": math.inf}, 8)
self.assertEqual(generation_options({"temperature": None, "top_p": None}, 8),
(8, 0.7, 0.9, None, ()))
# response_format -> grammar plumbing (draft source, never a constraint)
opts = generation_options({"max_tokens": 4, "response_format": {"type": "json_object"}}, 8)
self.assertIn("root ::=", opts[3])
schema = {"type": "object", "properties": {"a": {"type": "string"}}, "required": ["a"]}
opts = generation_options({"max_tokens": 4, "response_format":
{"type": "json_schema", "json_schema": {"schema": schema}}}, 8)
self.assertEqual(json.loads(opts[3]), schema)
opts = generation_options({"max_tokens": 4, "response_format":
{"type": "gbnf", "grammar": 'root ::= "x"'}}, 8)
self.assertEqual(opts[3], 'root ::= "x"')
with self.assertRaises(APIError):
generation_options({"response_format": {"type": "yaml"}}, 8)
with self.assertRaises(APIError):
generation_options({"response_format": {"type": "json_schema", "json_schema": {}}}, 8)
# a json_schema that is not an object is the client's mistake too: a 400
# naming the parameter, not an AttributeError the handler turns into a
# 500 "engine failed" (which OpenAI SDKs retry)
for json_schema in ('{"schema": {}}', [schema], 5, True):
with self.subTest(json_schema=json_schema):
with self.assertRaises(APIError) as caught:
generation_options({"response_format": {"type": "json_schema",
"json_schema": json_schema}}, 8)
self.assertEqual(caught.exception.status, 400)
self.assertEqual(caught.exception.param, "response_format")
with self.assertRaises(APIError): # non-dict response_format
generation_options({"response_format": "json"}, 8)
with self.assertRaises(APIError): # empty gbnf
generation_options({"response_format": {"type": "gbnf", "grammar": " "}}, 8)
with self.assertRaises(APIError): # oversized grammar (> 1 MiB pre-check)
generation_options({"response_format": {"type": "gbnf", "grammar": "x" * ((1 << 20) + 1)}}, 8)
# malformed GBNF passes the gateway by design: the ENGINE fail-softs it
# (draft source only — bad grammar costs the speedup, never the request)
opts = generation_options({"response_format": {"type": "gbnf", "grammar": "not a grammar ::="}}, 8)
self.assertEqual(opts[3], "not a grammar ::=")
def test_tool_choice_function_that_is_not_an_object_is_a_400(self):
"""The same read, six places: generation_options and five renderers.
`(tool_choice.get("function") or {}).get("name")` raised AttributeError
on {"type": "function", "function": "search"} -- the name written where
the object goes -- and that became a 500 "engine failed" instead of the
400 generation_options already had two lines further down. The renderers
crashed on the identical expression before validation was even reached.
"""
import openai_server as srv
tools = [{"type": "function", "function": {"name": "search"}}]
messages = [{"role": "user", "content": "hi"}]
for function in ("search", ["search"], 5, True):
with self.subTest(function=function):
choice = {"type": "function", "function": function}
with self.assertRaises(APIError) as caught:
generation_options({"tools": tools, "tool_choice": choice}, 8)
self.assertEqual(caught.exception.status, 400)
self.assertEqual(caught.exception.param, "tool_choice")
# the renderers must get past the read so that 400 is what runs
for render in (render_chat, srv.render_chat_glm53, render_chat_kimi,
render_chat_v4, srv.render_chat_dsv41):
render(list(messages), tools=list(tools), tool_choice=choice)
# A well-formed choice, and the legacy {"type": "function", "name": ...}
# spelling, still force the tool.
for choice in ({"type": "function", "function": {"name": "search"}},
{"type": "function", "name": "search"}):
with self.subTest(choice=choice):
generation_options({"tools": tools, "tool_choice": choice}, 8)
self.assertIn("You must call the function `search`",
render_chat(list(messages), tools=list(tools),
tool_choice=choice))
def test_tool_function_that_is_not_an_object_is_a_400(self):
"""The same slip as tool_choice, on tools[] this time.
generation_options already has 400 "Tool function must be an object".
The GLM and DeepSeek declaration blocks then did fn.items() on
{"type": "function", "function": "search"} and raised AttributeError
before that 400 ran, so do_POST answered 500 "engine failed".
"""
import openai_server as srv
messages = [{"role": "user", "content": "hi"}]
for function in ("search", ["search"], 5, True):
with self.subTest(function=function):
tools = [{"type": "function", "function": function}]
with self.assertRaises(APIError) as caught:
generation_options({"tools": tools}, 8)
self.assertEqual(caught.exception.status, 400)
self.assertEqual(caught.exception.param, "tools.0.function")
for render in (render_chat, srv.render_chat_glm53, render_chat_kimi,
render_chat_v4, srv.render_chat_dsv41):
try:
render(list(messages), tools=list(tools))
except APIError as error:
self.assertEqual(error.status, 400)
well = [{"type": "function", "function": {"name": "search"}}]
generation_options({"tools": well}, 8)
self.assertIn('"name": "search"', render_chat(list(messages), tools=well))
def test_coli_temp_is_the_default_for_requests_that_omit_temperature(self):
with patch.dict("openai_server.os.environ", {"COLI_TEMP": "0.25"}):
self.assertEqual(generation_options({}, 8)[1], 0.25)
self.assertEqual(generation_options({"temperature": 0}, 8)[1], 0.0)
for invalid in ("malformed", "5", "-1", "nan", "1e999"):
with self.subTest(invalid=invalid):
with patch.dict("openai_server.os.environ", {"COLI_TEMP": invalid}):
self.assertEqual(generation_options({}, 8)[1], 0.7)
def test_validates_stop_sequences(self):
self.assertEqual(generation_options({"stop": "END"}, 8)[4], ("END",))
self.assertEqual(generation_options({"stop": ["ONE", "TWO"]}, 8)[4],
("ONE", "TWO"))
for value in ("", [], [""], ["1", "2", "3", "4", "5"], 7, ["ok", 7]):
with self.subTest(value=value), self.assertRaises(APIError):
generation_options({"stop": value}, 8)
def test_keepalive_choice_matches_the_endpoint_chunk_shape(self):
# Chat chunks carry a `delta`; the diagnostic marker rides reasoning_content,
# off the visible answer.
chat = _keepalive_choice(True, False)
self.assertEqual(chat["delta"], {"reasoning_content": ""})
self.assertNotIn("text", chat)
self.assertEqual(_keepalive_choice(True, True)["delta"],
{"reasoning_content": "."})
# Legacy /v1/completions chunks carry `text`, never `delta`, or a strict
# client (OpenAI SDK: CompletionChoice.text is required) rejects the ping.
# No side channel, so the marker never appears: a "." would land in the text.
for visible in (False, True):
comp = _keepalive_choice(False, visible)
self.assertEqual(comp["text"], "")
self.assertNotIn("delta", comp)
def test_glm_chat_defaults_role_stops_without_changing_other_policies(self):
with patch("openai_server.ARCH", "glm"):
self.assertEqual(stop_policy({}, True), (DEFAULT_CHAT_STOP_SEQUENCES, True))
self.assertEqual(stop_policy({}, False), ((), False))
self.assertEqual(stop_policy({"stop": "END"}, True), (("END",), False))
self.assertEqual(stop_policy({
"stop": "END", "x_colibri_ignore_leading_stop": True,
}, True), (("END",), True))
with patch("openai_server.ARCH", "inkling"):
self.assertEqual(stop_policy({}, True), ((), False))
self.assertEqual(stop_policy({"stop": "END"}, True), (("END",), False))
with self.assertRaises(APIError):
stop_policy({"x_colibri_ignore_leading_stop": "yes"}, True)
class Qwen36ToolCallTest(unittest.TestCase):
WEATHER = [{
"type": "function",
"function": {
"name": "weather",
"description": "Get weather for a city.",
"parameters": {
"type": "object",
"properties": {
"city": {"type": "string"},
"days": {"type": "integer"},
},
"required": ["city"],
},
},
}]
def test_qwen36_renders_tool_declaration(self):
prompt = render_chat_qwen(
[{"role": "user", "content": "Weather in Rome?"}],
tools=self.WEATHER,
)
self.assertIn(
"# Tools\n\nYou have access to the following functions:\n\n<tools>",
prompt,
)
self.assertIn('"name": "weather"', prompt)
self.assertIn("</tools>", prompt)
self.assertIn("<function=example_function_name>", prompt)
def test_qwen36_tool_choice_none_suppresses_declaration(self):
prompt = render_chat_qwen(
[{"role": "user", "content": "Weather?"}],
tools=self.WEATHER,
tool_choice="none",
)
self.assertNotIn("<tools>", prompt)
def test_qwen36_renders_assistant_tool_call(self):
prompt = render_chat_qwen([
{"role": "user", "content": "Weather in Rome?"},
{
"role": "assistant",
"content": "",
"tool_calls": [{
"id": "call_1",
"type": "function",
"function": {
"name": "weather",
"arguments": '{"city":"Rome","days":3}',
},
}],
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": "sunny",
},
], tools=self.WEATHER)
self.assertIn(
"<tool_call>\n"
"<function=weather>\n"
"<parameter=city>\n"
"Rome\n"
"</parameter>\n"
"<parameter=days>\n"
"3\n"
"</parameter>\n"
"</function>\n"
"</tool_call>",
prompt,
)
self.assertIn(
"<|im_start|>user\n"
"<tool_response>\n"
"sunny\n"
"</tool_response><|im_end|>",
prompt,
)
def test_qwen36_consecutive_tool_results_share_user_turn(self):
prompt = render_chat_qwen([
{"role": "user", "content": "Check both."},
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "a",
"type": "function",
"function": {
"name": "weather",
"arguments": '{"city":"Rome"}',
},
},
{
"id": "b",
"type": "function",
"function": {
"name": "weather",
"arguments": '{"city":"Berlin"}',
},
},
],
},
{"role": "tool", "tool_call_id": "a", "content": "sunny"},
{"role": "tool", "tool_call_id": "b", "content": "cloudy"},
], tools=self.WEATHER)
self.assertIn(
"<|im_start|>user\n"
"<tool_response>\n"
"sunny\n"
"</tool_response>\n"
"<tool_response>\n"
"cloudy\n"
"</tool_response><|im_end|>",
prompt,
)
def test_qwen36_parser_accepts_native_call(self):
text, calls = parse_qwen_tool_calls(
"Checking.\n\n"
"<tool_call>\n"
"<function=weather>\n"
"<parameter=city>\n"
"Rome\n"
"</parameter>\n"
"<parameter=days>\n"
"3\n"
"</parameter>\n"
"</function>\n"
"</tool_call>",
self.WEATHER,
)
self.assertEqual(text, "Checking.")
self.assertEqual(len(calls), 1)
self.assertEqual(calls[0]["function"]["name"], "weather")
args = json.loads(calls[0]["function"]["arguments"])
self.assertEqual(args["city"], "Rome")
self.assertEqual(args["days"], 3)
def test_qwen36_parser_preserves_text_around_native_call(self):
reply = (
"Checking.\n\n"
"<tool_call>\n"
"<function=weather>\n"
"<parameter=city>\n"
"Karlsruhe\n"
"</parameter>\n"
"</function>\n"
"</tool_call>"
)
text, calls = parse_qwen_tool_calls(reply, self.WEATHER)
self.assertEqual(text, "Checking.")
self.assertEqual(len(calls), 1)
self.assertEqual(calls[0]["function"]["name"], "weather")
self.assertEqual(
json.loads(calls[0]["function"]["arguments"]),
{"city": "Karlsruhe"},
)
def test_qwen36_preserves_current_query_reasoning_content(self):
prompt = render_chat_qwen([
{"role": "user", "content": "Think about this."},
{
"role": "assistant",
"reasoning_content": "current reasoning",
"content": "answer",
},
{"role": "tool", "content": "result"},
])
self.assertIn(
"<|im_start|>assistant\n"
"<think>\ncurrent reasoning\n</think>\n\n"
"answer<|im_end|>",
prompt,
)
def test_qwen36_drops_reasoning_before_last_user_query(self):
prompt = render_chat_qwen([
{"role": "user", "content": "First question."},
{
"role": "assistant",
"reasoning_content": "old private reasoning",
"content": "first answer",
},
{"role": "user", "content": "Second question."},
])
self.assertNotIn("old private reasoning", prompt)
self.assertIn(
"<|im_start|>assistant\nfirst answer<|im_end|>",
prompt,
)
def test_qwen36_extracts_native_think_history(self):
prompt = render_chat_qwen([
{"role": "user", "content": "Think about this."},
{
"role": "assistant",
"content": (
"<think>\n"
"native reasoning\n"
"</think>\n\n"
"visible answer"
),
},
{"role": "tool", "content": "result"},
])
self.assertEqual(prompt.count("native reasoning"), 1)
self.assertIn(
"<|im_start|>assistant\n"
"<think>\nnative reasoning\n</think>\n\n"
"visible answer<|im_end|>",
prompt,
)
def test_qwen36_rejects_non_string_reasoning_content(self):
with self.assertRaises(APIError):
render_chat_qwen([
{"role": "user", "content": "Question."},
{
"role": "assistant",
"reasoning_content": {"not": "a string"},
"content": "answer",
},
])
class StopFilterTest(unittest.TestCase):
def test_explicit_stop_composes_with_inkling_stream_split(self):
content = []
reasoning = []
splitter = InklingStreamSplit(content.append, reasoning.append)
stop_filter = StopFilter(("END",), splitter.feed)
for chunk in ("<|content_thinking|>why<|content_text|>answer EN", "Dignored"):
stop_filter.feed(chunk)
stop_filter.finish()
splitter.close()
self.assertEqual("".join(reasoning), "why")
self.assertEqual("".join(content), "answer ")
self.assertEqual(stop_filter.matched, "END")
def test_hides_match_split_across_chunks(self):
output = []
stop_filter = StopFilter(("STOP",), output.append)
for chunk in ("answer S", "TO", "Pignored"):
stop_filter.feed(chunk)
stop_filter.finish()
self.assertEqual("".join(output), "answer ")
self.assertEqual(stop_filter.matched, "STOP")
def test_flushes_partial_prefix_when_generation_finishes(self):
output = []
stop_filter = StopFilter(("STOP",), output.append)
stop_filter.feed("answer ST")
stop_filter.finish()
self.assertEqual("".join(output), "answer ST")
def test_optional_patient_mode_ignores_only_leading_matches(self):
output = []
stop_filter = StopFilter(("<|user|>",), output.append, ignore_leading=True)
for chunk in ("<|us", "er|>answer", "<|user|>ignored"):
stop_filter.feed(chunk)
stop_filter.finish()
self.assertEqual("".join(output), "answer")
self.assertEqual(stop_filter.matched, "<|user|>")
self.assertEqual(stop_filter.leading_matches_ignored, 1)
def test_patient_mode_preserves_remainder_after_same_chunk_leading_match(self):
output = []
stop_filter = StopFilter(("STOP",), output.append, ignore_leading=True)
stop_filter.feed("STOPuseful STOPdiscarded")
stop_filter.finish()
self.assertEqual("".join(output), "useful ")
self.assertEqual(stop_filter.matched, "STOP")
def test_strict_mode_still_stops_on_a_leading_match(self):
output = []
stop_filter = StopFilter(("STOP",), output.append)
stop_filter.feed("STOPignored")
stop_filter.finish()
self.assertEqual(output, [])
self.assertEqual(stop_filter.matched, "STOP")
class SpanMapPrimitivesTest(unittest.TestCase):
"""The stage-map algebra, on literal maps written out by hand -- no stage builds any of
these, so nothing here can be satisfied by the same code it polices."""
def test_cut_map_records_the_survivors_around_each_cut(self):
# "abcXYdef" minus [3, 5): two survivors, and the second one's output position is 3
# because exactly three characters precede it.
self.assertEqual(_cut_span_map(8, [(3, 5)]), [(0, 3, 0), (5, 8, 3)])
def test_adjacent_cuts_leave_one_merged_survivor_not_two(self):
# "abcd" minus [1, 2) and [2, 3) is one span on each side, never two abutting ones.
self.assertEqual(_cut_span_map(4, [(1, 2), (2, 3)]), [(0, 1, 0), (3, 4, 1)])
def test_composition_chains_two_stages_into_one_map(self):
# Stage one deletes "XY" from "abcXYdef" and hands "abcdef" on; stage two deletes
# "cd" from that, leaving "abef". Composed, the map speaks the original string's
# coordinates: 'a','b' at 0,1 and 'e','f' at 6,7.
first = [(0, 3, 0), (5, 8, 3)]
second = [(0, 2, 0), (4, 6, 2)]
self.assertEqual(_compose_span_maps(first, second), [(0, 2, 0), (6, 8, 2)])
def test_composition_is_associative_over_three_stages(self):
first, second, third = [(0, 3, 0), (5, 8, 3)], [(0, 2, 0), (4, 6, 2)], [(1, 4, 0)]
self.assertEqual(_compose_span_maps(_compose_span_maps(first, second), third),
_compose_span_maps(first, _compose_span_maps(second, third)))
def test_projection_of_a_fully_deleted_range_is_none_not_zero_zero(self):
# The distinction a prefix walk never makes: "deleted" is not "at offset 0".
self.assertIsNone(_project_span([(0, 2, 0), (6, 8, 2)], 3, 5))
def test_projection_spans_a_hole_because_survivors_close_up(self):
# Input [1, 7) covers 'b' (survives at 1) and 'e' (survives at 2) with the deleted
# middle between them: the image is the contiguous [1, 3).
self.assertEqual(_project_span([(0, 2, 0), (6, 8, 2)], 1, 7), (1, 3))
def test_cuts_out_of_order_produce_the_map_their_deletion_does(self):
# The map is the deletion, whatever order the ranges arrive in. Walking an
# unordered list as if it were sorted produces a map no deletion corresponds to,
# silently, and every stage downstream then reads the wrong coordinates.
text = "abcXYdefZW"
for cuts in ([(3, 5), (8, 10)], [(8, 10), (3, 5)]):
with self.subTest(cuts=cuts):
spans = _cut_span_map(len(text), cuts)
self.assertEqual(spans, [(0, 3, 0), (5, 8, 3)])
self.assertEqual("".join(text[a:b] for a, b, _o in spans), "abcdef")
def test_an_empty_marker_yields_no_cuts(self):
# `str.find("")` returns the cursor forever, so a marker that is somehow empty
# would spin rather than fail. Unreachable today -- both markers are non-empty
# module constants -- and the guard is one line.
self.assertEqual(_marker_cuts("abc", ""), [])
self.assertEqual(_marker_cuts("abcabc", "bc"), [(1, 3), (4, 6)])
def test_projection_clamps_a_range_that_runs_off_the_end(self):
# Records outlive the map: after a stop match the engine keeps sending frames the
# stop filter never sees, and those must project to nothing.
self.assertEqual(_project_span([(0, 3, 0)], 2, 9), (2, 3))
self.assertIsNone(_project_span([(0, 3, 0)], 4, 9))
class StopFilterSpanAccountingTest(unittest.TestCase):
"""The stop filter's map, with its own oracle: every expected span below is written out
by hand from the chunk sequence, never read back from the filter."""
def test_normal_flush_spans_the_characters_it_released(self):
# "one S" holds back the "S" (a live prefix of "STOP"); the next chunk resolves it.
# Both flushes are one contiguous run of the raw stream, so the canonical map is
# one span covering "one Swo", not two abutting ones.
output = []
stop_filter = StopFilter(("STOP",), output.append, track_spans=True)
stop_filter.feed("one S")
self.assertEqual(stop_filter.spans, [(0, 4, 0)])
stop_filter.feed("wo")
stop_filter.finish()
self.assertEqual("".join(output), "one Swo")
self.assertEqual(stop_filter.spans, [(0, 7, 0)])
def test_finish_spans_the_pending_tail_it_releases(self):
# "ST" is still held when the engine stops, so the map must grow at finish().
output = []
stop_filter = StopFilter(("STOP",), output.append, track_spans=True)
stop_filter.feed("tail ST")
self.assertEqual(stop_filter.spans, [(0, 5, 0)])
stop_filter.finish()
self.assertEqual("".join(output), "tail ST")
self.assertEqual(stop_filter.spans, [(0, 7, 0)])
def test_ignored_leading_marker_is_a_hole_in_the_map(self):
# "<|user|>" occupies raw [0, 8) and reaches no one, so the map starts at 8 and
# "Hello world" lands at output 0. A consumer reading this cannot conclude that
# record 0 was emitted.
output = []
stop_filter = StopFilter(("<|user|>",), output.append, ignore_leading=True,
track_spans=True)
for chunk in ("<|user|>", "Hello", " world"):
stop_filter.feed(chunk)
stop_filter.finish()
self.assertEqual("".join(output), "Hello world")
self.assertEqual(stop_filter.spans, [(8, 19, 0)])
def test_matched_stop_spans_only_the_prefix_it_emitted(self):
# "He" then "llo" against stop "lo" emits "Hel" and stops. Raw "Hello" is five
# characters; only [0, 3) was ever released.
output = []
stop_filter = StopFilter(("lo",), output.append, track_spans=True)
for chunk in ("He", "llo", "X"):
stop_filter.feed(chunk)
stop_filter.finish()
self.assertEqual("".join(output), "Hel")
self.assertEqual(stop_filter.matched, "lo")
self.assertEqual(stop_filter.spans, [(0, 3, 0)])
def test_stop_split_across_two_feeds_closes_the_map_at_the_match(self):
# The sequence arrives as "S" / "TO" / "P": the filter holds a growing partial
# prefix across three calls and must not count the held characters as emitted. Raw
# is "answer STOPignored" (18 characters); the map ends at 7.
output = []
stop_filter = StopFilter(("STOP",), output.append, track_spans=True)
for chunk in ("answer S", "TO", "Pignored"):
stop_filter.feed(chunk)
stop_filter.finish()
self.assertEqual("".join(output), "answer ")
self.assertEqual(stop_filter.matched, "STOP")
self.assertEqual(stop_filter.spans, [(0, 7, 0)])
def test_a_swallowed_marker_and_a_match_in_the_same_feed_call(self):
# Both events resolve inside a single feed(), where only the loop's local cursor
# has advanced past the swallowed marker. Reading the wrong one shifts the whole
# map to 0: "answer" would be reported as raw [0, 6) instead of [8, 14).
# "<|user|>" [0,8) ignored - "answer" [8,14) emitted - "<|user|>tail" dropped
output = []
stop_filter = StopFilter(("<|user|>",), output.append, ignore_leading=True,
track_spans=True)
stop_filter.feed("<|user|>answer<|user|>tail")
stop_filter.finish()
self.assertEqual("".join(output), "answer")
self.assertEqual(stop_filter.matched, "<|user|>")
self.assertEqual(stop_filter.leading_matches_ignored, 1)
self.assertEqual(stop_filter.spans, [(8, 14, 0)])
def test_two_ignored_markers_then_a_real_match_leave_both_holes(self):
# Raw is
# "<|user|>" "<|user|>" "ok " "STOP" "rest" = [0,8) [8,16) [16,19) [19,23) [23,27)
# The map is the single interval [16, 19): everything else was dropped here.
output = []
stop_filter = StopFilter(("<|user|>", "STOP"), output.append, ignore_leading=True,
track_spans=True)
for chunk in ("<|user|>", "<|user|>", "ok ", "STOPrest"):
stop_filter.feed(chunk)
stop_filter.finish()
self.assertEqual("".join(output), "ok ")
self.assertEqual(stop_filter.leading_matches_ignored, 2)
self.assertEqual(stop_filter.matched, "STOP")
self.assertEqual(stop_filter.spans, [(16, 19, 0)])
def test_map_survives_arbitrary_chunking_of_the_same_raw_stream(self):
# Fed in three-character chunks, so every marker and the stop sequence straddle a
# feed() boundary. Two holes, written out by hand from the string:
# [0, 8) the first "<|user|>" ignored (leading)
# [8, 10) the two spaces emitted at output 0 -- they flush before the
# second marker is visible, and blanks never set useful_content_seen,
# so the marker after them is still leading
# [10,18) the second "<|user|>" ignored (leading)
# [18,29) "alpha beta " emitted at output 2
# [29,37) "STOPtail" dropped at the match
raw = "<|user|> <|user|>alpha beta STOPtail"
output = []
stop_filter = StopFilter(("<|user|>", "STOP"), output.append, ignore_leading=True,
track_spans=True)
for size in range(0, len(raw), 3):
stop_filter.feed(raw[size:size + 3])
stop_filter.finish()
self.assertEqual(stop_filter.spans, [(8, 10, 0), (18, 29, 2)])
self.assertEqual("".join(output), " alpha beta ")
# ...and the relational form the literals above make non-vacuous: slicing the raw
# stream by the map rebuilds exactly what the filter emitted.
self.assertEqual("".join(raw[start:end] for start, end, _out in stop_filter.spans),
"".join(output))
def test_spans_are_not_collected_unless_the_caller_asks(self):
output = []
stop_filter = StopFilter(("STOP",), output.append)
stop_filter.feed("plain text")
stop_filter.finish()
self.assertEqual("".join(output), "plain text")
self.assertEqual(stop_filter.spans, [])
class StageSpanReportingTest(unittest.TestCase):
"""The thinking split and the tool-call parse report maps that describe the text they
return: slicing the stage's input by its map rebuilds the stage's output exactly."""
def _assert_map_rebuilds(self, source, produced, span_map):
self.assertEqual("".join(source[start:end] for start, end, _out in span_map),
produced)
def test_thinking_split_maps_both_buckets(self):
raw = THINK_OPEN + "why" + THINK_CLOSE + "answer"
thinking, answer, (thinking_map, answer_map) = split_thinking_reply_spans(raw)
self.assertEqual((thinking, answer), ("why", "answer"))
self._assert_map_rebuilds(raw, thinking, thinking_map)
self._assert_map_rebuilds(raw, answer, answer_map)
def test_thinking_split_map_is_empty_when_the_bucket_is(self):
raw = "plain answer"
thinking, answer, (thinking_map, answer_map) = split_thinking_reply_spans(
raw, enable_thinking=False)
self.assertEqual((thinking, answer), ("", "plain answer"))
self.assertEqual(thinking_map, [])
self._assert_map_rebuilds(raw, answer, answer_map)
def test_tool_call_parse_maps_the_content_it_returns(self):
tools = [{"function": {"name": "search",
"parameters": {"properties": {"q": {"type": "string"}}}}}]
raw = ("before <tool_call>search<arg_key>q</arg_key><arg_value>x</arg_value>"
"</tool_call> after")
content, calls, box_map, content_map = parse_tool_calls_spans(raw, tools)
self.assertEqual(len(calls), 1)
self.assertEqual(content, "before after")
self._assert_map_rebuilds(raw, content, content_map)
# The box map stops after the tool-call removal, before the strip: it is what tells
# a character consumed as tool-call syntax apart from one consumed as markup.
self._assert_map_rebuilds(raw, "before after", box_map)
def test_tool_call_parse_map_covers_the_thinking_prefix_it_drops(self):
raw = "reasoning" + THINK_CLOSE + " visible "
content, calls, _box_map, content_map = parse_tool_calls_spans(raw, None)
self.assertEqual((content, calls), ("visible", []))
self._assert_map_rebuilds(raw, content, content_map)
def test_shims_return_exactly_what_the_span_forms_do(self):
raw = THINK_OPEN + "why" + THINK_CLOSE + " answer "
self.assertEqual(split_thinking_reply(raw), split_thinking_reply_spans(raw)[:2])
self.assertEqual(parse_tool_calls(raw), parse_tool_calls_spans(raw)[:2])
self.assertEqual(parse_arch_tool_calls(raw, None),
parse_arch_tool_calls_spans(raw, None)[:2])
def test_a_qwen38_template_on_the_qwen36_engine_parses_as_qwen38_without_maps(self):
# The parser follows the template the checkpoint ships (#1757), not the engine
# family: a qwen36 engine running Qwen3.8's template emits Qwen3.8's XML calls.
# That parser reports no maps, which is safe only because the logprobs channel
# is refused on that engine (ChatFlavorLogprobsGateTest).
tool = {"type": "function", "function": {
"name": "weather", "description": "w",
"parameters": {"type": "object",
"properties": {"city": {"type": "string"},
"days": {"type": "integer"}}}}}
raw = ("Sure.\n\n<tool_call>\n<function=weather>\n<parameter=city>\nRome\n"
"</parameter>\n<parameter=days>\n3\n</parameter>\n</function>\n</tool_call>")
with patch("openai_server.ARCH", "qwen36"), patch("openai_server.CHAT_FLAVOR", "qwen38"):
spans = parse_arch_tool_calls_spans(raw, [tool])
public = parse_arch_tool_calls(raw, [tool])
self.assertEqual(len(spans), 4)
content, calls, box_map, content_map = spans
self.assertEqual(len(calls), 1)
self.assertEqual(calls[0]["function"]["name"], "weather")
self.assertEqual(json.loads(calls[0]["function"]["arguments"]),
{"city": "Rome", "days": 3})
self.assertEqual(content, "Sure.")
self.assertIsNone(box_map)
self.assertIsNone(content_map)
self.assertEqual(len(public), 2)
self.assertEqual(public[0], "Sure.")
self.assertEqual(len(public[1]), 1)
self.assertEqual(public[1][0]["function"]["name"], "weather")
def test_the_mimo_parser_maps_the_content_it_returns(self):
# The logprobs channel is open on mimo, so its parser has to say which
# characters of the reply survive as content: every call block, then the
# whitespace around what is left, exactly as the plain parse removes them.
tool = {"type": "function", "function": {
"name": "weather", "parameters": {"type": "object",
"properties": {"city": {"type": "string"},
"days": {"type": "integer"}}}}}
raw = (" Sure. <tool_call>\n<function=weather>\n<parameter=city>Rome</parameter>\n"
"<parameter=days>3</parameter>\n</function>\n</tool_call> then this. ")
with patch("openai_server.ARCH", "mimo"):
content, calls, box_map, content_map = parse_arch_tool_calls_spans(raw, [tool])
public = parse_arch_tool_calls(raw, [tool])
self.assertEqual(content, "Sure. then this.")
self.assertEqual(json.loads(calls[0]["function"]["arguments"]),
{"city": "Rome", "days": 3})
self._assert_map_rebuilds(raw, content, content_map)
self._assert_map_rebuilds(raw, " Sure. then this. ", box_map)
self.assertEqual(public[0], content)
self.assertEqual([c["function"] for c in public[1]], [c["function"] for c in calls])
def test_the_qwen38_engine_parses_its_own_calls_with_no_flavor_set(self):
# The native arm of the same dispatch: chat_flavor() falls back to ARCH, so a
# qwen38 engine with no flavor recorded still reaches the qwen38 parser.
tool = {"type": "function", "function": {
"name": "weather", "description": "w",
"parameters": {"type": "object",
"properties": {"city": {"type": "string"}}}}}
raw = ("Sure.\n\n<tool_call>\n<function=weather>\n<parameter=city>\nRome\n"
"</parameter>\n</function>\n</tool_call>")
with patch("openai_server.ARCH", "qwen38"), patch("openai_server.CHAT_FLAVOR", None):
spans = parse_arch_tool_calls_spans(raw, [tool])
public = parse_arch_tool_calls(raw, [tool])
self.assertEqual(len(spans), 4)
self.assertEqual(len(spans[1]), 1)
self.assertEqual(spans[1][0]["function"]["name"], "weather")
self.assertEqual(json.loads(spans[1][0]["function"]["arguments"]), {"city": "Rome"})
self.assertIsNone(spans[2])
self.assertIsNone(spans[3])
self.assertEqual(len(public), 2)
self.assertEqual(len(public[1]), 1)
self.assertEqual(public[1][0]["function"]["name"], "weather")
class ProtocolTest(unittest.TestCase):
def test_reads_payload_and_extended_status(self):
stream = io.BytesIO(b"hello" + END + b"STAT 2 3.5 44 1.2 7 1\n")
chunks = []
stats = read_engine_turn(stream, END, chunks.append)
self.assertEqual(b"".join(chunks), b"hello")
self.assertEqual(stats["prompt_tokens"], 7)
self.assertTrue(stats["length_limited"])
def test_reads_caps_between_ready_and_stat(self):
# A vision engine says what it loaded BEFORE its status line, so the gateway
# knows the served modalities before it answers its first request. An engine
# that says nothing leaves the dict empty and the status parse untouched.
caps = {}
stream = io.BytesIO(READY + b"CAPS vision=1 other=x\nSTAT 0 0 0 0\n")
stats = read_engine_turn(stream, READY, lambda _: None, caps)
self.assertEqual(caps, {"vision": "1", "other": "x"})
self.assertEqual(stats["completion_tokens"], 0)
caps = {}
read_engine_turn(io.BytesIO(READY + b"STAT 0 0 0 0\n"), READY, lambda _: None, caps)
self.assertEqual(caps, {})
def test_rejects_invalid_kv_pool_before_engine_start(self):
with self.assertRaisesRegex(ValueError, "kv_slots"):
serve("/missing", kv_slots=0)
def test_occupied_port_fails_before_engine_start(self):
listener = socket.socket()
listener.bind(("127.0.0.1", 0))
listener.listen()
try:
with patch("openai_server.subprocess.Popen") as popen:
with self.assertRaises(OSError):
serve("/missing", port=listener.getsockname()[1])
popen.assert_not_called()
finally:
listener.close()
class GenerationMetricsTest(unittest.TestCase):
def setUp(self):
self.server = APIServer(("127.0.0.1", 0), FakeEngine(), "test")
self.addCleanup(self.server.server_close)
def test_records_first_output_once_and_preserves_text_and_stats(self):
output = []
with patch("openai_server.time.monotonic", side_effect=[10, 10.5, 13]):
stats = self.server.generate("prompt", 4, 0, 1, output.append)
self.assertEqual(output, ["Hé", "llo"])
self.assertEqual(stats["completion_tokens"], 2)
metrics = self.server.scheduler.prometheus()
self.assertIn("colibri_scheduler_first_output_seconds_sum 0.5\n", metrics)
self.assertIn("colibri_scheduler_first_output_seconds_count 1\n", metrics)
self.assertIn("colibri_scheduler_engine_call_seconds_sum 3.0\n", metrics)
self.assertIn("colibri_scheduler_engine_call_seconds_count 1\n", metrics)
def test_empty_output_does_not_count_but_tool_output_does(self):
text, tools = [], []
def generate(prompt, maximum, temperature, top_p, on_text, **kwargs):
on_text("")
kwargs["on_tool"]("")
kwargs["on_tool"]("tool payload")
on_text("tail")
return {"completion_tokens": 2}
with patch.object(self.server.engine, "generate", side_effect=generate), \
patch("openai_server.time.monotonic", side_effect=[10, 12, 15]):
self.server.generate("prompt", 4, 0, 1, text.append, on_tool=tools.append)
self.assertEqual(text, ["", "tail"])
self.assertEqual(tools, ["", "tool payload"])
metrics = self.server.scheduler.prometheus()
self.assertIn("colibri_scheduler_first_output_seconds_sum 2.0\n", metrics)
self.assertIn("colibri_scheduler_first_output_seconds_count 1\n", metrics)
def test_error_and_cancellation_before_output_do_not_invent_first_output(self):
for error in (RuntimeError("failed"), ClientCancelled()):
with patch.object(self.server.engine, "generate", side_effect=error), \
patch("openai_server.time.monotonic", side_effect=[10, 14]):
with self.assertRaises(type(error)):
self.server.generate("prompt", 4, 0, 1, lambda text: None)
metrics = self.server.scheduler.prometheus()
self.assertIn("colibri_scheduler_first_output_seconds_count 0\n", metrics)
self.assertIn("colibri_scheduler_engine_call_seconds_sum 8.0\n", metrics)
self.assertIn("colibri_scheduler_engine_call_seconds_count 2\n", metrics)
def test_failure_after_output_keeps_both_observations(self):
def generate(prompt, maximum, temperature, top_p, on_text):
on_text("partial")
raise RuntimeError("failed after output")
with patch.object(self.server.engine, "generate", side_effect=generate), \
patch("openai_server.time.monotonic", side_effect=[10, 11, 12]):
with self.assertRaises(RuntimeError):
self.server.generate("prompt", 4, 0, 1, lambda text: None)
metrics = self.server.scheduler.prometheus()
self.assertIn("colibri_scheduler_first_output_seconds_count 1\n", metrics)
self.assertIn("colibri_scheduler_engine_call_seconds_count 1\n", metrics)
class SchedulerTest(unittest.TestCase):
def test_engine_failure_is_not_a_completed_request(self):
scheduler = GenerationScheduler()
with self.assertRaisesRegex(RuntimeError, "engine failed"):
with scheduler.admit():
raise RuntimeError("engine failed")
stats = scheduler.snapshot()
self.assertEqual(stats["failed"], 1)
self.assertEqual(stats["completed"], 0)
self.assertEqual(stats["active"], 0)
with scheduler.admit():
pass
self.assertEqual(scheduler.snapshot()["completed"], 1)
def test_prometheus_histograms_measure_admission_and_slot_occupancy(self):
scheduler = GenerationScheduler()
with patch("openai_server.time.monotonic", side_effect=[10, 10.25, 10.25, 12.25]):
with scheduler.admit():
active = scheduler.prometheus()
self.assertIn("colibri_scheduler_active 1\n", active)
self.assertIn("colibri_scheduler_slot_duration_seconds_count 0\n", active)
metrics = scheduler.prometheus()
self.assertIn("# TYPE colibri_scheduler_completed_total counter\n", metrics)
self.assertIn("colibri_scheduler_completed_total 1\n", metrics)
self.assertIn("colibri_scheduler_queue_wait_seconds_sum 0.25\n", metrics)
self.assertIn('colibri_scheduler_queue_wait_seconds_bucket{le="0.1"} 0\n', metrics)
self.assertIn('colibri_scheduler_queue_wait_seconds_bucket{le="0.5"} 1\n', metrics)
self.assertIn('colibri_scheduler_queue_wait_seconds_bucket{le="+Inf"} 1\n', metrics)
self.assertIn("colibri_scheduler_slot_duration_seconds_sum 2.0\n", metrics)
self.assertIn("colibri_scheduler_slot_duration_seconds_count 1\n", metrics)
def test_failed_and_cancelled_admissions_are_timed(self):
scheduler = GenerationScheduler()
for error in (RuntimeError("failed"), ClientCancelled()):
with self.assertRaises(type(error)):
with scheduler.admit():
raise error
metrics = scheduler.prometheus()
self.assertIn("colibri_scheduler_failed_total 1\n", metrics)
self.assertIn("colibri_scheduler_cancelled_total 1\n", metrics)
self.assertIn("colibri_scheduler_completed_total 0\n", metrics)
self.assertIn("colibri_scheduler_slot_duration_seconds_count 2\n", metrics)
def test_admits_up_to_capacity_without_serializing(self):
scheduler = GenerationScheduler(max_queue=0, queue_timeout=1, capacity=2)
with scheduler.admit() as first:
with scheduler.admit() as second:
self.assertEqual({first[1], second[1]}, {0, 1})
self.assertEqual(scheduler.snapshot()["active"], 2)
def test_rejects_when_waiting_queue_is_full(self):
scheduler = GenerationScheduler(max_queue=0, queue_timeout=1)
with scheduler.admit():
with self.assertRaises(APIError) as caught:
with scheduler.admit():
pass
self.assertEqual(caught.exception.status, 429)
self.assertEqual(caught.exception.code, "queue_full")
self.assertEqual(scheduler.snapshot()["rejected"], 1)
def test_any_slot_request_uses_capacity_not_reserved_by_older_waiters(self):
scheduler = GenerationScheduler(capacity=2)
# State immediately after both slots are released, before the older
# slot-0 waiter reacquires the condition lock.
older = (object(), 0)
scheduler.queue.append(older)
with patch.object(scheduler.condition, "wait", side_effect=AssertionError("unused free slot")):
with scheduler.admit() as (_, slot):
self.assertEqual(slot, 1)
self.assertEqual(list(scheduler.queue), [older])
self.assertEqual(scheduler.free_slots, {0, 1})
def test_older_any_slot_and_same_slot_waiters_keep_priority(self):
for older_slot, requested in ((None, None), (None, 1), (0, 0)):
with self.subTest(older_slot=older_slot, requested=requested):
scheduler = GenerationScheduler(capacity=2)
older = (object(), older_slot)
scheduler.queue.append(older)
with patch.object(scheduler.condition, "wait",
side_effect=lambda _timeout: scheduler.queue.remove(older)) as wait:
with scheduler.admit(slot=requested) as (_, slot):
self.assertEqual(slot, 0 if requested is None else requested)
wait.assert_called_once()
self.assertEqual(scheduler.snapshot()["queued"], 0)
def test_full_queue_still_admits_unreserved_free_slot(self):
for requested in (None, 1):
with self.subTest(requested=requested):
scheduler = GenerationScheduler(max_queue=1, capacity=2)
with scheduler.admit(slot=0):
older = (object(), 0)
scheduler.queue.append(older)
try:
with scheduler.admit(slot=requested) as (_, slot):
self.assertEqual(slot, 1)
self.assertEqual(scheduler.snapshot()["active"], 2)
self.assertEqual(list(scheduler.queue), [older])
finally:
scheduler.queue.remove(older)
self.assertEqual(scheduler.snapshot()["rejected"], 0)
self.assertEqual(scheduler.snapshot()["completed"], 2)
def test_full_queue_does_not_bypass_older_slot_reservations(self):
for older_slot, requested in ((None, None), (None, 1), (0, 0)):
with self.subTest(older_slot=older_slot, requested=requested):
scheduler = GenerationScheduler(max_queue=1, capacity=2)
older = (object(), older_slot)
scheduler.queue.append(older)
with self.assertRaises(APIError) as caught:
with scheduler.admit(slot=requested):
self.fail("bypassed older waiter")
self.assertEqual(caught.exception.code, "queue_full")
self.assertEqual(list(scheduler.queue), [older])
self.assertEqual(scheduler.snapshot()["rejected"], 1)
def test_zero_queue_rejects_busy_pinned_slot_with_spare_capacity(self):
scheduler = GenerationScheduler(max_queue=0, queue_timeout=0.01, capacity=2)
with scheduler.admit(slot=0):
with self.assertRaises(APIError) as caught:
with scheduler.admit(slot=0):
self.fail("busy pinned slot admitted")
self.assertEqual(caught.exception.code, "queue_full")
with scheduler.admit(slot=1) as (_, slot):
self.assertEqual(slot, 1)
stats = scheduler.snapshot()
self.assertEqual((stats["rejected"], stats["timed_out"], stats["queued"]), (1, 0, 0))
self.assertEqual((stats["admitted"], stats["completed"], stats["active"]), (2, 2, 0))
self.assertIn("colibri_scheduler_queue_wait_seconds_count 2\n", scheduler.prometheus())
def test_times_out_and_cancels_queued_requests(self):
scheduler = GenerationScheduler(max_queue=2, queue_timeout=0.02)
with scheduler.admit():
with self.assertRaises(APIError) as timed_out:
with scheduler.admit():
pass
with self.assertRaises(ClientCancelled):
with scheduler.admit(lambda: True):
pass
stats = scheduler.snapshot()
self.assertEqual(timed_out.exception.code, "queue_timeout")
self.assertEqual(stats["timed_out"], 1)
self.assertEqual(stats["cancelled"], 1)
def test_queue_deadline_wins_when_slot_becomes_free(self):
for released_at in (0.9, 1.0, 1.1):
with self.subTest(released_at=released_at):
scheduler = GenerationScheduler(queue_timeout=1)
now = [0.0]
with patch("openai_server.time.monotonic", side_effect=lambda: now[0]):
holder = scheduler.admit()
holder.__enter__()
def release(_timeout):
now[0] = released_at
holder.__exit__(None, None, None)
with patch.object(scheduler.condition, "wait", side_effect=release):
if released_at < 1:
with scheduler.admit():
pass
else:
with self.assertRaises(APIError) as caught:
with scheduler.admit():
pass
self.assertEqual(caught.exception.code, "queue_timeout")
stats = scheduler.snapshot()
expected = 2 if released_at < 1 else 1
self.assertEqual((stats["admitted"], stats["completed"]), (expected, expected))
self.assertEqual((stats["active"], stats["queued"], stats["timed_out"]),
(0, 0, int(released_at >= 1)))
self.assertIn(f"colibri_scheduler_slot_duration_seconds_count {expected}\n",
scheduler.prometheus())
with scheduler.admit():
pass
def test_cancelled_request_does_not_acquire_a_free_slot(self):
scheduler = GenerationScheduler()
with self.assertRaises(ClientCancelled):
with scheduler.admit(lambda: True):
self.fail("cancelled request admitted")
stats = scheduler.snapshot()
self.assertEqual((stats["active"], stats["queued"], stats["admitted"], stats["cancelled"]),
(0, 0, 0, 1))
self.assertIn("colibri_scheduler_queue_wait_seconds_count 0\n", scheduler.prometheus())
with scheduler.admit():
pass
def test_cancellation_wins_when_a_waiting_slot_becomes_free(self):
scheduler = GenerationScheduler(queue_timeout=1)
waiting = threading.Event()
cancelled = threading.Event()
outcomes = []
holder = scheduler.admit()
holder.__enter__()
def is_cancelled():
waiting.set()
return cancelled.is_set()
def run():
try:
with scheduler.admit(is_cancelled):
outcomes.append("admitted")
except ClientCancelled:
outcomes.append("cancelled")
thread = threading.Thread(target=run)
thread.start()
observed = waiting.wait(1)
# Publish cancellation and release capacity under the same lock, so
# the waiter must observe both on its next scheduling pass.
with scheduler.condition:
cancelled.set()
holder.__exit__(None, None, None)
thread.join(2)
self.assertTrue(observed)
self.assertFalse(thread.is_alive())
self.assertEqual(outcomes, ["cancelled"])
stats = scheduler.snapshot()
self.assertEqual((stats["admitted"], stats["completed"], stats["cancelled"]), (1, 1, 1))
self.assertEqual((stats["active"], stats["queued"]), (0, 0))
self.assertIn("colibri_scheduler_slot_duration_seconds_count 1\n", scheduler.prometheus())
def test_counts_admitted_client_cancellation_without_completion(self):
scheduler = GenerationScheduler(max_queue=0, queue_timeout=1)
with self.assertRaises(ClientCancelled):
with scheduler.admit():
raise ClientCancelled()
stats = scheduler.snapshot()
self.assertEqual(stats["active"], 0)
self.assertEqual(stats["admitted"], 1)
self.assertEqual(stats["completed"], 0)
self.assertEqual(stats["cancelled"], 1)
with scheduler.admit():
pass
self.assertEqual(scheduler.snapshot()["completed"], 1)
def test_admits_waiters_in_fifo_order(self):
scheduler = GenerationScheduler(max_queue=2, queue_timeout=1)
entered = threading.Event()
release = threading.Event()
order = []
def run(name, block=False):
with scheduler.admit():
order.append(name)
if block:
entered.set()
release.wait(1)
first = threading.Thread(target=run, args=("first", True))
second = threading.Thread(target=run, args=("second",))
third = threading.Thread(target=run, args=("third",))
first.start(); entered.wait(1)
second.start()
for _ in range(100):
if scheduler.snapshot()["queued"] == 1: break
threading.Event().wait(0.005)
third.start()
for _ in range(100):
if scheduler.snapshot()["queued"] == 2: break
threading.Event().wait(0.005)
release.set()
first.join(1); second.join(1); third.join(1)
self.assertEqual(order, ["first", "second", "third"])
self.assertEqual(scheduler.snapshot()["completed"], 3)
def test_close_rejects_waiters(self):
scheduler = GenerationScheduler(max_queue=1, queue_timeout=1)
entered = threading.Event()
release = threading.Event()
errors = []
def active():
with scheduler.admit():
entered.set(); release.wait(1)
def waiting():
try:
with scheduler.admit(): pass
except APIError as error:
errors.append(error.code)
first = threading.Thread(target=active); first.start(); entered.wait(1)
second = threading.Thread(target=waiting); second.start()
scheduler.close(); release.set(); first.join(1); second.join(1)
self.assertEqual(errors, ["scheduler_closed"])
class BlockingStream:
def __init__(self, initial=b""):
self.buffer = bytearray(initial)
self.closed = False
self.condition = threading.Condition()
def feed(self, data):
with self.condition:
self.buffer.extend(data)
self.condition.notify_all()
def read(self, size=1):
with self.condition:
while len(self.buffer) < size and not self.closed:
self.condition.wait()
if not self.buffer and self.closed:
return b""
size = min(size, len(self.buffer))
data = bytes(self.buffer[:size])
del self.buffer[:size]
return data
def readline(self):
with self.condition:
while b"\n" not in self.buffer and not self.closed:
self.condition.wait()
if not self.buffer and self.closed:
return b""
end = self.buffer.find(b"\n")
size = len(self.buffer) if end < 0 else end + 1
data = bytes(self.buffer[:size])
del self.buffer[:size]
return data
def close(self):
with self.condition:
self.closed = True
self.condition.notify_all()
class FakeProcess:
def __init__(self, on_write):
self.stdout = BlockingStream(READY + b"STAT 0 0 0 0\n")
self.stdin = self
self.on_write = on_write
self.writes = []
self.returncode = None
def write(self, data):
# `_write_all` (production) hands every write a `memoryview` slice,
# first write included -- the fakes below pattern-match frame bytes
# (`frame.split()`, equality against a literal), so normalise here
# rather than asking each one to know about the view.
data = bytes(data)
self.writes.append(data)
self.on_write(self, data)
return len(data)
def flush(self):
pass
def poll(self):
return self.returncode
def terminate(self):
self.returncode = 0
self.stdout.close()
def wait(self, timeout=None):
return self.returncode
def kill(self):
self.terminate()
class DispatcherTest(unittest.TestCase):
def test_inkling_audio_request_and_response_transcript_is_byte_exact(self):
prompt = "<|message_user|><|content_audio_input|><|audio|><|end_message|>"
payload = prompt.encode("utf-8")
audio = bytes(range(16)) * 5
expected = (f"SUBMIT 1 0 {len(payload)} 4 0.25 0.9 {len(audio)}\n".encode() +
payload + audio + b"\n")
def respond(process, frame):
self.assertEqual(frame, expected)
process.stdout.feed(
b"DATA 1 4\nA\n\xc3\xa9\n"
b"DONE 1 STAT 1 2.500 50.0 1.25 7 0\n"
)
process = FakeProcess(respond)
with patch("openai_server.ARCH", "inkling"), \
patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("inkling", "model")
chunks = []
stats = engine.generate(prompt, 4, 0.25, 0.9, chunks.append,
audio=audio)
engine.close()
self.assertEqual(process.writes, [expected])
self.assertEqual(chunks, ["A\né"])
self.assertEqual(stats["prompt_tokens"], 7)
def test_v4_request_and_response_transcript_is_byte_exact(self):
prompt = "<|begin▁of▁sentence|>System<|User|>Hello<|Assistant|>"
payload = prompt.encode("utf-8")
prefix = len("<|begin▁of▁sentence|>System".encode("utf-8"))
expected = (f"SUBMIT 1 0 {len(payload)} 4 0.25 0.9 0 {prefix}\n".encode() +
payload + b"\n")
def respond(process, frame):
self.assertEqual(frame, expected)
process.stdout.feed(
b"ACCEPT 1 42\n"
b"DATA 1 4\nA\n\xc3\xa9\n"
b"DONE 1 STAT 1 2.500 50.0 1.25 42 0 17\n"
)
process = FakeProcess(respond)
with patch("openai_server.ARCH", "deepseek_v4"), \
patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("deepseek_v4", "model")
chunks = []
stats = engine.generate(prompt, 4, 0.25, 0.9, chunks.append)
engine.close()
self.assertEqual(process.writes, [expected])
self.assertEqual(chunks, ["A\né"])
self.assertEqual(stats["completion_tokens"], 1)
self.assertEqual(stats["prompt_tokens"], 42)
def test_kimi_request_and_response_transcript_is_byte_exact(self):
prompt = render_chat_kimi([
{"role": "system", "content": "Be precise."},
{"role": "user", "content": "你好\nKimi"},
{"role": "assistant", "reasoning_content": "because",
"content": "你好。"},
{"role": "user", "content": "Continue"},
], enable_thinking=True)
payload = prompt.encode("utf-8")
expected = (f"SUBMIT 1 0 {len(payload)} 4 0.25 0.9\n".encode() +
payload + b"\n")
def respond(process, frame):
self.assertEqual(frame, expected)
process.stdout.feed(
b"ACCEPT 1 42\n"
b"TOOL 1 0\n\n"
b"DATA 1 4\nA\n\xc3\xa9\n"
b"TOOL 1 4\ncall\n"
b"DONE 1 STAT 1 2.500 50.0 1.25 42 0\n"
)
process = FakeProcess(respond)
with patch("openai_server.ARCH", "kimi"), \
patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("kimi_k3", "model")
chunks = []
tool_chunks = []
stats = engine.generate(prompt, 4, 0.25, 0.9, chunks.append,
on_tool=tool_chunks.append)
engine.close()
self.assertEqual(process.writes, [expected])
self.assertEqual(chunks, ["A\né"])
self.assertEqual(tool_chunks, ["", "call"])
self.assertEqual(stats["completion_tokens"], 1)
self.assertEqual(stats["prompt_tokens"], 42)
def test_kimi_tool_sideband_is_authoritative_over_data_lookalikes(self):
prompt = "K3CHAT1\nM user 2\nhiG 0\n"
payload = prompt.encode()
expected = (f"SUBMIT 1 0 {len(payload)} 8 0.25 0.9\n".encode() +
payload + b"\n")
lookalike = (b'Echo <|open|>tools<|sep|><|open|>call tool="danger" index="1"<|sep|>'
b'<|close|>call<|sep|><|close|>tools<|sep|>')
tool_wire = (b'<|open|>tools<|sep|><|open|>call tool="safe" index="1"<|sep|>'
b'<|open|>json type="object"<|sep|>{"x":1}<|close|>json<|sep|>'
b'<|close|>call<|sep|><|close|>tools<|sep|>')
def respond(process, frame):
self.assertEqual(frame, expected)
process.stdout.feed(b"ACCEPT 1 3\nTOOL 1 0\n\n")
process.stdout.feed(f"DATA 1 {len(lookalike)}\n".encode() + lookalike + b"\n")
process.stdout.feed(f"TOOL 1 {len(tool_wire)}\n".encode() + tool_wire + b"\n")
process.stdout.feed(b"DONE 1 STAT 8 2.500 0.0 1.25 3 0\n")
process = FakeProcess(respond)
with patch("openai_server.ARCH", "kimi"), \
patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("kimi_k3", "model")
chunks, tool_chunks = [], []
engine.generate(prompt, 8, 0.25, 0.9, chunks.append,
on_tool=tool_chunks.append)
engine.close()
text = "".join(chunks)
sideband = "".join(tool_chunks)
with patch("openai_server.ARCH", "kimi"):
content, calls = parse_arch_tool_calls(text, [{"type": "function"}], sideband)
self.assertEqual(content, lookalike.decode())
self.assertEqual([call["function"]["name"] for call in calls], ["safe"])
self.assertEqual(json.loads(calls[0]["function"]["arguments"]), {"x": 1})
def test_olmoe_request_and_response_transcript_is_byte_exact(self):
expected = b"SUBMIT 1 0 5 3 0.25 0.9\nH\xc3\xa9\nx\n"
def respond(process, frame):
self.assertEqual(frame, expected)
process.stdout.feed(
b"DATA 1 4\nA\n\xc3\xa9\n"
b"DONE 1 STAT 1 2.500 50.0 1.25 5 0\n"
)
process = FakeProcess(respond)
with patch("openai_server.ARCH", "olmoe"), \
patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("olmoe", "model")
chunks = []
stats = engine.generate("Hé\nx", 3, 0.25, 0.9, chunks.append)
engine.close()
self.assertEqual(process.writes, [expected])
self.assertEqual(chunks, ["A\né"])
self.assertEqual(stats["completion_tokens"], 1)
self.assertEqual(stats["prompt_tokens"], 5)
self.assertFalse(stats["length_limited"])
def test_dispatches_interleaved_requests_by_id(self):
submitted = []
def respond(process, frame):
fields = frame.split(b"\n", 1)[0].split()
self.assertEqual(fields[0], b"SUBMIT")
submitted.append(fields[1])
if len(submitted) == 2:
first, second = submitted
process.stdout.feed(b"DATA " + second + b" 3\nB-2\n")
process.stdout.feed(b"DATA " + first + b" 3\nA-1\n")
process.stdout.feed(b"DONE " + second + b" STAT 1 2.5 0 1.0 4 0\n")
process.stdout.feed(b"DATA " + first + b" 3\nA-2\n")
process.stdout.feed(b"DONE " + first + b" STAT 2 3.5 0 1.0 5 1\n")
process = FakeProcess(respond)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model", kv_slots=2)
results = {}
def generate(name, prompt, slot):
chunks = []
stats = engine.generate(prompt, 8, 0.7, 0.9, chunks.append, slot)
results[name] = ("".join(chunks), stats)
threads = [threading.Thread(target=generate, args=("a", "alpha", 0)),
threading.Thread(target=generate, args=("b", "beta", 1))]
for thread in threads:
thread.start()
for thread in threads:
thread.join(timeout=2)
self.assertFalse(thread.is_alive())
engine.close()
self.assertEqual(results["a"][0], "A-1A-2")
self.assertTrue(results["a"][1]["length_limited"])
self.assertEqual(results["b"][0], "B-2")
headers = [frame.split(b"\n", 1)[0].split() for frame in process.writes]
self.assertEqual({int(header[2]) for header in headers}, {0, 1})
self.assertEqual({header[3] for header in headers}, {b"4", b"5"})
def test_routes_engine_error_to_request(self):
def respond(process, frame):
request_id = frame.split()[1]
process.stdout.feed(b"ERROR " + request_id + b" slot is busy\n")
process = FakeProcess(respond)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
with self.assertRaisesRegex(RuntimeError, "slot is busy"):
engine.generate("hello", 4, 0.7, 0.9, lambda _: None)
engine.close()
def test_close_wakes_pending_generation_and_is_idempotent(self):
process = FakeProcess(lambda _process, _frame: None)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
errors = []
def generate():
try:
engine.generate("hello", 4, 0.7, 0.9, lambda _: None)
except RuntimeError as error:
errors.append(str(error))
thread = threading.Thread(target=generate)
thread.start()
for _ in range(100):
with engine.pending_lock:
if engine.pending:
break
threading.Event().wait(0.01)
engine.close()
engine.close()
thread.join(timeout=2)
self.assertFalse(thread.is_alive())
self.assertEqual(errors, ["colibri engine is shutting down"])
self.assertFalse(engine.dispatcher.is_alive())
with engine.pending_lock:
self.assertFalse(engine.pending)
with self.assertRaisesRegex(RuntimeError, "shutting down"):
engine.generate("again", 4, 0.7, 0.9, lambda _: None)
def test_protocol_corruption_fails_request_and_stops_dispatcher(self):
def respond(process, frame):
request_id = frame.split()[1]
process.stdout.feed(b"DATA " + request_id + b" -1\n")
process = FakeProcess(respond)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
with self.assertRaisesRegex(RuntimeError, "DATA size"):
engine.generate("hello", 4, 0.7, 0.9, lambda _: None)
# the cause comes with every later failure, not only "dispatcher stopped" (#1941)
with self.assertRaisesRegex(RuntimeError, "dispatcher stopped: .*DATA size"):
engine.generate("again", 4, 0.7, 0.9, lambda _: None)
engine.close()
def test_an_engine_that_dies_names_its_exit_status(self):
def respond(process, frame):
process.returncode = -9 # the out-of-memory killer, as #1941 may have met
process.stdout.close()
process = FakeProcess(respond)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
with self.assertRaisesRegex(RuntimeError, "exited unexpectedly .*signal 9, SIGKILL"):
engine.generate("hello", 4, 0.7, 0.9, lambda _: None)
with self.assertRaisesRegex(RuntimeError, "dispatcher stopped: .*signal 9"):
engine.generate("again", 4, 0.7, 0.9, lambda _: None)
engine.close()
def test_engine_exit_status_words(self):
self.assertEqual(engine_exit_status(1), "exit status 1")
self.assertIn("SIGKILL", engine_exit_status(-9))
self.assertEqual(engine_exit_status(0xC0000005), "exit status 0xC0000005, access violation")
self.assertEqual(engine_exit_status(0xC0000017), "exit status 0xC0000017, out of memory")
def test_decodes_utf8_split_across_data_frames(self):
def respond(process, frame):
request_id = frame.split()[1]
process.stdout.feed(b"DATA " + request_id + b" 1\n\xc3\n")
process.stdout.feed(b"DATA " + request_id + b" 1\n\xa9\n")
process.stdout.feed(b"DONE " + request_id + b" STAT 1 1 0 1 1 0\n")
process = FakeProcess(respond)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
chunks = []
engine.generate("hello", 4, 0.7, 0.9, chunks.append)
engine.close()
self.assertEqual(chunks, ["é"])
def test_records_profile_snapshots_from_prof_lines(self):
def respond(process, frame):
request_id = frame.split()[1]
process.stdout.feed(b"DATA " + request_id + b" 2\nok\n")
process.stdout.feed(b"PROF 2.500 7 12 0.400 0.100 0.900 0.600 0.200 15\n")
process.stdout.feed(b"DONE " + request_id + b" STAT 12 4.8 0 1.0 7 0\n")
process = FakeProcess(respond)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
engine.generate("hello", 16, 0.7, 0.9, lambda _: None)
engine.close()
self.assertEqual(engine.profile_seq, 1)
self.assertEqual(list(engine.profile), [{
"wall_s": 2.5, "prompt_tokens": 7, "completion_tokens": 12,
"expert_disk_s": 0.4, "expert_wait_s": 0.1, "expert_matmul_s": 0.9,
"attention_s": 0.6, "lm_head_s": 0.2, "forwards": 15,
}])
def test_accepts_u7a_echo_and_extended_data_frames(self):
# U7a forward-compat: the engine's opt-in per-token numeric channel --
# ECHO frames for echoed prompt positions and DATA frames extended
# with "<lp> <k> [tid tlp]*k" -- must NOT trip the dispatcher's
# catch-all (which kills every in-flight request). Text delivery and
# the DONE stats stay exactly as for legacy frames; the numeric
# fields are consumed by the server feature half (U7b).
def respond(process, frame):
request_id = frame.split()[1]
process.stdout.feed(b"ACCEPT " + request_id + b" 3\n")
process.stdout.feed(b"ECHO " + request_id + b" 2 0 nan 0\nHi\n")
process.stdout.feed(
b"ECHO " + request_id +
b" 6 1 -0.105361 2 7 -0.105361 9 -2.302585\n world\n")
process.stdout.feed(
b"ECHO " + request_id +
b" 1 2 -1.203973 2 4 -0.803973 6 -1.203973\n!\n")
process.stdout.feed(
b"DATA " + request_id +
b" 2 -0.223144 2 3 -0.223144 8 -1.723144\nok\n")
process.stdout.feed(
b"DONE " + request_id + b" STAT 1 2.5 0 1.0 3 0\n")
process = FakeProcess(respond)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
chunks = []
stats = engine.generate("Hi world!", 4, 0.0, 1.0, chunks.append)
self.assertEqual(chunks, ["ok"])
self.assertEqual(stats["completion_tokens"], 1)
self.assertEqual(stats["prompt_tokens"], 3)
self.assertIsNone(engine.dispatcher_error)
engine.close()
def test_unknown_frame_still_stops_dispatcher(self):
# The catch-all that makes an unrecognized frame a hard failure is
# load-bearing for the U7a compatibility asymmetry (a new engine's
# frame reaching an OLD server kills the dispatcher -- the reason the
# engine half ships first and stays opt-in). Accepting ECHO/extended
# DATA must not have widened acceptance beyond those frames.
def respond(process, frame):
request_id = frame.split()[1]
process.stdout.feed(b"LOGPROB " + request_id + b" 0.5\n")
process = FakeProcess(respond)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
with self.assertRaisesRegex(RuntimeError, "invalid engine response: LOGPROB"):
engine.generate("hello", 4, 0.7, 0.9, lambda _: None)
engine.close()
def test_cancels_generation_after_consumer_disconnects(self):
request_id = None
def respond(process, frame):
nonlocal request_id
fields = frame.split()
if fields[0] == b"SUBMIT":
request_id = fields[1]
process.stdout.feed(b"DATA " + request_id + b" 1\nx\n")
elif fields[0] == b"CANCEL":
self.assertEqual(fields[1], request_id)
process.stdout.feed(b"ERROR " + request_id + b" CANCELLED\n")
process = FakeProcess(respond)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
output = []
# Il consumatore se ne va DOPO aver ricevuto qualcosa: e' lo scenario che
# il nome promette, e va costruito invece che sperato. Con cancelled che
# tornava True da subito, la corsa era fra il ramo idle -- che cancella
# prima di ogni dato -- e l'arrivo del frame: su Windows vinceva il primo
# e l'asserzione su output falliva senza che nulla fosse rotto (#1328).
# Qui il flag si alza dentro il sink, e nel ramo "data" decode() consegna
# il token PRIMA che cancelled() venga interrogato: l'ordine e' garantito
# dal codice, non dallo scheduler.
#
# Il tetto sui poll non e' decorativo. Senza, un guasto che impedisce la
# consegna del token lascerebbe il flag basso per sempre: niente cancel,
# niente eccezione, e il test si APPENDE invece di fallire -- misurato
# rompendo decode() apposta. Il ramo idle interroga cancelled() ogni 50 ms,
# quindi dopo un secondo si cancella comunque e l'asserzione su output
# fallisce subito, dicendo la cosa giusta. Deterministico quando funziona,
# rapido a fallire quando no.
#
# Il caso "cancella prima del primo frame" resta coperto, in modo
# deterministico, da test_cancels_generation_before_first_frame (#908):
# prima i due si sovrapponevano a caso e uno dei due vinceva a sorte.
disconnected = False
polls = 0
def sink(text):
nonlocal disconnected
output.append(text)
disconnected = True
def consumer_gone():
nonlocal polls
polls += 1
return disconnected or polls > 20
with self.assertRaises(ClientCancelled):
engine.generate("hello", 8, 0.7, 0.9, sink, cancelled=consumer_gone)
engine.close()
self.assertEqual(output, ["x"])
self.assertEqual(process.writes[-1].split(), [b"CANCEL", request_id])
def test_cancels_generation_before_first_frame(self):
# #908: a client that disconnects while the engine is still prefilling
# (no DATA frame has arrived) must cancel too. cancelled() used to be
# polled only in the "data" branch, so the CANCEL never went out and
# the turn ran to its token limit while this thread stayed blocked.
# The fake engine emits nothing until it sees CANCEL -- exactly the
# pre-first-frame regime -- and must still get one.
request_id = None
def respond(process, frame):
nonlocal request_id
fields = frame.split()
if fields[0] == b"SUBMIT":
request_id = fields[1]
elif fields[0] == b"CANCEL":
self.assertEqual(fields[1], request_id)
process.stdout.feed(b"ERROR " + request_id + b" CANCELLED\n")
process = FakeProcess(respond)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
flag = {"cancelled": False}
outcome = []
def generate():
try:
engine.generate("hello", 8, 0.7, 0.9, lambda _: None,
cancelled=lambda: flag["cancelled"])
except ClientCancelled:
outcome.append("cancelled")
thread = threading.Thread(target=generate)
thread.start()
for _ in range(200):
if any(frame.startswith(b"SUBMIT") for frame in process.writes):
break
time.sleep(0.01)
flag["cancelled"] = True
thread.join(timeout=2)
self.assertFalse(thread.is_alive())
engine.close()
self.assertEqual(outcome, ["cancelled"])
self.assertEqual(process.writes[-1].split(), [b"CANCEL", request_id])
def test_stops_generation_through_successful_done_path(self):
request_id = None
def respond(process, frame):
nonlocal request_id
fields = frame.split()
if fields[0] == b"SUBMIT":
request_id = fields[1]
process.stdout.feed(b"DATA " + request_id + b" 1\nx\n")
elif fields[0] == b"STOP":
self.assertEqual(fields[1], request_id)
process.stdout.feed(b"DONE " + request_id + b" STAT 1 1 0 1 2 0\n")
process = FakeProcess(respond)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
output = []
stats = engine.generate("hello", 8, 0.7, 0.9, output.append,
stopped=lambda: output == ["x"])
engine.close()
self.assertEqual(output, ["x"])
self.assertEqual(stats["completion_tokens"], 1)
self.assertEqual(process.writes[-1].split(), [b"STOP", request_id])
def _capture_frames(body, path="/v1/completions"):
"""Sends `body` to `path` against a FakeProcess-backed Engine/APIServer
and returns (status, parsed_response, frames_written_to_the_engine).
Shared by the test classes below so the harness lives in one place.
"""
frames = []
def respond(process, frame):
frames.append(frame)
rid = frame.split()[1]
process.stdout.feed(b"DATA " + rid + b" 5\nHello\n")
process.stdout.feed(b"DONE " + rid + b" STAT 1 2.5 0 1.0 4 0\n")
process = FakeProcess(respond)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
server = APIServer(("127.0.0.1", 0), engine, "test-model", "secret", 16)
thread = threading.Thread(target=server.serve_forever, args=(0.01,), daemon=True)
thread.start()
try:
data = json.dumps(body).encode()
headers = {"Authorization": "Bearer secret", "Content-Type": "application/json"}
request = Request(f"http://127.0.0.1:{server.server_port}{path}",
data=data, headers=headers)
with urlopen(request, timeout=2) as response:
status = response.status
parsed = json.load(response)
finally:
server.scheduler.close()
server.shutdown()
server.server_close()
thread.join(timeout=2)
engine.close()
return status, parsed, frames
class BaseWireContractTest(unittest.TestCase):
"""Pins the SUBMIT frame written for a request that uses none of the
optional fields; any change here is a wire-format change and must be
deliberate.
"""
def test_no_new_fields_request_emits_the_base_submit_header(self):
_, _, frames = _capture_frames(
{"model": "test-model", "prompt": "Complete me", "temperature": 0, "max_tokens": 4})
self.assertEqual(frames, [b"SUBMIT 1 0 11 4 0 0.9\nComplete me\n"])
class OpenAIHonestySetTest(unittest.TestCase):
def test_refuses_unsupported_result_shaping_fields(self):
cases = (
({"best_of": 2}, "best_of", "unsupported_value"),
({"logit_bias": {"42": 100}}, "logit_bias", "unsupported_value"),
({"suffix": " after"}, "suffix", "unsupported_parameter"),
({"modalities": ["audio"]}, "modalities", "unsupported_value"),
)
for fields, param, code in cases:
with self.subTest(param=param), self.assertRaises(APIError) as caught:
generation_options(fields, 16)
self.assertEqual(caught.exception.status, 400)
self.assertEqual(caught.exception.param, param)
self.assertEqual(caught.exception.code, code)
self.assertIn(f"`{param}`", caught.exception.message)
def test_accepts_noop_values(self):
generation_options({"best_of": 1, "logit_bias": {}, "suffix": None,
"modalities": ["text"]}, 16)
def test_modalities_rejects_unsupported_or_malformed_values(self):
cases = (
("audio", "invalid_value"),
([], "invalid_value"),
([7], "invalid_value"),
(["video"], "unsupported_value"),
(["text", "video"], "unsupported_value"),
)
for value, code in cases:
with self.subTest(value=value):
with self.assertRaises(APIError) as caught:
generation_options({"modalities": value}, 16)
self.assertEqual(caught.exception.status, 400)
self.assertEqual(caught.exception.param, "modalities")
self.assertEqual(caught.exception.code, code)
def test_intentionally_ignored_fields_return_200_and_do_not_reach_engine(self):
base = {"model": "test-model", "prompt": "Complete me",
"temperature": 0, "max_tokens": 4}
ignored = {
"store": True,
"metadata": {"trace": "request-1"},
"service_tier": "default",
"user": "user-1",
"safety_identifier": "safe-1",
"parallel_tool_calls": True,
"prompt_cache_key": "cache-1",
"verbosity": "low",
"web_search_options": {},
"moderation": True,
"stream_options": {"include_obfuscation": True},
}
status_plain, _, frames_plain = _capture_frames(base)
status_ignored, _, frames_ignored = _capture_frames({**base, **ignored})
self.assertEqual(status_plain, 200)
self.assertEqual(status_ignored, 200)
self.assertEqual(frames_ignored, frames_plain)
class SeedOptionTest(unittest.TestCase):
"""`generation_options()` accepts a `seed` field without raising."""
def test_seed_is_accepted_by_generation_options(self):
generation_options({"seed": 1234, "prompt": "hi"}, 16) # must not raise
class SeedWireFrameTest(unittest.TestCase):
"""`seed` is accepted and ignored. A stub-response equality check alone
is vacuous here (the scripted `respond` closure inside `_capture_frames`
always returns the same canned text regardless of any request field) --
the real proof is that the byte-exact SUBMIT frames the dispatcher
writes to the engine process (see DispatcherTest above) never carry the
seed value at all, seeded or not.
"""
def test_seed_accepted_and_absent_from_submit_frame(self):
base = {"model": "test-model", "prompt": "Complete me", "temperature": 0, "max_tokens": 4}
status_plain, body_plain, frames_plain = _capture_frames(base)
status_seeded, body_seeded, frames_seeded = _capture_frames({**base, "seed": 1234})
self.assertEqual(status_plain, 200)
self.assertEqual(status_seeded, 200)
self.assertEqual(body_seeded["choices"][0], body_plain["choices"][0])
# Each call uses a freshly-constructed Engine, so both first requests are
# assigned request id "1" -- the wire frames are directly byte-comparable,
# no field needs normalizing. Comparing the whole frame list (not just
# the first frame) closes "reaches no wire frame" literally: if `seed`
# ever leaked onto any frame, this equality would break.
self.assertEqual(frames_seeded, frames_plain)
def test_seed_accepted_and_absent_from_chat_submit_frame(self):
# Same proof as above, on /v1/chat/completions: generation_options()
# is shared by both endpoints, but the SUBMIT frame is built from
# the chat-rendered prompt, so this is not implied by the completions
# case above -- a divergence between the two call sites would only
# show up here.
base = {"model": "test-model", "messages": [{"role": "user", "content": "Hi"}],
"temperature": 0, "max_tokens": 4}
status_plain, body_plain, frames_plain = _capture_frames(base, path="/v1/chat/completions")
status_seeded, body_seeded, frames_seeded = _capture_frames(
{**base, "seed": 1234}, path="/v1/chat/completions")
self.assertEqual(status_plain, 200)
self.assertEqual(status_seeded, 200)
self.assertEqual(body_seeded["choices"][0], body_plain["choices"][0])
self.assertEqual(frames_seeded, frames_plain)
class CapSentinelShimTest(unittest.TestCase):
# #379 cap-sentinel shim, arch-keyed (#386 r2, F3): an absent cap is
# "platform-auto" only for the glm engine (colibri.c coli_resolve_cap);
# inkling reads cap <= 0 as "fit the expert LRU to all available RAM"
# (inkling.c), so leaking the sentinel to a non-glm arch silently changes
# its memory behavior. The key is the MODEL's config.json model_type, not
# the engine binary's file name -- COLI_ENGINE users package the glm
# engine as glm52/colibri-1.2/glm-metal, and basename keying disabled the
# platform default for exactly them (and an inkling binary someone names
# `glm` would get the leak back). Engine() is the one funnel every launch
# passes through, so the translation is pinned at the argv it emits, over
# the full matrix: arch x arbitrary executable name x cap absent/explicit.
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
def _model(self, model_type):
model = Path(self.tmp.name) / f"model-{model_type}"
model.mkdir(exist_ok=True)
(model / "config.json").write_text(json.dumps({"model_type": model_type}))
return str(model)
def _spawn_argv(self, executable, model, **kwargs):
process = FakeProcess(lambda _process, _frame: None)
with patch("openai_server.subprocess.Popen", return_value=process) as popen:
engine = Engine(executable, model, **kwargs)
engine.close()
return popen.call_args[0][0]
def test_matrix_arch_times_engine_name_times_cap(self):
# COLI_ENGINE axis: the executable name carries no information; only
# the model arch and the explicitness of cap may matter.
executables = ("colibri", "glm", "glm52", "colibri-1.2", "glm-metal",
"/opt/custom/inkling", "kimi_k3.exe")
cases = ( # (model_type, cap kwargs, expected argv cap)
("glm_moe_dsa", {}, "0"), # absent -> glm sentinel
("inkling", {}, "8"), # absent -> legacy 8
("kimi_k3", {}, "8"), # absent -> legacy 8
("glm_moe_dsa", {"cap": 5}, "5"), # explicit -> verbatim
("inkling", {"cap": 5}, "5"),
("kimi_k3", {"cap": 5}, "5"),
("glm_moe_dsa", {"cap": 0}, "0"), # explicit 0 -> verbatim
("inkling", {"cap": 0}, "0"), # upstream RAM-auto, by request
("kimi_k3", {"cap": 0}, "0"),
)
for model_type, kwargs, want in cases:
model = self._model(model_type)
for executable in executables:
self.assertEqual(
self._spawn_argv(executable, model, **kwargs),
[executable, want],
f"arch={model_type} exe={executable} kwargs={kwargs}")
def test_synthetic_engine_without_config_uses_explicit_arch(self):
self.assertEqual(self._spawn_argv("engine", "/nonexistent/model"),
["engine", "0"])
model = Path(self.tmp.name) / "model-broken"
model.mkdir()
(model / "config.json").write_text("{not json")
with self.assertRaisesRegex(ValueError, "invalid config.json"):
self._spawn_argv("engine", str(model))
def test_cap_for_arch_is_the_single_translation_point(self):
# 0 is the "you decide" sentinel, and it goes to the engines that
# actually do decide: glm resolves it platform-aware, olmoe sizes its
# expert cache from the RAM budget once the dense weights are resident
# (#1443). The others still get the legacy eight slots per layer.
self.assertEqual(cap_for_arch("glm", None), 0)
self.assertEqual(cap_for_arch("olmoe", None), 0)
self.assertEqual(cap_for_arch("inkling", None), 8)
self.assertEqual(cap_for_arch("kimi", None), 8)
self.assertEqual(cap_for_arch("glm", 3), 3)
self.assertEqual(cap_for_arch("inkling", 3), 3)
self.assertEqual(cap_for_arch("inkling", 0), 0) # explicit 0 is explicit
self.assertEqual(cap_for_arch("glm", 0), 0)
def test_profile_cap_is_below_explicit_cap_and_above_implicit_default(self):
profiled = {"COLI_PROFILE_CAP": "24"}
self.assertEqual(cap_for_arch("inkling", None, profiled), 24)
self.assertEqual(cap_for_arch("inkling", 7, profiled), 7)
self.assertEqual(cap_for_arch("inkling", None,
{"COLI_PROFILE_CAP": "invalid"}), 8)
planned = {"COLI_PLAN_CAP": "11"}
self.assertEqual(cap_for_arch("qwen38", None, planned), 11)
self.assertEqual(cap_for_arch(
"qwen38", None, {"COLI_PLAN_CAP": "11", "COLI_PROFILE_CAP": "7"}), 7)
self.assertEqual(cap_for_arch(
"qwen38", 5, {"COLI_PLAN_CAP": "11", "COLI_PROFILE_CAP": "7"}), 5)
def test_engine_consumes_profile_cap_without_leaking_private_env(self):
process = FakeProcess(lambda _process, _frame: None)
model = self._model("inkling")
with patch("openai_server.subprocess.Popen", return_value=process) as popen:
engine = Engine("custom-engine", model,
env={"COLI_PROFILE_CAP": "24", "KEEP": "yes"})
engine.close()
command = popen.call_args[0][0]
child_env = popen.call_args[1]["env"]
self.assertEqual(command, ["custom-engine", "24"])
self.assertNotIn("COLI_PROFILE_CAP", child_env)
self.assertEqual(child_env["KEEP"], "yes")
def test_engine_consumes_planned_cap_without_leaking_private_env(self):
process = FakeProcess(lambda _process, _frame: None)
model = self._model("qwen4_exp_text")
with patch("openai_server.subprocess.Popen", return_value=process) as popen:
engine = Engine("qwen38", model,
env={"COLI_PLAN_CAP": "13", "KEEP": "yes"})
engine.close()
self.assertEqual(popen.call_args[0][0], ["qwen38", "13"])
child_env = popen.call_args[1]["env"]
self.assertNotIn("COLI_PLAN_CAP", child_env)
self.assertEqual(child_env["KEEP"], "yes")
def test_v41_ram_plan_reaches_engine_argv(self):
model = self._model("deepseek_v41")
for settings, expected_ram in (({"RAM_GB": "120", "CTX": "8192"}, 120),
({"CTX": "8192"}, 0),
({"RAM_GB": "auto", "CTX": "8192"}, 0)):
with self.subTest(settings=settings):
process = FakeProcess(lambda _process, _frame: None)
with patch("resource_plan.build_plan", return_value={
"tiers": {"ram": {"cache_slots_per_layer": 96}}}) as planner, \
patch("openai_server.subprocess.Popen", return_value=process) as popen:
engine = Engine("deepseek_v41", model, env=settings)
engine.close()
planner.assert_called_once_with(model, ram_gb=expected_ram,
context=8192, gpu_indices=[])
self.assertEqual(popen.call_args[0][0], ["deepseek_v41", "96"])
def test_v41_explicit_and_calibrated_caps_bypass_planning(self):
for cap, env, expected in ((7, {}, 7), (0, {}, 0),
(None, {"COLI_PROFILE_CAP": "12"}, 12),
(None, {"COLI_PLAN_CAP": "24"}, 24)):
with self.subTest(cap=cap, env=env), patch("resource_plan.build_plan") as planner:
self.assertEqual(cap_for_arch("deepseek_v41", cap, env, model="model"),
expected)
planner.assert_not_called()
def test_v41_insufficient_ram_refuses_before_spawning(self):
model = self._model("deepseek_v41")
with patch("resource_plan.build_plan", return_value={
"tiers": {"ram": {"cache_slots_per_layer": 0}}}), \
patch("openai_server.subprocess.Popen") as popen:
with self.assertRaisesRegex(ValueError, "one expert slot"):
Engine("deepseek_v41", model, env={"RAM_GB": "8"})
popen.assert_not_called()
def test_model_arch_reads_model_type(self):
self.assertEqual(model_arch(self._model("glm_moe_dsa")), "glm")
self.assertEqual(model_arch(self._model("inkling")), "inkling")
self.assertEqual(model_arch(self._model("kimi_k3")), "kimi")
self.assertEqual(model_arch(self._model("deepseek_v4")), "deepseek_v4")
self.assertEqual(model_arch(self._model("olmoe")), "olmoe")
self.assertEqual(model_arch(self._model("qwen4_exp")), "qwen38")
self.assertEqual(model_arch(self._model("qwen4_exp_text")), "qwen38")
with self.assertRaisesRegex(ValueError, "cannot read config.json"):
model_arch("/nonexistent")
def test_direct_v4_server_gets_bounded_dspark_defaults(self):
env = {"V4_MTP_CONF": "0.7"}
with patch("resource_plan.physical_cpu_count",
side_effect=AssertionError("V4 server sized the team")), \
patch("openai_server.sys.platform", "linux"):
tune_child_env(env, "deepseek_v4")
self.assertNotIn("OMP_NUM_THREADS", env)
self.assertEqual(env["OMP_PROC_BIND"], "close")
self.assertEqual(env["V4_DRAFT"], "0")
self.assertEqual(env["V4_MTP"], "0")
self.assertEqual(env["V4_MTP_DRAFT"], "3")
self.assertEqual(env["V4_MTP_GB"], "0.45")
self.assertEqual(env["V4_MTP_CONF"], "0.7") # explicit override wins
self.assertEqual(env["V4_MTP_GPU"], "0") # GPU drafting opt-in, off by default
def test_direct_v4_server_preserves_explicit_omp_threads(self):
env = {"OMP_NUM_THREADS": "3"}
tune_child_env(env, "deepseek_v4")
self.assertEqual(env["OMP_NUM_THREADS"], "3")
def test_direct_v4_server_honours_omp_kill_switch(self):
env = {"COLI_NO_OMP_TUNE": "1"}
tune_child_env(env, "deepseek_v4")
for key in ("OMP_NUM_THREADS", "OMP_WAIT_POLICY", "GOMP_SPINCOUNT",
"OMP_DYNAMIC", "OMP_PROC_BIND", "OMP_PLACES"):
self.assertNotIn(key, env)
class HTTPTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.engine = FakeEngine()
cls.server = APIServer(("127.0.0.1", 0),cls.engine,"test-model","secret",16,kv_slots=2)
cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True)
cls.thread.start()
cls.base = f"http://127.0.0.1:{cls.server.server_port}"
@classmethod
def tearDownClass(cls):
cls.server.scheduler.close()
cls.server.shutdown()
cls.server.server_close()
cls.thread.join(timeout=2)
def request(self, path, body=None, key="secret"):
headers = {"Authorization": f"Bearer {key}"}
data = None
if body is not None:
data = json.dumps(body).encode()
headers["Content-Type"] = "application/json"
return urlopen(Request(self.base + path, data=data, headers=headers), timeout=2)
def test_lists_models_and_checks_auth(self):
with self.request("/v1/models") as response:
self.assertEqual(json.load(response)["data"][0]["id"], "test-model")
with self.assertRaises(HTTPError) as caught:
self.request("/v1/models", key="wrong")
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 401)
def test_unsupported_modalities_fail_before_engine_work(self):
cases = (
("/v1/completions", {"prompt": "hi", "modalities": ["video"]},
"unsupported_value"),
("/v1/chat/completions", {"messages": [{"role": "user", "content": "hi"}],
"modalities": "audio"}, "invalid_value"),
)
for path, fields, code in cases:
with self.subTest(path=path):
calls_before = len(self.engine.calls)
with self.assertRaises(HTTPError) as caught:
self.request(path, {"model": "test-model", **fields})
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
error = json.load(caught.exception)["error"]
self.assertEqual(error["param"], "modalities")
self.assertEqual(error["code"], code)
self.assertEqual(len(self.engine.calls), calls_before)
def test_a_malformed_tools_or_messages_is_a_client_error_on_every_family(self):
"""`tools: 5` or `messages: null` answered HTTP 500 on most families.
generation_options() validates `tools`, but chat_completion() renders the prompt (and
runs the image expanders) before it calls generation(), and those iterate `tools` and
`messages` without checking them, so the TypeError reached do_POST's catch-all. A
`parameters` that is not an object got past every check and failed in
parse_tool_calls() after the whole generation had run.
"""
import family_registry
cases = [
({"tools": 5}, "tools"),
({"tools": True}, "tools"),
({"tools": [{"type": "function", "function": {"name": "f", "parameters": "x"}}]},
"tools.0.function.parameters"),
({"tools": [{"type": "function", "function": {
"name": "f", "parameters": {"type": "object", "properties": "x"}}}]},
"tools.0.function.parameters.properties"),
({"tools": [{"type": "function", "function": {
"name": "f", "parameters": {"type": "object", "required": "a"}}}]},
"tools.0.function.parameters.required"),
({"messages": None}, "messages"),
({"messages": 5}, "messages"),
]
for arch in family_registry.family_ids():
for extra, param in cases:
with self.subTest(arch=arch, param=param, value=next(iter(extra.values()))):
body = {"model": "test-model",
"messages": [{"role": "user", "content": "hi"}], **extra}
with patch("openai_server.ARCH", arch):
with self.assertRaises(HTTPError) as caught:
self.request("/v1/chat/completions", body)
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
self.assertEqual(json.loads(caught.exception.read())["error"]["param"], param)
def test_metrics_counts_http_engine_failure_without_success(self):
before = self.server.scheduler.snapshot()
with patch.object(self.engine, "generate", side_effect=RuntimeError("injected failure")):
with self.assertRaises(HTTPError) as caught:
self.request("/v1/chat/completions", {
"model": "test-model",
"messages": [{"role": "user", "content": "hello"}], "max_tokens": 1})
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 500)
after = self.server.scheduler.snapshot()
self.assertEqual(after["failed"], before["failed"] + 1)
self.assertEqual(after["completed"], before["completed"])
with self.request("/metrics") as response:
text = response.read().decode()
self.assertIn(f'colibri_scheduler_failed_total {after["failed"]}\n', text)
self.assertIn("colibri_scheduler_active 0\n", text)
def test_metrics_exposes_prometheus_text_with_auth(self):
with self.request("/metrics") as response:
self.assertEqual(response.headers["Content-Type"],
"text/plain; version=0.0.4; charset=utf-8")
self.assertEqual(response.headers["Cache-Control"], "no-store")
text = response.read().decode()
self.assertIn("colibri_scheduler_capacity 2\n", text)
self.assertIn("# TYPE colibri_scheduler_queue_wait_seconds histogram\n", text)
for key in ("wrong", ""):
with self.assertRaises(HTTPError) as caught:
self.request("/metrics", key=key)
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 401)
with self.assertRaises(HTTPError) as caught:
urlopen(self.base + "/metrics", timeout=2)
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 401)
def test_health_reports_scheduler_and_kv_slots(self):
with self.request("/health") as response:
health = json.load(response)
scheduler = health["scheduler"]
# the family, for coli chat's per-family defaults (authed probes only)
self.assertEqual(health["arch"], openai_server.ARCH)
self.assertEqual(scheduler["max_queue"], 8)
self.assertIn("queued", scheduler)
self.assertEqual(health["kv_slots"], 2)
def test_health_reports_the_continuation_switch(self):
"""The web UI shows Continue only when this is true: with the switch off a
trailing assistant turn is answered fresh, which a Continue button would
present as a resumption. Unauthed probes keep the bare liveness shape."""
for value, expected in (("1", True), ("0", False)):
with patch.dict(os.environ, {"COLI_CONTINUE_ASSISTANT": value}), \
self.request("/health") as response:
self.assertIs(json.load(response)["continue_assistant"], expected)
with urlopen(self.base + "/health", timeout=2) as response:
self.assertNotIn("continue_assistant", json.load(response))
def test_profile_requires_auth(self):
"""/profile is served before require_auth(), so it needs its own gate.
This test previously asserted the opposite -- that the telemetry was
readable without a key. That was not a client requirement: the web
dashboard sends `Authorization: Bearer` to /profile exactly as it does
to /health and /experts (web/src/lib/api.ts). The turns carry prompt and
completion token counts and per-phase timings for the last 120 requests,
which describes what the operator is running and how much of it, so an
anonymous caller now gets the same empty shape those two endpoints give.
"""
turn = {"wall_s": 2.5, "prompt_tokens": 7, "completion_tokens": 12,
"expert_disk_s": 0.4, "expert_wait_s": 0.1, "expert_matmul_s": 0.9,
"attention_s": 0.6, "lm_head_s": 0.2, "forwards": 15}
self.engine.profile = [turn]
self.engine.profile_seq = 1
try:
with urlopen(self.base + "/profile", timeout=2) as response:
self.assertEqual(json.load(response), {"seq": 0, "turns": []},
"unauthenticated caller received telemetry")
with self.request("/profile") as response:
self.assertEqual(json.load(response), {"seq": 1, "turns": [turn]},
"authenticated caller lost access")
finally:
del self.engine.profile, self.engine.profile_seq
def test_browser_preflight(self):
request = Request(self.base + "/v1/chat/completions", method="OPTIONS", headers={
"Origin": "http://localhost:5173",
"Access-Control-Request-Method": "POST",
"Access-Control-Request-Headers": "authorization,content-type",
})
with urlopen(request, timeout=2) as response:
self.assertEqual(response.status, 204)
self.assertEqual(response.headers["Access-Control-Allow-Origin"], "http://localhost:5173")
self.assertIn("Authorization", response.headers["Access-Control-Allow-Headers"])
def test_chat_completion(self):
with self.request("/v1/chat/completions", {
"model": "test-model", "messages": [{"role": "user", "content": "Hi"}],
"max_tokens": 4, "cache_slot": 1,
}) as response:
body = json.load(response)
queue_wait = response.headers.get("x-colibri-queue-wait-ms")
self.assertEqual(body["object"], "chat.completion")
self.assertEqual(body["choices"][0]["message"]["content"], "Héllo")
self.assertEqual(body["usage"], {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9})
self.assertIsNotNone(queue_wait)
self.assertIn("<|user|>Hi<|assistant|><think></think>", self.engine.calls[-1][0])
self.assertEqual(self.engine.calls[-1][4], 1)
def test_kimi_chat_completion_uses_multiturn_wire_payload(self):
with patch("openai_server.ARCH", "kimi"):
with self.request("/v1/chat/completions", {
"model": "test-model",
"messages": [
{"role": "user", "content": "你好"},
{"role": "assistant", "content": "你好。"},
{"role": "user", "content": "Continue"},
],
"enable_thinking": False,
}) as response:
body = json.load(response)
self.assertEqual(body["choices"][0]["message"]["content"], "Héllo")
self.assertEqual(
self.engine.calls[-1][0],
"K3CHAT1\n"
"M user 6\n你好"
"M assistant 9\n你好。"
"M user 8\nContinue"
"G 0\n",
)
def test_chat_completion_stops_across_engine_chunks(self):
before = self.engine.stop_requests
with self.request("/v1/chat/completions", {
"model": "test-model", "messages": [{"role": "user", "content": "Hi"}],
"stop": "éll",
}) as response:
body = json.load(response)
self.assertEqual(body["choices"][0]["message"]["content"], "H")
self.assertEqual(body["choices"][0]["finish_reason"], "stop")
self.assertEqual(self.engine.stop_requests, before + 1)
def test_patient_stop_extension_ignores_a_leading_match(self):
before = self.engine.stop_requests
with self.request("/v1/chat/completions", {
"model": "test-model", "messages": [{"role": "user", "content": "Hi"}],
"stop": "H", "x_colibri_ignore_leading_stop": True,
}) as response:
body = json.load(response)
self.assertEqual(body["choices"][0]["message"]["content"], "éllo")
self.assertEqual(self.engine.stop_requests, before)
def test_patient_stop_extension_requires_a_boolean(self):
with self.assertRaises(HTTPError) as caught:
self.request("/v1/chat/completions", {
"model": "test-model", "messages": [{"role": "user", "content": "Hi"}],
"stop": "H", "x_colibri_ignore_leading_stop": "yes",
})
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
def test_rejects_invalid_cache_slot(self):
with self.assertRaises(HTTPError) as caught:
self.request("/v1/chat/completions", {
"model": "test-model", "messages": [{"role": "user", "content": "Hi"}],
"cache_slot": 2,
})
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
def test_olmoe_never_files_the_answer_as_reasoning(self):
# #984: OLMoE has no thinking mode, so its engine never emits </think>.
# With thinking left on, the reasoning splitter kept the whole answer in
# reasoning_content and streamed an empty `content` -- content dropped.
# Forcing thinking off for olmoe must return the answer as content
# whether or not the client asked for reasoning, streaming or not.
for streaming in (False, True):
with self.subTest(stream=streaming), \
patch("openai_server.ARCH", "olmoe"):
with self.request("/v1/chat/completions", {
"model": "test-model",
"messages": [{"role": "user", "content": "Hi"}],
"reasoning_effort": "high", # a client asking to think
"stream": streaming,
}) as response:
raw = response.read().decode()
if streaming:
payloads = [json.loads(line[6:]) for line in raw.splitlines()
if line.startswith("data: ") and line != "data: [DONE]"]
content = "".join((c.get("delta") or {}).get("content", "")
for p in payloads for c in p["choices"])
reasoning = "".join((c.get("delta") or {}).get("reasoning_content", "")
for p in payloads for c in p["choices"])
else:
msg = json.loads(raw)["choices"][0]["message"]
content = msg.get("content") or ""
reasoning = msg.get("reasoning_content") or ""
self.assertIn("Hé", content, "the answer must arrive as content")
self.assertEqual(reasoning, "", "olmoe must not produce reasoning")
def test_streaming_chat_completion(self):
with self.request("/v1/chat/completions", {
"model": "test-model", "messages": [{"role": "user", "content": "Hi"}],
"stream": True, "stream_options": {"include_usage": True},
}) as response:
stream = response.read().decode()
self.assertIn('\"delta\":{\"role\":\"assistant\",\"content\":\"\"}', stream)
self.assertIn('\"object\":\"chat.completion.chunk\"', stream)
self.assertIn('\"content\":\"Hé\"', stream)
self.assertIn('\"usage\":{\"prompt_tokens\":7,\"completion_tokens\":2,\"total_tokens\":9}', stream)
self.assertTrue(stream.endswith("data: [DONE]\n\n"))
def test_streaming_stop_never_exposes_partial_sequence(self):
before = self.engine.stop_requests
with self.request("/v1/chat/completions", {
"model": "test-model", "messages": [{"role": "user", "content": "Hi"}],
"stream": True, "stop": "éll",
}) as response:
raw = response.read().decode()
payloads = [json.loads(line[6:]) for line in raw.splitlines()
if line.startswith("data: ") and line != "data: [DONE]"]
content = "".join((choice.get("delta") or {}).get("content", "")
for payload in payloads for choice in payload["choices"])
self.assertEqual(content, "H")
self.assertEqual(payloads[-1]["choices"][0]["finish_reason"], "stop")
self.assertEqual(self.engine.stop_requests, before + 1)
def test_legacy_completion(self):
with self.request("/v1/completions", {
"model": "test-model", "prompt": "Complete me", "temperature": 0,
}) as response:
body = json.load(response)
self.assertEqual(body["object"], "text_completion")
self.assertEqual(body["choices"][0]["text"], "Héllo")
self.assertEqual(self.engine.calls[-1][0], "Complete me")
def test_rejects_empty_legacy_completion(self):
with self.assertRaises(HTTPError) as caught:
self.request("/v1/completions", {"model": "test-model", "prompt": ""})
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
self.assertEqual(json.load(caught.exception)["error"]["param"], "prompt")
def test_rejects_invalid_stream_options(self):
with self.assertRaises(HTTPError) as caught:
self.request("/v1/chat/completions", {
"model": "test-model", "messages": [{"role": "user", "content": "Hi"}],
"stream": True, "stream_options": "usage",
})
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
def test_unpaired_surrogate_is_a_client_error(self):
"""JSON can spell a lone UTF-16 surrogate ("\\ud83d": a client that cut a
string between the two halves of an emoji). json.loads accepts it, no
UTF-8 can carry it, and Engine.generate's first step, prompt.encode(),
raised: HTTP 500 "The colibri engine failed". Invalid UTF-8 in the raw
body is already a 400; this is the same invalid text, escaped."""
real_generate = self.engine.generate
def encoding_generate(prompt, *args, **kwargs):
prompt.encode("utf-8") # what Engine.generate does before anything else
return real_generate(prompt, *args, **kwargs)
cut = "emoji \ud83d"
cases = (("/v1/chat/completions", {"messages": [{"role": "user", "content": cut}]}),
("/v1/completions", {"prompt": cut}),
("/v1/messages", {"max_tokens": 4, "messages": [{"role": "user", "content": cut}]}))
with patch.object(self.engine, "generate", side_effect=encoding_generate):
for path, body in cases:
with self.subTest(path=path):
with self.assertRaises(HTTPError) as caught:
self.request(path, dict(body, model="test-model"))
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
def test_tool_choice_with_a_non_object_function_is_a_client_error(self):
"""{"type": "function", "function": "search"} answered HTTP 500.
generation_options() already has the 400 for a `tool_choice` function
object with no name, but it -- and the five renderers that read the
forced tool the same way -- did `(tool_choice.get("function") or {})
.get("name")`, so a `function` that is not an object raised
AttributeError first and do_POST answered 500 "The colibri engine failed
to process the request." for a request the engine never saw. Every arch
and both OpenAI endpoints, because the crash was on the read.
"""
tools = [{"type": "function", "function": {"name": "search"}}]
for arch in ("glm", "glm53", "kimi", "deepseek_v4", "deepseek_v41",
"olmoe", "inkling", "qwen36", "qwen38"):
for function in ("search", ["search"], 5):
with self.subTest(arch=arch, function=function):
with patch("openai_server.ARCH", arch):
with self.assertRaises(HTTPError) as caught:
self.request("/v1/chat/completions", {
"model": "test-model",
"messages": [{"role": "user", "content": "hi"}],
"tools": tools,
"tool_choice": {"type": "function", "function": function}})
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
# /v1/completions has no renderer in front of generation_options, so it
# reached the same read directly.
with self.assertRaises(HTTPError) as caught:
self.request("/v1/completions", {
"model": "test-model", "prompt": "hi", "tools": tools,
"tool_choice": {"type": "function", "function": "search"}})
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
def test_qwen36_preserve_thinking_defaults_to_the_fed_history(self):
"""#1759: with thinking off the past turn keeps the empty block it was generated
after, so a standard client's resent history matches the engine's state. With
thinking on the template default stays: a standard client does not send the
reasoning back, and an empty block would claim the model did not think. The
request key overrides either way."""
history = [{"role": "user", "content": "Capital of France?"},
{"role": "assistant", "content": "Paris."},
{"role": "user", "content": "And of Italy?"}]
kept = "<|im_start|>assistant\n<think>\n\n</think>\n\nParis.<|im_end|>"
bare = "<|im_start|>assistant\nParis.<|im_end|>"
cases = (({}, kept), ({"enable_thinking": True}, bare),
({"preserve_thinking": False}, bare),
({"enable_thinking": True, "preserve_thinking": True},
"<|im_start|>assistant\n<think>\n\n</think>\n\nParis.<|im_end|>"))
with patch("openai_server.ARCH", "qwen36"):
for extra, expected in cases:
with self.subTest(extra=extra):
with self.request("/v1/chat/completions", {
"model": "test-model", "messages": history, **extra}) as response:
self.assertEqual(response.status, 200)
self.assertIn(expected, self.engine.calls[-1][0])
with self.assertRaises(HTTPError) as caught:
self.request("/v1/chat/completions", {
"model": "test-model", "messages": history, "preserve_thinking": "yes"})
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
def test_qwen38_template_defaults_to_xhigh_thinking_on_the_qwen36_engine(self):
"""What the flavor changes over the wire: with no thinking field, a Qwen3.8
template reasons at xhigh by default, as the qwen38 family does."""
history = [{"role": "user", "content": "Capital of France?"}]
with patch("openai_server.ARCH", "qwen36"), patch("openai_server.CHAT_FLAVOR", "qwen38"):
with self.request("/v1/chat/completions", {
"model": "test-model", "messages": history}) as response:
self.assertEqual(response.status, 200)
self.assertEqual(self.engine.calls[-1][0], render_chat_qwen38(history, True, "xhigh"))
def test_a_well_formed_forced_tool_choice_still_runs(self):
"""The read above must not change the shape clients actually send."""
with self.request("/v1/chat/completions", {
"model": "test-model",
"messages": [{"role": "user", "content": "hi"}],
"tools": [{"type": "function", "function": {"name": "search"}}],
"tool_choice": {"type": "function",
"function": {"name": "search"}}}) as response:
self.assertEqual(response.status, 200)
self.assertIn("You must call the function `search`", self.engine.calls[-1][0])
# The OpenAI-legacy spelling {"type": "function", "name": ...} still
# forces the tool: _tool_choice_name keeps that fallback.
with self.request("/v1/chat/completions", {
"model": "test-model",
"messages": [{"role": "user", "content": "hi"}],
"tools": [{"type": "function", "function": {"name": "search"}}],
"tool_choice": {"type": "function", "name": "search"}}) as response:
self.assertEqual(response.status, 200)
self.assertIn("You must call the function `search`", self.engine.calls[-1][0])
def test_tool_with_a_non_object_function_is_a_client_error(self):
"""{"type": "function", "function": "search"} on tools[] answered HTTP 500.
generation_options already has the 400, but /v1/chat/completions renders
first. GLM, GLM-5.3, DeepSeek V4 and V4.1 did fn.items() on a string
and the AttributeError became do_POST's 500 "The colibri engine failed
to process the request." for a request the engine never saw.
"""
for arch in ("glm", "glm53", "kimi", "deepseek_v4", "deepseek_v41",
"olmoe", "inkling", "qwen36", "qwen38"):
for function in ("search", ["search"], 5):
with self.subTest(arch=arch, function=function):
with patch("openai_server.ARCH", arch):
with self.assertRaises(HTTPError) as caught:
self.request("/v1/chat/completions", {
"model": "test-model",
"messages": [{"role": "user", "content": "hi"}],
"tools": [{"type": "function",
"function": function}]})
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
with self.assertRaises(HTTPError) as caught:
self.request("/v1/completions", {
"model": "test-model", "prompt": "hi",
"tools": [{"type": "function", "function": "search"}]})
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
def test_a_well_formed_tool_function_still_runs(self):
"""The read above must not change the shape clients actually send."""
with self.request("/v1/chat/completions", {
"model": "test-model",
"messages": [{"role": "user", "content": "hi"}],
"tools": [{"type": "function",
"function": {"name": "search"}}]}) as response:
self.assertEqual(response.status, 200)
self.assertIn('"name": "search"', self.engine.calls[-1][0])
def test_tool_call_arguments_that_are_not_an_object_do_not_fail_the_request(self):
"""A replayed tool call with `arguments: "[1, 2]"` answered HTTP 500.
render_chat reached `(args or {}).items()` with a parsed list, and the
AttributeError became do_POST's catch-all 500 "The colibri engine failed
to process the request." -- which OpenAI SDKs retry, against an engine
that was never asked anything.
"""
body = {"model": "test-model", "messages": [
{"role": "user", "content": "run it"},
{"role": "assistant", "content": "", "tool_calls": [
{"id": "x", "type": "function",
"function": {"name": "fn", "arguments": "[1, 2]"}}]},
{"role": "tool", "tool_call_id": "x", "content": "done"},
{"role": "user", "content": "and now?"},
]}
with self.request("/v1/chat/completions", body) as response:
self.assertEqual(response.status, 200)
self.assertEqual(json.load(response)["object"], "chat.completion")
def test_a_past_tool_call_whose_function_is_not_an_object_is_a_client_error(self):
"""A replayed tool call with `function: "search"` answered HTTP 500.
The fallback, Kimi and Qwen3.8 renderers already answer 400. GLM, GLM-5.3
and DeepSeek V4/V4.1 called .get() on the value, and the AttributeError
became do_POST's 500 "The colibri engine failed to process the request."
"""
def history(function):
return {"model": "test-model", "messages": [
{"role": "user", "content": "run it"},
{"role": "assistant", "content": "", "tool_calls": [
{"id": "x", "type": "function", "function": function}]},
{"role": "tool", "tool_call_id": "x", "content": "done"},
{"role": "user", "content": "and now?"},
]}
cases = [("search", "messages.1.tool_calls.0.function"),
(["search"], "messages.1.tool_calls.0.function"),
(5, "messages.1.tool_calls.0.function"),
(None, "messages.1.tool_calls.0.function"),
({"name": 5, "arguments": "{}"}, "messages.1.tool_calls.0.function.name")]
for arch in ("glm", "glm53", "deepseek_v4", "deepseek_v41"):
for function, param in cases:
with self.subTest(arch=arch, function=function):
with patch("openai_server.ARCH", arch):
with self.assertRaises(HTTPError) as caught:
self.request("/v1/chat/completions", history(function))
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
self.assertEqual(json.loads(caught.exception.read())["error"]["param"], param)
with self.subTest(arch=arch, function="well formed"):
with patch("openai_server.ARCH", arch):
with self.request("/v1/chat/completions",
history({"name": "fn", "arguments": "{}"})) as response:
self.assertEqual(response.status, 200)
def test_past_tool_calls_that_are_not_an_array_are_a_client_error(self):
"""A replayed assistant turn with `tool_calls: 5` answered HTTP 500 on Qwen and Kimi.
The other renderers already answer 400 "`tool_calls` must be an array.". The Qwen
renderer's _qwen_tool_calls and Kimi's _k3_order_tool_results iterated the value as
it came, and the TypeError became do_POST's 500 "The colibri engine failed to process the
request."
"""
def history(tool_calls):
return {"model": "test-model", "messages": [
{"role": "user", "content": "run it"},
{"role": "assistant", "content": "", "tool_calls": tool_calls},
{"role": "tool", "tool_call_id": "x", "content": "done"},
{"role": "user", "content": "and now?"},
]}
for arch in ("glm", "glm53", "qwen36", "qwen38", "kimi", "deepseek_v4",
"deepseek_v41", "mimo"):
for tool_calls in (5, 1.5, True, "call", {"id": "x"}):
with self.subTest(arch=arch, tool_calls=tool_calls):
with patch("openai_server.ARCH", arch):
with self.assertRaises(HTTPError) as caught:
self.request("/v1/chat/completions", history(tool_calls))
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
self.assertEqual(json.loads(caught.exception.read())["error"]["param"],
"messages.1.tool_calls")
with self.subTest(arch=arch, tool_calls="well formed"):
with patch("openai_server.ARCH", arch):
with self.request("/v1/chat/completions", history([
{"id": "x", "type": "function",
"function": {"name": "fn", "arguments": "{}"}}])) as response:
self.assertEqual(response.status, 200)
def test_a_text_part_whose_text_is_not_a_string_is_a_client_error(self):
"""{"type": "text", "text": 5} answered HTTP 500 on GLM-5.3, Qwen3.8 and V4.1.
The other renderers read content through content_text(), which already
answers 400. The image expanders of these three, and the GLM-5.3
renderer, join the text parts themselves, and "".join raised TypeError,
which do_POST turns into its catch-all 500.
"""
def request(text):
return {"model": "test-model",
"messages": [{"role": "user", "content": [
{"type": "text", "text": "look at "},
{"type": "text", "text": text}]}]}
for arch in ("glm53", "qwen38", "deepseek_v41"):
for text in (5, None, ["hi"], {"value": "hi"}):
with self.subTest(arch=arch, text=text):
with patch("openai_server.ARCH", arch):
with self.assertRaises(HTTPError) as caught:
self.request("/v1/chat/completions", request(text))
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
self.assertEqual(json.loads(caught.exception.read())["error"]["param"],
"messages.0.content.1.text")
with self.subTest(arch=arch, text="well formed"):
with patch("openai_server.ARCH", arch):
with self.request("/v1/chat/completions", request("this")) as response:
self.assertEqual(response.status, 200)
self.assertIn("look at this", self.engine.calls[-1][0])
class ClientHangupTest(unittest.TestCase):
"""A client that disconnects mid-response must not print a traceback.
`coli chat` polls /health on a 2 s timeout while the model loads and drops
each connection the moment it has an answer; Ctrl-C during a stream closes
the socket by design -- the banner tells the user to do exactly that. Both
reach the handler as BrokenPipeError, and socketserver logs an unhandled
exception per occurrence, so a normal DeepSeek V4 start buried the loading
spinner under stack traces and every cancelled answer looked like a crash.
"""
def setUp(self):
self.engine = FakeEngine()
self.server = APIServer(("127.0.0.1", 0), self.engine, "test-model",
None, 16, kv_slots=1)
self.errors = []
# socketserver routes an escaped exception here; the base class prints
# it to stderr. Recording instead of printing is what lets the test
# assert on it rather than on captured output.
self.server.handle_error = lambda request, address: self.errors.append(
sys.exc_info()[1])
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
self.thread.start()
self.addCleanup(self.thread.join, 2)
self.addCleanup(self.server.server_close)
self.addCleanup(self.server.shutdown)
self.addCleanup(self.server.scheduler.close)
def _hang_up_after_request(self, path):
"""Send a request, then close without reading the response."""
sock = socket.create_connection(("127.0.0.1", self.server.server_port), 2)
sock.sendall(f"GET {path} HTTP/1.1\r\nHost: 127.0.0.1\r\n"
f"Connection: close\r\n\r\n".encode())
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack("ii", 1, 0)) # RST rather than a clean FIN
sock.close()
time.sleep(0.3)
def test_hangup_on_health_is_not_an_error(self):
for _ in range(5):
self._hang_up_after_request("/health")
self.assertEqual(self.errors, [],
"client disconnect surfaced as a server error")
def test_the_server_still_answers_afterwards(self):
"""The real damage would be a handler thread lost to the exception."""
self._hang_up_after_request("/health")
with urlopen(f"http://127.0.0.1:{self.server.server_port}/health",
timeout=2) as response:
self.assertEqual(json.load(response)["status"], "ok")
def _abort(self, *args):
"""Windows' spelling of the same disconnect, raised where it really lands.
In #854's log the traceback runs do_GET -> send_json -> end_headers ->
flush_headers -> wfile.write -> sendall, so raising from end_headers
reproduces the exact shape on any platform.
"""
raise ConnectionAbortedError(
10053, "An established connection was aborted by the software in your "
"host machine")
def test_windows_aborted_connection_is_not_an_error(self):
"""ConnectionAbortedError is a SIBLING of BrokenPipeError and
ConnectionResetError under ConnectionError, not a subclass of either --
so catching the pair caught the POSIX spellings and let the Windows one
escape. #854 is pages of WinError 10053 tracebacks from a healthy start.
"""
self.assertFalse(
issubclass(ConnectionAbortedError, (BrokenPipeError, ConnectionResetError)),
"the old except clause would have covered this; the test proves nothing")
with patch.object(APIHandler, "end_headers", self._abort):
try:
with urlopen(f"http://127.0.0.1:{self.server.server_port}/health", timeout=2):
pass
except Exception:
pass # the client sees a broken response; that is fine
time.sleep(0.3)
self.assertEqual(self.errors, [],
"WinError 10053 surfaced as a server error (#854)")
def test_the_server_survives_an_aborted_connection(self):
"""Same as the hangup case: the damage is a lost handler, not the log."""
with patch.object(APIHandler, "end_headers", self._abort):
try:
with urlopen(f"http://127.0.0.1:{self.server.server_port}/health", timeout=2):
pass
except Exception:
pass
time.sleep(0.3)
with urlopen(f"http://127.0.0.1:{self.server.server_port}/health",
timeout=2) as response:
self.assertEqual(json.load(response)["status"], "ok")
class StaticServingTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
root = Path(self.tmp.name)
dist = root / "dist"
dist.mkdir()
(dist / "index.html").write_text("dashboard", encoding="utf-8")
sibling = root / "dist-private"
sibling.mkdir()
(sibling / "secret.txt").write_text("private", encoding="utf-8")
self.web_dist = patch.object(APIHandler, "WEB_DIST", dist)
self.web_dist.start()
self.server = APIServer(("127.0.0.1", 0), FakeEngine(), "test-model")
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
self.thread.start()
self.base = f"http://127.0.0.1:{self.server.server_port}"
def tearDown(self):
self.server.scheduler.close()
self.server.shutdown()
self.server.server_close()
self.thread.join(timeout=2)
self.web_dist.stop()
self.tmp.cleanup()
def test_static_root_stays_inside_dist_directory(self):
with urlopen(self.base + "/", timeout=2) as response:
self.assertEqual(response.read(), b"dashboard")
with self.assertRaises(HTTPError) as caught:
urlopen(self.base + "/%2e%2e/dist-private/secret.txt", timeout=2)
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 404)
class SchedulerHTTPTest(unittest.TestCase):
def setUp(self):
self.engine = BlockingEngine()
self.server = APIServer(("127.0.0.1", 0), self.engine, "test-model",
max_tokens=16, max_queue=0)
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
self.thread.start()
self.url = f"http://127.0.0.1:{self.server.server_port}/v1/chat/completions"
def tearDown(self):
self.engine.release.set()
self.server.scheduler.close()
self.server.shutdown(); self.server.server_close(); self.thread.join(timeout=2)
def request(self):
body = json.dumps({"model": "test-model", "messages": [
{"role": "user", "content": "Hi"}]}).encode()
return urlopen(Request(self.url, data=body, headers={"Content-Type": "application/json"}), timeout=2)
def test_queue_full_returns_429_before_generation(self):
first_errors = []
def first_request():
try:
with self.request() as response: response.read()
except Exception as error:
first_errors.append(error)
first = threading.Thread(target=first_request); first.start()
self.assertTrue(self.engine.entered.wait(1))
with self.assertRaises(HTTPError) as caught:
self.request()
self.addCleanup(caught.exception.close)
error = json.loads(caught.exception.read())["error"]
self.assertEqual(caught.exception.code, 429)
self.assertEqual(caught.exception.headers["Retry-After"], "1")
self.assertEqual(error["code"], "queue_full")
self.engine.release.set(); first.join(2)
self.assertEqual(first_errors, [])
ORDER_TOOL = [{"type": "function", "function": {
"name": "lookup_order",
"parameters": {"type": "object", "properties": {
"order_id": {"type": "string"},
"qty": {"type": "integer"},
"express": {"type": "boolean"},
}, "required": ["order_id"]}}}]
class ToolArgumentTypeTest(unittest.TestCase):
"""The model emits every argument as text. Without the schema, a string-typed value that
happens to look numeric is json.loads()'d into an int and the tool gets the wrong type."""
def _args(self, reply, tools=ORDER_TOOL):
_, calls = parse_tool_calls(reply, tools)
self.assertEqual(len(calls), 1)
return json.loads(calls[0]["function"]["arguments"])
def test_string_parameter_holding_digits_stays_a_string(self):
args = self._args("<tool_call>lookup_order"
"<arg_key>order_id</arg_key><arg_value>12345</arg_value></tool_call>")
self.assertEqual(args["order_id"], "12345")
self.assertIsInstance(args["order_id"], str)
def test_declared_numeric_and_boolean_parameters_are_decoded(self):
args = self._args("<tool_call>lookup_order"
"<arg_key>order_id</arg_key><arg_value>A-1</arg_value>"
"<arg_key>qty</arg_key><arg_value>2</arg_value>"
"<arg_key>express</arg_key><arg_value>true</arg_value></tool_call>")
self.assertEqual(args, {"order_id": "A-1", "qty": 2, "express": True})
self.assertIsInstance(args["qty"], int)
self.assertIs(args["express"], True)
def test_unknown_parameter_keeps_permissive_decoding(self):
args = self._args("<tool_call>lookup_order"
"<arg_key>extra</arg_key><arg_value>7</arg_value></tool_call>")
self.assertEqual(args["extra"], 7)
class DeepSeekV4ToolCallTest(unittest.TestCase):
"""DeepSeek V4 (#916): DSML tool blocks. Schemas render into the first system/developer
message, assistant tool_calls render as <|DSML|invoke> blocks, tool results merge into
user turns as <tool_result>, and model output parses back into OpenAI tool_calls."""
DSML = "|DSML|"
WEATHER = [{"type": "function", "function": {
"name": "get_weather", "description": "current weather",
"parameters": {"type": "object", "properties": {
"city": {"type": "string"}, "days": {"type": "integer"}},
"required": ["city"]}}}]
CALL = [{"id": "call_1", "type": "function", "function": {
"name": "get_weather",
"arguments": json.dumps({"city": "Paris", "days": 3})}}]
def test_tools_declared_on_first_system_message(self):
prompt = render_chat_v4([{"role": "system", "content": "Be brief."},
{"role": "user", "content": "Weather?"}], tools=self.WEATHER)
self.assertLess(prompt.index("Be brief."), prompt.index("## Tools"))
self.assertIn(f'"{self.WEATHER[0]["function"]["name"]}"', prompt)
self.assertTrue(prompt.startswith("<|begin▁of▁sentence|>"))
def test_tools_prepended_when_no_system_message(self):
prompt = render_chat_v4([{"role": "user", "content": "hi"}], tools=self.WEATHER)
# The official encoder renders tools on an empty system message: bos + "\n\n" + tools.
self.assertTrue(prompt.startswith("<|begin▁of▁sentence|>\n\n## Tools"))
def test_tool_choice_none_suppresses_declaration(self):
prompt = render_chat_v4([{"role": "user", "content": "hi"}],
tools=self.WEATHER, tool_choice="none")
self.assertNotIn("## Tools", prompt)
def test_developer_message_is_wrapped_in_user_token(self):
# encoding_dsv4.py wraps developer content in <|User|>; system stays bare.
prompt = render_chat_v4([{"role": "developer", "content": "Be terse."},
{"role": "user", "content": "hi"}], tools=self.WEATHER)
self.assertIn("<|User|>Be terse.", prompt)
self.assertNotIn("<|User|>## Tools", prompt)
def test_thinking_mode_prepends_effort_prompt(self):
# encoding_dsv4.py prepends the level prompt after BOS in thinking mode.
prompt = render_chat_v4([{"role": "user", "content": "hi"}], enable_thinking=True,
reasoning_effort="high")
self.assertTrue(prompt.startswith("<|begin▁of▁sentence|>Reasoning Effort: High."))
self.assertTrue(prompt.endswith("<|Assistant|><think>"))
# V4-native level names work too; low adds nothing.
self.assertTrue(render_chat_v4([{"role": "user", "content": "hi"}],
enable_thinking=True, reasoning_effort="max").startswith(
"<|begin▁of▁sentence|>Reasoning Effort: Maximum."))
self.assertFalse(render_chat_v4([{"role": "user", "content": "hi"}],
enable_thinking=True, reasoning_effort="low").startswith(
"<|begin▁of▁sentence|>Reasoning Effort:"))
def test_assistant_tool_calls_render_as_dsml(self):
prompt = render_chat_v4([{"role": "user", "content": "Paris?"},
{"role": "assistant", "content": None,
"tool_calls": self.CALL}])
self.assertIn(f'<{self.DSML}invoke name="get_weather">', prompt)
self.assertIn(f'<{self.DSML}parameter name="city" string="true">Paris</{self.DSML}parameter>',
prompt)
self.assertIn(f'<{self.DSML}parameter name="days" string="false">3</{self.DSML}parameter>',
prompt)
self.assertTrue(prompt.endswith("</think>"))
def test_tool_results_merge_into_one_user_turn(self):
prompt = render_chat_v4([{"role": "user", "content": "Paris?"},
{"role": "assistant", "content": None, "tool_calls": self.CALL},
{"role": "tool", "tool_call_id": "call_1", "content": "21c"},
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
{"role": "user", "content": "And London?"}])
self.assertEqual(prompt.count("<|User|>"), 2)
self.assertIn("<tool_result>21c</tool_result>", prompt)
self.assertIn("<tool_result>sunny</tool_result>", prompt)
self.assertIn("And London?", prompt)
def test_parse_dsml_reply_to_openai_tool_calls(self):
raw = (f"Here you go.\n\n<{self.DSML}tool_calls>\n"
f"<{self.DSML}invoke name=\"get_weather\">\n"
f"<{self.DSML}parameter name=\"city\" string=\"true\">Paris</{self.DSML}parameter>\n"
f"<{self.DSML}parameter name=\"days\" string=\"false\">3</{self.DSML}parameter>\n"
f"</{self.DSML}invoke>\n</{self.DSML}tool_calls><|end▁of▁sentence|>")
content, calls = parse_dsv4_tool_calls(raw)
self.assertEqual(content, "Here you go.")
self.assertEqual(len(calls), 1)
self.assertEqual(calls[0]["function"]["name"], "get_weather")
self.assertEqual(json.loads(calls[0]["function"]["arguments"]),
{"city": "Paris", "days": 3})
def test_parse_truncated_block_leaks_no_markers(self):
content, calls = parse_dsv4_tool_calls("Almost.\n\n<|DSML|tool_calls><|DSML|invoke na")
self.assertEqual(calls, [])
self.assertEqual(content, "Almost.")
self.assertNotIn("DSML", content)
class EngineErrorFrameTest(unittest.TestCase):
"""#401: an over-long prompt used to be silently truncated to the first CTX-2 tokens, so the
model answered from a mutilated prompt and the client got HTTP 200 with junk. The engine now
refuses, and the refusal has to reach the client as a 400 it can act on -- not a 500."""
def test_context_exceeded_becomes_a_400_the_client_can_act_on(self):
err = _engine_error(["CONTEXT_EXCEEDED", "8321", "4094"], "CONTEXT_EXCEEDED 8321 4094")
self.assertIsInstance(err, APIError)
self.assertEqual(err.status, 400)
self.assertEqual(err.code, "context_length_exceeded")
self.assertEqual(err.param, "messages")
self.assertIn("4094", err.message)
self.assertIn("8321", err.message)
def test_other_engine_errors_stay_runtime_errors(self):
for frame in (["SLOT_BUSY"], ["BAD_REQUEST"], []):
err = _engine_error(frame, " ".join(frame) or "engine request failed")
self.assertIsInstance(err, RuntimeError)
self.assertNotIsInstance(err, APIError)
def test_malformed_context_frame_does_not_crash_the_dispatcher(self):
err = _engine_error(["CONTEXT_EXCEEDED"], "CONTEXT_EXCEEDED")
self.assertIsInstance(err, APIError)
self.assertEqual(err.status, 400)
class UnclosedToolCallTest(unittest.TestCase):
"""#401: the model opens <tool_call>, emits a well-formed call, then stops without the
closing tag (budget ran out, or quantization mangled it). The strict regex needs both tags,
so the client used to get zero tool_calls -- a total failure from a recoverable output."""
NO_ARG_TOOL = ORDER_TOOL + [{"type": "function",
"function": {"name": "list_orders", "parameters": {}}}]
def _calls(self, reply, tools=ORDER_TOOL):
return parse_tool_calls(reply, tools)
def test_unclosed_box_is_recovered(self):
content, calls = self._calls("<tool_call>lookup_order"
"<arg_key>order_id</arg_key><arg_value>A-1</arg_value>")
self.assertEqual(len(calls), 1)
self.assertEqual(json.loads(calls[0]["function"]["arguments"]), {"order_id": "A-1"})
self.assertEqual(content, "")
def test_mangled_closing_tag_is_recovered(self):
_, calls = self._calls("<tool_call>lookup_order"
"<arg_key>order_id</arg_key><arg_value>A-1</arg_value></tool_cal")
self.assertEqual(len(calls), 1)
self.assertEqual(json.loads(calls[0]["function"]["arguments"]), {"order_id": "A-1"})
def test_leading_prose_is_kept_as_content(self):
content, calls = self._calls("Let me check.\n<tool_call>lookup_order"
"<arg_key>order_id</arg_key><arg_value>A-1</arg_value>")
self.assertEqual(len(calls), 1)
self.assertEqual(content, "Let me check.")
def test_closed_call_followed_by_an_unclosed_one(self):
_, calls = self._calls("<tool_call>lookup_order"
"<arg_key>order_id</arg_key><arg_value>A-1</arg_value></tool_call>"
"<tool_call>lookup_order"
"<arg_key>order_id</arg_key><arg_value>B-2</arg_value>")
self.assertEqual([json.loads(c["function"]["arguments"])["order_id"] for c in calls],
["A-1", "B-2"])
def test_bare_declared_name_recovers_a_zero_argument_call(self):
_, calls = self._calls("<tool_call>list_orders", self.NO_ARG_TOOL)
self.assertEqual(len(calls), 1)
self.assertEqual(calls[0]["function"]["name"], "list_orders")
self.assertEqual(json.loads(calls[0]["function"]["arguments"]), {})
def test_prose_mentioning_the_marker_does_not_fabricate_a_call(self):
content, calls = self._calls("To call a tool, write <tool_call> and then the name.")
self.assertEqual(calls, [])
self.assertIn("<tool_call>", content)
def test_undeclared_name_without_arguments_is_not_recovered(self):
_, calls = self._calls("<tool_call>drop_all_tables")
self.assertEqual(calls, [])
def test_well_formed_output_is_untouched(self):
content, calls = self._calls("Done.<tool_call>lookup_order"
"<arg_key>order_id</arg_key><arg_value>A-1</arg_value>"
"</tool_call>")
self.assertEqual(len(calls), 1)
self.assertEqual(content, "Done.")
class StrictGLMToolCallTest(unittest.TestCase):
"""Opt-in calls must be complete before a client can execute them."""
CALL = ("<tool_call>lookup_order<arg_key>order_id</arg_key>"
"<arg_value>00123</arg_value></tool_call>")
def test_complete_call_preserves_declared_string(self):
content, calls = openai_server.parse_glm_tool_calls_strict("Checking. " + self.CALL,
ORDER_TOOL)
self.assertEqual(content, "Checking.")
self.assertEqual(json.loads(calls[0]["function"]["arguments"]), {"order_id": "00123"})
def test_incomplete_duplicate_undeclared_and_invalid_calls_fail_closed(self):
bad = (self.CALL[:-len("</tool_call>")],
self.CALL.replace("</tool_call>",
"<arg_key>order_id</arg_key><arg_value>other</arg_value></tool_call>"),
self.CALL.replace("lookup_order", "other_function"),
self.CALL.replace("</tool_call>",
"<arg_key>qty</arg_key><arg_value>many</arg_value></tool_call>"),
self.CALL.replace("order_id", "other_key"),
"<tool_call>lookup_order</tool_call>")
for reply in bad:
with self.subTest(reply=reply), self.assertRaises(APIError) as caught:
openai_server.parse_glm_tool_calls_strict(reply, ORDER_TOOL)
self.assertEqual(caught.exception.code, "invalid_model_tool_call")
def test_http_refuses_length_limited_tool_and_unsupported_stream_before_dispatch(self):
engine = ScriptedEngine(chunks=(self.CALL,), length_limited=True)
base = _spawn_test_server(self, engine)
body = {"model": "test-model", "messages": [{"role": "user", "content": "order?"}],
"tools": ORDER_TOOL, "strict_tool_calls": True}
with patch.object(openai_server, "ARCH", "glm"):
status, error = _error_body(self, lambda: _post_chat(base, body))
self.assertEqual((status, error["code"]), (502, "invalid_model_tool_call"))
self.assertEqual(len(engine.calls), 1)
status, error = _error_body(self, lambda: _post_chat(base, {**body, "stream": True}))
self.assertEqual((status, error["code"]), (400, "unsupported_parameter"))
self.assertEqual(len(engine.calls), 1)
status, error = _error_body(self, lambda: _post_chat(base, {**body,
"strict_tool_calls": "true"}))
self.assertEqual((status, error["param"]), (400, "strict_tool_calls"))
self.assertEqual(len(engine.calls), 1)
status, error = _error_body(self, lambda: _post_chat(base, {**body, "tools": []}))
self.assertEqual((status, error["param"]), (400, "strict_tool_calls"))
self.assertEqual(len(engine.calls), 1)
def test_http_complete_call_finishes_with_tool_calls(self):
engine = ScriptedEngine(chunks=(self.CALL,))
base = _spawn_test_server(self, engine)
body = {"model": "test-model", "messages": [{"role": "user", "content": "order?"}],
"tools": ORDER_TOOL, "strict_tool_calls": True}
with patch.object(openai_server, "ARCH", "glm"):
with _post_chat(base, body) as response:
result = json.load(response)
choice = result["choices"][0]
self.assertEqual(choice["finish_reason"], "tool_calls")
self.assertEqual(json.loads(choice["message"]["tool_calls"][0]["function"]["arguments"]),
{"order_id": "00123"})
class ToolChoiceTest(unittest.TestCase):
def test_none_does_not_offer_the_tools(self):
prompt = render_chat([{"role": "user", "content": "hi"}], tools=ORDER_TOOL,
tool_choice="none")
self.assertNotIn("<tools>", prompt)
def test_auto_offers_the_tools(self):
prompt = render_chat([{"role": "user", "content": "hi"}], tools=ORDER_TOOL,
tool_choice="auto")
self.assertIn("<tools>", prompt)
def test_required_instructs_the_model_to_call_one(self):
prompt = render_chat([{"role": "user", "content": "hi"}], tools=ORDER_TOOL,
tool_choice="required")
self.assertIn("<tools>", prompt)
self.assertIn("must call one of the functions", prompt)
def test_named_function_restricts_to_that_function(self):
tools = ORDER_TOOL + [{"type": "function", "function": {"name": "other", "parameters": {}}}]
prompt = render_chat([{"role": "user", "content": "hi"}], tools=tools,
tool_choice={"type": "function", "function": {"name": "lookup_order"}})
self.assertIn("must call the function `lookup_order`", prompt)
self.assertNotIn('"other"', prompt)
def test_rejects_unknown_string_and_unknown_function(self):
with self.assertRaises(APIError):
generation_options({"messages": [], "tools": ORDER_TOOL, "tool_choice": "maybe"}, 128)
with self.assertRaises(APIError):
generation_options({"messages": [], "tools": ORDER_TOOL,
"tool_choice": {"type": "function",
"function": {"name": "nope"}}}, 128)
def test_rejects_tool_choice_without_tools(self):
with self.assertRaises(APIError):
generation_options({"messages": [], "tool_choice": "required"}, 128)
# #1698 Tier 2: `required` was prompt-level on four renderers and dropped by
# the four others that accept it, so a client got the tools and no instruction
# to use them. These pin the one-line difference per renderer, because the
# failure is silent: the request succeeds, the model just answers in prose.
# The renderers that render a tool block and therefore have somewhere to put
# the instruction. Kimi K3 is absent on purpose: its wire format carries a
# dedicated tool-choice message rather than prose (tested below).
BLOCK_RENDERERS = (render_chat, render_chat_v4, render_chat_dsv41, render_chat_qwen,
render_chat_qwen38, render_chat_glm53, render_chat_mimo)
def _auto_and_required(self, render, **kwargs):
messages = [{"role": "user", "content": "Where is order 7?"}]
return (render(messages, tools=ORDER_TOOL, tool_choice="auto", **kwargs),
render(messages, tools=ORDER_TOOL, tool_choice="required", **kwargs))
def test_required_is_auto_plus_exactly_the_instruction(self):
for render in self.BLOCK_RENDERERS:
with self.subTest(renderer=render.__name__):
auto, required = self._auto_and_required(render)
self.assertEqual(required.count(TOOL_CHOICE_REQUIRED_INSTRUCTION), 1)
# Nothing else moves: the instruction is the whole difference
# between `auto` and `required` on every one of these families.
self.assertEqual(
required.replace(TOOL_CHOICE_REQUIRED_INSTRUCTION, "", 1), auto)
def test_required_instruction_comes_after_the_declarations(self):
# "the functions above" has to be true, so the instruction goes after the
# last declared function and not inside or before the tool block.
for render in self.BLOCK_RENDERERS:
with self.subTest(renderer=render.__name__):
_, required = self._auto_and_required(render)
self.assertLess(required.rindex("lookup_order"),
required.index(TOOL_CHOICE_REQUIRED_INSTRUCTION))
def test_required_sits_between_the_tool_block_and_the_system_text(self):
# Qwen3.6/3.8 build one system turn as [tool block][client system text];
# the instruction belongs to the tool half, so it goes between the two
# and not after the client's own text.
system = [{"role": "system", "content": "Be terse."},
{"role": "user", "content": "Where is order 7?"}]
for render in (render_chat_qwen, render_chat_qwen38):
with self.subTest(renderer=render.__name__):
required = render(system, tools=ORDER_TOOL, tool_choice="required")
self.assertLess(required.index("# Tools"),
required.index(TOOL_CHOICE_REQUIRED_INSTRUCTION))
self.assertLess(required.index(TOOL_CHOICE_REQUIRED_INSTRUCTION),
required.index("Be terse."))
def test_mimo_required_stays_inside_the_tool_system_turn(self):
# MiMo's declaration block is a system turn of its own, so there is
# nowhere outside it to put the line: it has to land after </tools> and
# before that turn's <|im_end|>, or the frame closes before the model
# has read it.
_, required = self._auto_and_required(render_chat_mimo)
close = required.index("</tools>")
self.assertLess(close, required.index(TOOL_CHOICE_REQUIRED_INSTRUCTION))
self.assertLess(required.index(TOOL_CHOICE_REQUIRED_INSTRUCTION),
required.index("<|im_end|>"))
def test_olmoe_required_under_the_tool_fallback(self):
with patch("openai_server._TOOL_FALLBACK", True):
auto, required = self._auto_and_required(render_chat_olmoe)
self.assertEqual(required.count(TOOL_CHOICE_REQUIRED_INSTRUCTION), 1)
self.assertEqual(required.replace(TOOL_CHOICE_REQUIRED_INSTRUCTION, "", 1), auto)
self.assertLess(required.rindex("lookup_order"),
required.index(TOOL_CHOICE_REQUIRED_INSTRUCTION))
def test_olmoe_without_the_fallback_refuses_required(self):
# No tool block, so no instruction to give: the honest outcome is the
# 400, not a `required` that is accepted and then ignored.
with patch("openai_server._TOOL_FALLBACK", False):
with self.assertRaisesRegex(APIError, "Tool use"):
render_chat_olmoe([{"role": "user", "content": "Hi"}], tools=ORDER_TOOL,
tool_choice="required")
def test_inkling_refuses_required(self):
with self.assertRaisesRegex(APIError, "not wired up"):
render_chat_inkling([{"role": "user", "content": "Hi"}], tools=ORDER_TOOL,
tool_choice="required")
def test_kimi_keeps_its_own_required_wording(self):
# Same policy, different wire format: a dedicated tool-choice message
# instead of prose appended to the tool block.
required = render_chat_kimi([{"role": "user", "content": "Hi"}],
tools=ORDER_TOOL, tool_choice="required")
self.assertIn("tool-choiceThe system is invoked with `tool_choice=required`.",
required)
self.assertIn("You MUST call tools in the next message.", required)
self.assertNotIn(TOOL_CHOICE_REQUIRED_INSTRUCTION, required)
class TrailingAssistantTurnTest(unittest.TestCase):
"""A trailing `assistant` message is a turn to CONTINUE, not one already finished.
The gateway used to fold it into a completed turn and append a fresh generation cue,
so the model wrote a second assistant turn and the client's opening was dropped. These
pin what replaced that: the switch that turns continuation on, what the prompt looks
like when it is on, and what is refused rather than silently reshaped.
The switch is COLI_CONTINUE_ASSISTANT and not a request field on purpose -- a body
extension would only be reachable by hand-written JSON, and the clients that want this
send a message list and nothing else.
"""
OPEN_TURN = [{"role": "user", "content": "capitale della Francia?"},
{"role": "assistant", "content": "La capitale e'"}]
@staticmethod
def on():
return patch.dict(os.environ, {"COLI_CONTINUE_ASSISTANT": "1"})
@staticmethod
def off():
return patch.dict(os.environ, {"COLI_CONTINUE_ASSISTANT": "0"})
def test_switch_on_leaves_the_turn_open(self):
with self.on(), patch("openai_server.ARCH", "glm53"):
self.assertFalse(resolve_generation_prompt(self.OPEN_TURN, {}))
prompt = render_chat_glm53(self.OPEN_TURN, enable_thinking=True,
add_generation_prompt=False)
# ends INSIDE the turn, on the client's opening -- no new cue after it
self.assertTrue(prompt.endswith("La capitale e'"), prompt[-60:])
self.assertFalse(prompt.endswith("<|assistant|><think>"))
self.assertEqual(prompt.count("<|assistant|>"), 1)
def test_default_is_on(self):
"""Continuation is the default now: unset (the ordinary deployment) continues a
trailing assistant turn. Only COLI_CONTINUE_ASSISTANT=0 turns it off."""
with patch.dict(os.environ, {}, clear=False), patch("openai_server.ARCH", "glm53"):
os.environ.pop("COLI_CONTINUE_ASSISTANT", None)
self.assertFalse(resolve_generation_prompt(self.OPEN_TURN, {}))
def test_switch_off_restores_old_behaviour(self):
"""COLI_CONTINUE_ASSISTANT=0 is the off-switch: a deployment that sets it behaves
exactly as the gateway did before continuation existed -- append a fresh cue."""
with self.off(), patch("openai_server.ARCH", "glm53"):
self.assertTrue(resolve_generation_prompt(self.OPEN_TURN, {}))
self.assertEqual(render_chat_glm53(self.OPEN_TURN, enable_thinking=True),
render_chat_glm53(self.OPEN_TURN, enable_thinking=True,
add_generation_prompt=True))
self.assertTrue(render_chat_glm53(self.OPEN_TURN,
enable_thinking=True).endswith("<|assistant|><think>"))
def test_no_trailing_assistant_turn_is_untouched_either_way(self):
messages = [{"role": "user", "content": "a"}]
for switch in (self.on(), self.off()):
with switch, patch("openai_server.ARCH", "glm53"):
self.assertTrue(resolve_generation_prompt(messages, {}))
def test_rejects_trailing_whitespace(self):
"""The template strips it, so the model would resume from different bytes than the
ones sent -- the same reason Anthropic's own prefill validator refuses it."""
messages = [{"role": "user", "content": "a"},
{"role": "assistant", "content": "La capitale e' "}]
with self.on(), patch("openai_server.ARCH", "glm53"):
with self.assertRaises(APIError):
resolve_generation_prompt(messages, {})
def test_rejects_an_empty_continuation(self):
"""An empty one ends the prompt on a closed, empty <think></think>: the
out-of-distribution position #1327 removed."""
for empty in ("", " ", None):
messages = [{"role": "user", "content": "a"},
{"role": "assistant", "content": empty}]
with self.on(), patch("openai_server.ARCH", "glm53"):
with self.assertRaises(APIError):
resolve_generation_prompt(messages, {})
def test_rejects_tools_and_tool_calls(self):
with self.on(), patch("openai_server.ARCH", "glm53"):
with self.assertRaises(APIError):
resolve_generation_prompt(self.OPEN_TURN, {"tools": ORDER_TOOL})
calls = [{"role": "user", "content": "a"},
{"role": "assistant", "content": "x", "tool_calls": [
{"type": "function", "function": {"name": "f", "arguments": "{}"}}]}]
with self.assertRaises(APIError):
resolve_generation_prompt(calls, {})
def test_unimplemented_family_passes_through(self):
"""Continuation is on by default, so a family whose renderer has no open-turn shape
yet must render as before -- append the cue -- not reject a request nobody opted into.
Every shipped family is now in CONTINUATION_FAMILIES (Kimi K3 too, via its C `C`
record), so the backstop is exercised with a hypothetical future arch: it must pass
through, not error, the day a new renderer lands before its open-turn shape does."""
self.assertNotIn("future_family", CONTINUATION_FAMILIES)
with self.on(), patch("openai_server.ARCH", "future_family"):
self.assertTrue(resolve_generation_prompt(self.OPEN_TURN, {}))
def test_continuation_open_turn_deepseek_v4(self):
"""deepseek_v4 has no authoritative vendored jinja template to diff against:
render_chat_v4 is pinned to the official encoding_dsv4.py, and the community
reap-150b template on the Hub diverges on the reasoning-block convention (a bare
</think> for a direct answer vs <think></think>). So this is the expected-string
pin the maintainer allows for such families -- the open turn is pinned to a literal
here, with the past-turn-minus-EOS invariant kept as an added check, in both
thinking modes."""
EOS = "<|end▁of▁sentence|>"
ASSISTANT = "<|Assistant|>"
# The literal open turn, written out so the test does not lean on another renderer
# call for its only expected value -- a bug that corrupts render_chat_v4 and the
# continuation path identically would slip past the comparison below but not this.
# Identical in both thinking modes: the open turn is the shape of the PAST turn,
# which carries no generation cue for enable_thinking to steer.
EXPECTED_OPEN = ("<|begin▁of▁sentence|><|User|>capitale della Francia?"
"<|Assistant|></think>La capitale e'")
for enable_thinking in (True, False):
cue = ASSISTANT + ("<think>" if enable_thinking else "</think>")
with patch("openai_server.ARCH", "deepseek_v4"):
normal = render_chat_v4(self.OPEN_TURN, enable_thinking=enable_thinking)
cont = render_chat_for_arch(self.OPEN_TURN, enable_thinking=enable_thinking,
add_generation_prompt=False)
self.assertEqual(cont, EXPECTED_OPEN)
# and the invariant tying it to the normal render: past turn without its EOS + cue
self.assertEqual(normal, cont + EOS + cue)
self.assertTrue(cont.endswith("La capitale e'"), cont[-40:])
self.assertFalse(cont.endswith(EOS))
def test_continuation_open_turn_deepseek_v41(self):
"""deepseek_v41, like deepseek_v4, is pinned to encoding.py rather than a diffable
vendored jinja template, so it gets the same expected-string pin: the open turn is a
literal here, with the past-turn-minus-EOS invariant kept as an added check, in both
thinking modes.
Unlike v4, the v41 open turn is NOT identical across the modes: thinking-on carries the
<|System|> Reasoning Effort line and closes the continued turn's (empty) reasoning as
<think></think>; thinking-off has neither. The empty <think></think> is the SAFE
position #1327 is about, not the out-of-distribution one -- it is followed by the
client's content ('La capitale e''), never left dangling at the end of the prompt,
which resolve_generation_prompt guarantees by refusing an empty continuation."""
EOS = "<|end▁of▁sentence|>"
ASSISTANT = "<|Assistant|>"
EFFORT = ("<|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the "
"more thorough the reasoning)\n\n")
# The literal open turns, written out so the test does not lean on another renderer
# call for its only expected value (see the deepseek_v4 test), one per thinking mode.
EXPECTED_OPEN = {
True: ("<|begin▁of▁sentence|>" + EFFORT + "<|User|>capitale della Francia?"
"<|Assistant|><think></think>La capitale e'"),
False: ("<|begin▁of▁sentence|><|User|>capitale della Francia?"
"<|Assistant|></think>La capitale e'"),
}
for enable_thinking in (True, False):
cue = ASSISTANT + ("<think>" if enable_thinking else "</think>")
with patch("openai_server.ARCH", "deepseek_v41"):
normal = render_chat_dsv41(self.OPEN_TURN, enable_thinking=enable_thinking)
cont = render_chat_for_arch(self.OPEN_TURN, enable_thinking=enable_thinking,
add_generation_prompt=False)
self.assertEqual(cont, EXPECTED_OPEN[enable_thinking])
# and the invariant tying it to the normal render: past turn without its EOS + cue
self.assertEqual(normal, cont + EOS + cue)
self.assertTrue(cont.endswith("La capitale e'"), cont[-40:])
self.assertFalse(cont.endswith(EOS))
def test_continuation_open_turn_inkling(self):
"""inkling uses its own markers, and render_chat_inkling deliberately deviates from
the template's generation cue (it prefills <|content_text|> in the thinking-off case
to force content mode, and defaults thinking off), so it is pinned with an
expected-string test: the open turn is pinned to a literal here, with the
past-turn-minus-terminators invariant kept as an added check, in both thinking modes."""
END = "<|end_message|><|content_model_end_sampling|>"
for enable_thinking in (True, False):
# eff is 0.9 with thinking on, 0.0 off; the off cue prefills the content channel
eff = "0.9" if enable_thinking else "0"
cue = "<|message_model|>" + ("" if enable_thinking else "<|content_text|>")
# The literal open turn, written out so the test does not lean on another renderer
# call for its only expected value (see the deepseek_v4 test). The two modes differ
# only in the system effort line; both end on the prefilled content channel.
expected = ("<|message_system|><|content_text|>Thinking effort level: " + eff +
"<|end_message|><|message_user|><|content_text|>capitale della Francia?"
"<|end_message|><|message_model|><|content_text|>La capitale e'")
with patch("openai_server.ARCH", "inkling"):
normal = render_chat_inkling(self.OPEN_TURN, enable_thinking=enable_thinking)
cont = render_chat_for_arch(self.OPEN_TURN, enable_thinking=enable_thinking,
add_generation_prompt=False)
self.assertEqual(cont, expected)
# and the invariant tying it to the normal render: past turn without terminators + cue
self.assertEqual(normal, cont + END + cue)
self.assertTrue(cont.endswith("La capitale e'"), cont[-40:])
self.assertFalse(cont.endswith("<|end_message|>"))
def test_only_the_final_assistant_turn_is_opened(self):
"""Across every continuation family: only the TRAILING assistant turn is opened, and
each earlier turn renders exactly as it does in a completed conversation. The
single-turn fixtures elsewhere cannot see this -- their assistant turn is trivially
last -- but the terminator is dropped by a per-family `index == last` check, written
out by hand in each renderer; a family that lost that check would open every assistant
turn, and nothing else in the suite would notice.
Family-agnostic on purpose, because the open-turn SHAPE is not uniform: qwen36 injects
an empty <think></think>, GLM carries no per-turn terminator at all, ChatML drops an
<|im_end|>. What IS uniform is that the completed render (a fresh generation cue
appended) and the open render share their entire prefix up to the final turn -- so the
SECOND user turn must survive into their common prefix. If the first assistant turn
lost its terminator in the open render, that prefix would break right after it, before
this text. Checked in both thinking modes; the set drives the loop so a newly added
family is covered the day it joins.
Kimi K3 is excluded: render_chat_for_arch returns its engine-side K3CHAT1 wire, not a
string prompt, so this string-level invariant doesn't apply -- its open turn is pinned
at the token level in tests/test_k3_chat_tools.c against the tiny tokenizer instead."""
multi = [{"role": "user", "content": "1+1?"},
{"role": "assistant", "content": "2"},
{"role": "user", "content": "capitale della Francia?"},
{"role": "assistant", "content": "La capitale e'"}]
for arch in sorted(CONTINUATION_FAMILIES):
if arch == "kimi":
continue
for enable_thinking in (True, False):
where = (arch, enable_thinking)
with patch("openai_server.ARCH", arch):
completed = render_chat_for_arch(multi, enable_thinking=enable_thinking)
opened = render_chat_for_arch(multi, enable_thinking=enable_thinking,
add_generation_prompt=False)
common = os.path.commonprefix([opened, completed])
# the prior assistant turn (and its terminator) rendered identically: the turn
# AFTER it survives into the shared prefix
self.assertIn("capitale della Francia?", common, where)
# and only the last turn is open -- the prompt ends on the client's opening
self.assertTrue(opened.endswith("La capitale e'"), (where, opened[-40:]))
def test_splitter_starts_in_content_mode_on_a_continued_turn(self):
"""Measured on glm53 int4, CPU: content '' with reasoning_chars 10 and 109,
clean stop, and a byte-correct open turn on the wire. The model was fine; the
splitter was primed from enable_thinking alone, so it waited for a </think> the
prompt had already passed and filed the whole answer as reasoning.
starts_in_reasoning's own docstring is about exactly this invariant -- a continued
turn is the third state it did not model. Nothing else in the suite covers it:
every other thinking test runs against a prompt with the generation cue appended,
where enable_thinking really does say where the block was left.
Pinned to glm53 because that is the only family the switch serves. Left on the
module default (ARCH = "glm") this exercised the continuation path on a family
resolve_generation_prompt refuses, and passed for the wrong reason.
The family rule itself -- whether a NEW turn starts inside the block -- belongs to
#1278 and is pinned by its own test; asserted here it would only duplicate it. What
this test owns is the axis crossing it: a continued turn opens no block, whatever
the family rule says."""
with patch("openai_server.ARCH", "glm53"):
self.assertTrue(starts_in_reasoning(True)) # cue: block left open
self.assertFalse(starts_in_reasoning(True, add_generation_prompt=False))
self.assertFalse(starts_in_reasoning(False, add_generation_prompt=False))
# what the engine actually returns after "...The capital of France is": no
# markers, because the turn's <think></think> is already behind it in the prompt
emitted = " Paris."
reasoning, answer = split_thinking_reply(emitted, enable_thinking=True,
add_generation_prompt=False)
self.assertEqual(answer, emitted)
self.assertEqual(reasoning, "")
# and the bug it replaces, so this test fails if the priming is ever reverted
reasoning, answer = split_thinking_reply(emitted, enable_thinking=True,
add_generation_prompt=True)
self.assertEqual(answer, "")
self.assertEqual(reasoning, emitted)
def test_rejects_a_continuation_with_nothing_to_continue_from(self):
lone = [{"role": "assistant", "content": "La capitale e'"}]
with self.on(), patch("openai_server.ARCH", "glm53"):
with self.assertRaises(APIError):
resolve_generation_prompt(lone, {})
class TrailingAssistantEndToEndTest(unittest.TestCase):
"""End-to-end, through the real HTTP handler and a fake engine (no model weights): a
request whose last message is an `assistant` turn must reach the engine as a CONTINUATION
prompt -- ending on the client's own opening, no generation cue appended -- and the text
the engine generates must come back as the message `content`.
The TrailingAssistantTurnTest cases pin the prompt string in isolation; none of them prove
the wiring from an HTTP request through resolve_generation_prompt to the engine and back,
which a serve() refactor could silently drop. /v1/messages is a translation layer onto the
same engine path (not a second one), so /v1/chat/completions covers both endpoints. The
generated text arriving as content (not reasoning_content) also shows the continued turn
primes the splitter into content mode over the wire -- the third state #1327 fixed."""
OPEN = {"model": "test-model",
"messages": [{"role": "user", "content": "capitale della Francia?"},
{"role": "assistant", "content": "La capitale e'"}]}
def _server(self, engine):
server = APIServer(("127.0.0.1", 0), engine, "test-model")
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
self.addCleanup(thread.join, 2)
self.addCleanup(server.server_close)
self.addCleanup(server.shutdown)
self.addCleanup(server.scheduler.close)
return server
def test_continuation_reaches_the_engine_and_returns_as_content(self):
engine = FakeEngine()
with patch("openai_server.ARCH", "glm53"), \
patch.dict(os.environ, {"COLI_CONTINUE_ASSISTANT": "1"}):
server = self._server(engine)
conn = http.client.HTTPConnection("127.0.0.1", server.server_port, timeout=3)
self.addCleanup(conn.close)
conn.request("POST", "/v1/chat/completions", body=json.dumps(self.OPEN),
headers={"Content-Type": "application/json"})
response = conn.getresponse()
status, payload = response.status, response.read()
self.assertEqual(status, 200, payload)
# the engine saw the continuation prompt: it ends on the client's opening, no cue
self.assertEqual(len(engine.calls), 1)
prompt = engine.calls[0][0]
self.assertTrue(prompt.endswith("La capitale e'"), prompt[-60:])
self.assertFalse(prompt.endswith("<|assistant|><think>"))
# and the generated text comes back as content, not misfiled as reasoning
message = json.loads(payload)["choices"][0]["message"]
self.assertEqual(message["content"], "Héllo")
self.assertFalse(message.get("reasoning_content"))
class AllowedHostsTest(unittest.TestCase):
"""#597: the DNS-rebinding guard must accept operator-trusted reverse-proxy
Host values, while still rejecting everything else by default."""
def _make_server(self, allowed_hosts=()):
server = APIServer(("127.0.0.1", 0), FakeEngine(), "test-model",
allowed_hosts=allowed_hosts)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
self.addCleanup(thread.join, 2)
self.addCleanup(server.server_close)
self.addCleanup(server.shutdown)
self.addCleanup(server.scheduler.close)
return server
def _get_models(self, port, host_header):
# http.client lets us set an arbitrary Host header (urlopen forces its own).
conn = http.client.HTTPConnection("127.0.0.1", port, timeout=2)
try:
conn.putrequest("GET", "/v1/models", skip_host=True)
conn.putheader("Host", host_header)
conn.endheaders()
return conn.getresponse().status
finally:
conn.close()
def test_allowlist_wiring_normalises_and_filters(self):
server = self._make_server(allowed_hosts=(" Proxy.Example.TS.net ", "", " "))
self.assertEqual(server.allowed_hosts, ("proxy.example.ts.net",))
def test_trusted_reverse_proxy_host_is_accepted(self):
server = self._make_server(allowed_hosts=("proxy.example.ts.net",))
port = server.server_port
# trusted name (with a port suffix, case-insensitive) passes the guard
self.assertEqual(self._get_models(port, "Proxy.Example.TS.net:8000"), 200)
# loopback still works, unaffected by the allowlist
self.assertEqual(self._get_models(port, "localhost"), 200)
def test_untrusted_host_is_rejected_by_default(self):
server = self._make_server() # no allowlist: loopback/bind only
self.assertEqual(self._get_models(server.server_port, "evil.example.com"), 403)
def test_untrusted_host_still_rejected_with_allowlist(self):
server = self._make_server(allowed_hosts=("proxy.example.ts.net",))
self.assertEqual(self._get_models(server.server_port, "evil.example.com"), 403)
def test_wildcard_accepts_any_host(self):
# #990: a Docker/LAN bind reached by an unpredictable IP. The wildcard is
# an explicit operator opt-out, so ANY host passes -- but loopback and a
# real name still work, i.e. it widens rather than replaces.
server = self._make_server(allowed_hosts=("*",))
port = server.server_port
self.assertEqual(self._get_models(port, "10.20.30.40:36873"), 200)
self.assertEqual(self._get_models(port, "colibri.example.com"), 200)
self.assertEqual(self._get_models(port, "localhost"), 200)
def test_rejection_message_names_the_host_and_the_fix(self):
server = self._make_server()
conn = http.client.HTTPConnection("127.0.0.1", server.server_port, timeout=2)
try:
conn.putrequest("GET", "/v1/models", skip_host=True)
conn.putheader("Host", "myserver.lan:36873")
conn.endheaders()
resp = conn.getresponse()
body = resp.read().decode()
finally:
conn.close()
self.assertEqual(resp.status, 403)
self.assertIn("myserver.lan", body) # the offending host, so the user can copy it
self.assertIn("--allowed-host", body) # and how to fix it
class ThinkingSplitUnitTest(unittest.TestCase):
"""#597 item 4: the GLM reasoning splitter, incl. mkelcb's cross-chunk cases."""
def test_single_chunk(self):
self.assertEqual(split_thinking_reply("abc</think>def"), ("abc", "def"))
def test_close_tag_split_across_chunks(self):
thinking, answer = [], []
s = ThinkingStreamSplit(thinking.append, answer.append)
s.feed("abc</thi"); s.feed("nk>def"); s.finish()
self.assertEqual(("".join(thinking), "".join(answer)), ("abc", "def"))
def test_open_tag_split_and_stray_open_marker(self):
thinking, answer = [], []
s = ThinkingStreamSplit(thinking.append, answer.append)
s.feed("abc<thi"); s.feed("nk>def</think>ghi"); s.finish()
self.assertEqual(("".join(thinking), "".join(answer)), ("abcdef", "ghi"))
def test_thinking_disabled_is_all_answer(self):
# initial_thinking=False: a pure answer with no markers must not be filed as reasoning
self.assertEqual(split_thinking_reply("plain answer", enable_thinking=False),
("", "plain answer"))
def test_glm53_starts_in_reasoning_even_with_thinking_off(self):
"""#1278: render_chat_glm53 opens <think> unconditionally (the template
has no switch; "off" only lowers the effort), so the reply always starts
inside the block. With the splitter started in text mode the reasoning
streamed as `content`, glued in front of the answer. The family, not the
client flag, decides where the output starts."""
import openai_server as srv
with patch("openai_server.ARCH", "glm53"):
self.assertTrue(srv.starts_in_reasoning(False))
self.assertEqual(split_thinking_reply("why</think>answer", enable_thinking=False),
("why", "answer"))
with patch("openai_server.ARCH", "glm"):
self.assertFalse(srv.starts_in_reasoning(False),
"GLM-5.2 closes the block in the prompt when thinking is off")
def test_glm53_bare_tool_call_turn_matches_what_the_model_wrote(self):
"""#1576: a replayed assistant turn has to be the tokens the model made.
The official template writes "\\n<tool_call>" and GLM-5.3 does not: on a
turn that is nothing but a tool call it writes "</think><tool_call>".
Rendering the newline anyway put one extra token into the replayed
prefix, and the reuse gate in glm53.c is all-or-nothing, so the entire
cached prefix went and the turn re-prefilled from scratch. The reporter
measured twenty minutes of it on a 3k-token agent history.
The turn that also carries text keeps its newline, and that is not an
oversight: there the model's own trailing newline is stripped and put
back, the tokens line up, and it is the case that works today.
"""
import openai_server as srv
bare = srv.render_chat_glm53([
{"role": "user", "content": "list the files"},
{"role": "assistant", "content": "",
"tool_calls": [{"type": "function",
"function": {"name": "bash",
"arguments": '{"command": "ls"}'}}]},
{"role": "tool", "content": "a.txt"},
])
self.assertIn("</think><tool_call>bash", bare,
"a bare tool call must follow </think> with no newline")
self.assertNotIn("</think>\n<tool_call>", bare)
with_text = srv.render_chat_glm53([
{"role": "user", "content": "list the files"},
{"role": "assistant", "content": "Let me look.",
"tool_calls": [{"type": "function",
"function": {"name": "bash",
"arguments": '{"command": "ls"}'}}]},
{"role": "tool", "content": "a.txt"},
])
self.assertIn("Let me look.\n<tool_call>bash", with_text,
"a turn with text keeps the separator it already had")
def test_missing_close_tag_surfaces_reasoning(self):
self.assertEqual(split_thinking_reply("thought with no end"),
("thought with no end", ""))
class _ChunkEngine(FakeEngine):
"""Engine that emits a caller-supplied chunk sequence, to exercise the streaming
reasoning splitter across arbitrary chunk boundaries."""
def __init__(self, chunks):
super().__init__()
self.chunks = list(chunks)
def generate(self, prompt, maximum, temperature, top_p, on_text, cache_slot=0,
cancelled=None, grammar=None, stopped=None, on_accept=None):
self.calls.append((prompt, maximum, temperature, top_p, cache_slot, grammar))
if on_accept is not None: # simulate the engine's ACCEPT frame (#597)
on_accept({"prompt_tokens": 7})
for chunk in self.chunks:
on_text(chunk)
if stopped and stopped():
self.stop_requests += 1
break
return {"prompt_tokens": 7, "completion_tokens": len(self.chunks), "length_limited": False}
class GlmReasoningStreamTest(unittest.TestCase):
"""#597 item 4 end-to-end: GLM reasoning streams as reasoning_content, the answer as
content, no <think>/</think> leaks, cross-chunk-safe, and reasoning never contaminates
the tool-call buffer."""
def _server(self, chunks):
server = APIServer(("127.0.0.1", 0), _ChunkEngine(chunks), "test-model")
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
self.addCleanup(thread.join, 2)
self.addCleanup(server.server_close)
self.addCleanup(server.shutdown)
self.addCleanup(server.scheduler.close)
return f"http://127.0.0.1:{server.server_port}"
def _post(self, base, body):
req = Request(base + "/v1/chat/completions",
data=json.dumps(body).encode(),
headers={"Content-Type": "application/json"})
with urlopen(req, timeout=3) as response:
return response.read().decode()
def _deltas(self, raw):
reasoning, content, tool_calls = [], [], []
for line in raw.splitlines():
if not line.startswith("data: ") or line == "data: [DONE]":
continue
for choice in json.loads(line[6:])["choices"]:
delta = choice.get("delta") or {}
if delta.get("reasoning_content"):
reasoning.append(delta["reasoning_content"])
if delta.get("content"):
content.append(delta["content"])
if delta.get("tool_calls"):
tool_calls.extend(delta["tool_calls"])
return "".join(reasoning), "".join(content), tool_calls
def test_streaming_splits_reasoning_from_answer(self):
base = self._server(["I think ", "42", "</think>", "The answer ", "is 42"])
raw = self._post(base, {"model": "test-model", "stream": True, "enable_thinking": True,
"messages": [{"role": "user", "content": "2+2?"}]})
reasoning, content, _ = self._deltas(raw)
self.assertEqual(reasoning, "I think 42")
self.assertEqual(content, "The answer is 42")
self.assertNotIn("<think>", raw)
self.assertNotIn("</think>", raw)
def test_streaming_close_tag_split_across_chunks(self):
base = self._server(["reason</thi", "nk>ans", "wer"])
raw = self._post(base, {"model": "test-model", "stream": True, "enable_thinking": True,
"messages": [{"role": "user", "content": "x"}]})
reasoning, content, _ = self._deltas(raw)
self.assertEqual(reasoning, "reason")
self.assertEqual(content, "answer")
self.assertNotIn("think>", raw)
def test_streaming_thinking_off_is_all_content(self):
base = self._server(["Just ", "the answer"])
raw = self._post(base, {"model": "test-model", "stream": True, "enable_thinking": False,
"messages": [{"role": "user", "content": "x"}]})
reasoning, content, _ = self._deltas(raw)
self.assertEqual(reasoning, "")
self.assertEqual(content, "Just the answer")
def test_streaming_reasoning_stays_out_of_tool_call(self):
base = self._server(["deciding to call", "</think>",
"<tool_call>get_weather<arg_key>city</arg_key>"
"<arg_value>Paris</arg_value></tool_call>"])
raw = self._post(base, {"model": "test-model", "stream": True, "enable_thinking": True,
"messages": [{"role": "user", "content": "weather?"}],
"tools": [{"type": "function", "function": {
"name": "get_weather", "parameters": {"type": "object",
"properties": {"city": {"type": "string"}}}}}]})
reasoning, content, tool_calls = self._deltas(raw)
self.assertEqual(reasoning, "deciding to call")
self.assertTrue(tool_calls, "expected a parsed tool call")
args = tool_calls[0]["function"]["arguments"]
self.assertIn("Paris", args)
self.assertNotIn("deciding", args) # reasoning must not leak into the tool arguments
self.assertNotIn("deciding", content) # nor into the visible answer
def test_nonstreaming_splits_reasoning(self):
base = self._server(["mulling ", "it over", "</think>", "final ", "answer"])
req = Request(base + "/v1/chat/completions",
data=json.dumps({"model": "test-model", "enable_thinking": True,
"messages": [{"role": "user", "content": "x"}]}).encode(),
headers={"Content-Type": "application/json"})
with urlopen(req, timeout=3) as response:
body = json.load(response)
message = body["choices"][0]["message"]
self.assertEqual(message["reasoning_content"], "mulling it over")
self.assertEqual(message["content"], "final answer")
class AcceptFrameTest(unittest.TestCase):
"""#597 item 6: the engine's ACCEPT frame gates the HTTP commit, and its invariants."""
def _engine(self, respond):
process = FakeProcess(respond)
with patch("openai_server.subprocess.Popen", return_value=process):
return Engine("glm", "model")
def test_accept_fires_before_any_data(self):
def respond(process, frame):
rid = frame.split()[1]
process.stdout.feed(b"ACCEPT " + rid + b" 42\n")
process.stdout.feed(b"DATA " + rid + b" 3\nHi!\n")
process.stdout.feed(b"DONE " + rid + b" STAT 1 2.5 0 1.0 4 0\n")
engine = self._engine(respond)
seen = []
engine.generate("hi", 8, 0.7, 0.9, lambda t: seen.append(("text", t)),
on_accept=lambda info: seen.append(("accept", info)))
engine.close()
self.assertEqual(seen[0], ("accept", {"prompt_tokens": 42}))
self.assertEqual("".join(t for k, t in seen if k == "text"), "Hi!")
def test_error_before_accept_never_commits(self):
def respond(process, frame):
rid = frame.split()[1]
process.stdout.feed(b"ERROR " + rid + b" CONTEXT_EXCEEDED 5000 4094\n")
engine = self._engine(respond)
accepts = []
with self.assertRaises(APIError) as caught:
engine.generate("hi", 8, 0.7, 0.9, lambda _: None,
on_accept=lambda info: accepts.append(info))
engine.close()
self.assertEqual(caught.exception.status, 400)
self.assertEqual(caught.exception.code, "context_length_exceeded")
self.assertEqual(accepts, []) # nothing committed -> HTTP layer can send a clean 400
def test_data_before_accept_still_commits_for_old_engine(self):
def respond(process, frame):
rid = frame.split()[1]
process.stdout.feed(b"DATA " + rid + b" 3\nHey\n")
process.stdout.feed(b"DONE " + rid + b" STAT 1 2.5 0 1.0 4 0\n")
engine = self._engine(respond)
accepts, chunks = [], []
engine.generate("hi", 8, 0.7, 0.9, chunks.append,
on_accept=lambda info: accepts.append(info))
engine.close()
self.assertEqual("".join(chunks), "Hey")
self.assertEqual(len(accepts), 1) # first DATA implies acceptance (no ACCEPT frame)
self.assertIsNone(accepts[0]["prompt_tokens"])
def test_duplicate_accept_is_a_protocol_error(self):
def respond(process, frame):
rid = frame.split()[1]
process.stdout.feed(b"ACCEPT " + rid + b" 10\n")
process.stdout.feed(b"ACCEPT " + rid + b" 10\n")
engine = self._engine(respond)
with self.assertRaisesRegex(RuntimeError, "duplicate ACCEPT"):
engine.generate("hi", 8, 0.7, 0.9, lambda _: None, on_accept=lambda _: None)
engine.close()
class _ContextExceededEngine(FakeEngine):
"""Engine that rejects the prompt before ACCEPT — on_accept is never called."""
def generate(self, prompt, maximum, temperature, top_p, on_text, cache_slot=0,
cancelled=None, grammar=None, stopped=None, on_accept=None):
raise APIError(400, "This model's maximum context length is 4094 tokens.",
"messages", "context_length_exceeded")
class StreamingContextRejectTest(unittest.TestCase):
"""#597 item 6: an oversized prompt on a *streaming* request must return a clean HTTP 400,
not a committed 200 SSE stream that only later discovers the overflow."""
def setUp(self):
self.server = APIServer(("127.0.0.1", 0), _ContextExceededEngine(), "test-model")
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
self.thread.start()
self.base = f"http://127.0.0.1:{self.server.server_port}"
def tearDown(self):
self.server.scheduler.close()
self.server.shutdown()
self.server.server_close()
self.thread.join(timeout=2)
def test_streaming_context_exceeded_is_clean_400(self):
req = Request(self.base + "/v1/chat/completions",
data=json.dumps({"model": "test-model", "stream": True,
"messages": [{"role": "user", "content": "x" * 100}]}).encode(),
headers={"Content-Type": "application/json"})
with self.assertRaises(HTTPError) as caught:
urlopen(req, timeout=3)
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400) # a real 400, not a 200 stream
body = json.load(caught.exception)
self.assertEqual(body["error"]["code"], "context_length_exceeded")
self.assertEqual(body["error"]["param"], "messages")
class _ExplodingEngine(FakeEngine):
"""ACCEPTs the prompt (committing the streaming 200), then dies mid-generation."""
def generate(self, prompt, maximum, temperature, top_p, on_text, cache_slot=0,
cancelled=None, grammar=None, stopped=None, on_accept=None):
if on_accept is not None:
on_accept({"prompt_tokens": 7})
on_text("partial")
raise RuntimeError("engine died mid-stream")
class KeepAliveFramingTest(unittest.TestCase):
"""#597 item 3: HTTP/1.1 persistence must not desynchronise.
The report was `Bad request syntax ('{...json body...}POST /v1/chat/completions HTTP/1.1')`
-- a previous body being parsed as the next request line. Two independent causes: an early
rejection returning before the body is read, and a streaming 200 that neither announced
close-framing nor stopped offering the socket for reuse when generation failed."""
CHAT = {"model": "test-model", "messages": [{"role": "user", "content": "x"}]}
def _server(self, engine=None, **kw):
server = APIServer(("127.0.0.1", 0), engine or FakeEngine(), "test-model", **kw)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
self.addCleanup(thread.join, 2)
self.addCleanup(server.server_close)
self.addCleanup(server.shutdown)
self.addCleanup(server.scheduler.close)
return server
def _conn(self, server):
conn = http.client.HTTPConnection("127.0.0.1", server.server_port, timeout=3)
self.addCleanup(conn.close)
return conn
def _post(self, conn, body=None, headers=None, path="/v1/chat/completions"):
payload = json.dumps(self.CHAT if body is None else body)
head = {"Content-Type": "application/json"}
head.update(headers or {})
conn.request("POST", path, body=payload, headers=head)
response = conn.getresponse()
return response.status, response.read()
def _raw(self, server, request_bytes, read=4096):
"""Byte-level exchange, for assertions about framing that a client library hides."""
sock = socket.create_connection(("127.0.0.1", server.server_port), timeout=3)
self.addCleanup(sock.close)
sock.sendall(request_bytes)
chunks = []
try:
while True:
chunk = sock.recv(read)
if not chunk:
break # server closed: the SSE message boundary
chunks.append(chunk)
except socket.timeout:
chunks.append(b"<STILL-OPEN>") # no EOF: the connection was left reusable
return b"".join(chunks).decode("utf-8", "replace")
def _request_bytes(self, body, host="127.0.0.1"):
payload = json.dumps(body)
return (f"POST /v1/chat/completions HTTP/1.1\r\nHost: {host}\r\n"
f"Content-Type: application/json\r\nContent-Length: {len(payload)}\r\n"
f"\r\n{payload}").encode()
# --- an early rejection must not leave its body in the socket -------------------------
def test_rejected_host_does_not_desync_the_next_request(self):
server = self._server()
conn = self._conn(server)
status, _ = self._post(conn, headers={"Host": "evil.example.com"})
self.assertEqual(status, 403)
status, payload = self._post(conn) # same connection, valid request
self.assertEqual(status, 200, "the 403's unread body desynchronised the connection")
self.assertEqual(json.loads(payload)["object"], "chat.completion")
def test_rejected_auth_does_not_desync_the_next_request(self):
server = self._server(api_key="secret")
conn = self._conn(server)
status, _ = self._post(conn) # no Authorization header
self.assertEqual(status, 401)
status, payload = self._post(conn, headers={"Authorization": "Bearer secret"})
self.assertEqual(status, 200, "the 401's unread body desynchronised the connection")
self.assertEqual(json.loads(payload)["object"], "chat.completion")
def test_unknown_model_does_not_desync_the_next_request(self):
server = self._server()
conn = self._conn(server)
status, _ = self._post(conn, body=dict(self.CHAT, model="nope"))
self.assertEqual(status, 404)
status, _ = self._post(conn)
self.assertEqual(status, 200)
def test_each_body_is_consumed_exactly_once_across_reused_requests(self):
"""The engine sees one prompt per request, with no body bytes bleeding between them."""
engine = FakeEngine()
server = self._server(engine)
conn = self._conn(server)
for index in range(4):
body = {"model": "test-model",
"messages": [{"role": "user", "content": f"question-{index}"}]}
status, _ = self._post(conn, body=body)
self.assertEqual(status, 200)
self.assertEqual(len(engine.calls), 4)
for index, call in enumerate(engine.calls):
self.assertIn(f"question-{index}", call[0])
self.assertNotIn("question-", call[0].split(f"question-{index}")[1],
"a later body leaked into an earlier prompt")
def test_interleaved_rejections_and_successes_stay_in_sync(self):
server = self._server(api_key="secret")
conn = self._conn(server)
good = {"Authorization": "Bearer secret"}
for _ in range(3):
self.assertEqual(self._post(conn)[0], 401)
self.assertEqual(self._post(conn, headers={"Host": "evil.example.com", **good})[0], 403)
self.assertEqual(self._post(conn, headers=good)[0], 200)
# --- bodies we refuse to swallow must close rather than desynchronise -----------------
def test_oversized_content_length_closes_instead_of_desyncing(self):
server = self._server()
raw = self._raw(server, (b"POST /v1/chat/completions HTTP/1.1\r\nHost: 127.0.0.1\r\n"
b"Content-Type: application/json\r\n"
b"Content-Length: 99999999\r\n\r\n{}"))
self.assertIn(" 400 ", raw.splitlines()[0])
self.assertNotIn("<STILL-OPEN>", raw,
"an over-limit body must close the connection, not keep it alive")
def test_unparseable_content_length_closes_instead_of_desyncing(self):
server = self._server()
raw = self._raw(server, (b"POST /v1/chat/completions HTTP/1.1\r\nHost: 127.0.0.1\r\n"
b"Content-Type: application/json\r\n"
b"Content-Length: abc\r\n\r\n{}"))
self.assertIn("400", raw.splitlines()[0])
self.assertNotIn("<STILL-OPEN>", raw)
# --- a streaming 200 is close-framed, and says so ------------------------------------
def test_streaming_response_announces_close_framing(self):
server = self._server()
raw = self._raw(server, self._request_bytes(dict(self.CHAT, stream=True)))
headers = raw.split("\r\n\r\n", 1)[0].lower()
self.assertIn("content-type: text/event-stream", headers)
self.assertIn("connection: close", headers,
"SSE has no Content-Length, so the close IS the boundary and must be declared")
self.assertIn("data: [DONE]", raw)
self.assertNotIn("<STILL-OPEN>", raw)
def test_anthropic_stream_announces_close_framing(self):
server = self._server()
payload = json.dumps({"model": "test-model", "stream": True, "max_tokens": 16,
"messages": [{"role": "user", "content": "x"}]})
raw = self._raw(server, (f"POST /v1/messages HTTP/1.1\r\nHost: 127.0.0.1\r\n"
f"Content-Type: application/json\r\n"
f"Content-Length: {len(payload)}\r\n\r\n{payload}").encode())
self.assertIn("connection: close", raw.split("\r\n\r\n", 1)[0].lower())
self.assertIn("event: message_stop", raw)
self.assertNotIn("<STILL-OPEN>", raw)
def test_stream_exit_stops_keepalive_before_releasing_slot(self):
class CancelledEngine(_ExplodingEngine):
def generate(self, *args, **kwargs):
try:
return super().generate(*args, **kwargs)
except RuntimeError:
raise ClientCancelled()
original_thread = threading.Thread
for path in ("/v1/chat/completions", "/v1/messages"):
for engine_type, outcome in ((_ExplodingEngine, "failed"),
(CancelledEngine, "cancelled")):
with self.subTest(path=path, outcome=outcome):
pumps = []
def thread_factory(*args, **kwargs):
thread = original_thread(*args, **kwargs)
target = kwargs.get("target")
if getattr(target, "__name__", "") in ("_keepalive", "keepalive"):
stop = next(cell.cell_contents for cell in target.__closure__
if isinstance(cell.cell_contents, threading.Event))
pumps.append((thread, stop))
return thread
server = self._server(engine_type())
try:
with patch.object(threading, "Thread", side_effect=thread_factory):
status, _ = self._post(self._conn(server),
dict(self.CHAT, stream=True, max_tokens=16), path=path)
self.assertEqual(status, 200)
self.assertEqual(len(pumps), 1)
self.assertFalse(pumps[0][0].is_alive(), "keepalive survived stream exit")
stats = server.scheduler.snapshot()
self.assertEqual(stats[outcome], 1)
self.assertEqual((stats["active"], stats["completed"]), (0, 0))
finally:
for thread, stop in pumps:
stop.set()
thread.join(2)
def test_engine_failure_after_commit_does_not_splice_a_second_response(self):
"""Once the 200 is out, a 500 status line would land inside the event stream."""
server = self._server(_ExplodingEngine())
raw = self._raw(server, self._request_bytes(dict(self.CHAT, stream=True)))
self.assertEqual(raw.count("HTTP/1."), 1,
"a second HTTP response was spliced into the committed SSE stream")
self.assertIn("partial", raw) # the events sent before the failure survive
self.assertNotIn("<STILL-OPEN>", raw)
def test_non_streaming_response_still_reuses_the_connection(self):
"""The fix must not turn every response into a close: plain JSON stays persistent."""
server = self._server()
conn = self._conn(server)
self.assertEqual(self._post(conn)[0], 200)
self.assertEqual(self._post(conn)[0], 200)
self.assertIsNotNone(conn.sock, "the JSON path should keep the connection open")
class ConversationCacheSlotTest(unittest.TestCase):
"""#634 Defect 1: a conversation must map to one stable KV slot across its turns."""
def _conv(self, *user_and_assistant_turns, system="you are a helpful assistant"):
messages = [{"role": "system", "content": system}]
for i, text in enumerate(user_and_assistant_turns):
messages.append({"role": "user" if i % 2 == 0 else "assistant", "content": text})
return messages
def test_single_slot_is_always_zero(self):
self.assertEqual(conversation_cache_slot(self._conv("hi"), 1), 0)
def test_empty_or_bad_input_is_zero(self):
self.assertEqual(conversation_cache_slot(None, 8), 0)
self.assertEqual(conversation_cache_slot([], 8), 0)
def test_slot_is_in_range(self):
for kv in (2, 3, 8, 16):
slot = conversation_cache_slot(self._conv("solve x"), kv)
self.assertTrue(0 <= slot < kv, f"slot {slot} out of range for kv={kv}")
def test_stable_across_turns_of_one_conversation(self):
# The engine caches the prefix; every turn of the same conversation must return
# the same slot so it lands on its warm KV instead of re-prefilling.
first_turn = self._conv("1+1=")
second_turn = self._conv("1+1=", "2", "and 2+2?")
third_turn = self._conv("1+1=", "2", "and 2+2?", "4", "thanks")
base = conversation_cache_slot(first_turn, 8)
self.assertEqual(conversation_cache_slot(second_turn, 8), base)
self.assertEqual(conversation_cache_slot(third_turn, 8), base)
def test_distinct_conversations_can_differ(self):
# Not a guarantee for any single pair (hashing collides sometimes), but across a
# spread of openings we must see more than one slot used, i.e. not everything on 0.
slots = {conversation_cache_slot(self._conv(f"task number {i}"), 8) for i in range(40)}
self.assertGreater(len(slots), 1)
def test_deterministic(self):
conv = self._conv("same question", "same answer", "again")
self.assertEqual(conversation_cache_slot(conv, 8), conversation_cache_slot(conv, 8))
class ConnectionLimitTest(unittest.TestCase):
"""Bounds on the accept loop, which nothing bounded before.
ThreadingHTTPServer spawns a thread per connection with no ceiling, and
`timeout` is per socket operation, so it restarts on every byte: a client
dripping one byte kept a thread and a slot forever. Threads at 8 MiB of
stack each made that a memory-exhaustion DoS, reachable before any Host
check or auth.
"""
def setUp(self):
self.engine = FakeEngine()
APIServer.MAX_CONNECTIONS = 6
APIServer.MAX_CONNECTIONS_PER_IP = 3
APIHandler.READ_DEADLINE = 2
self.addCleanup(setattr, APIServer, "MAX_CONNECTIONS", 64)
self.addCleanup(setattr, APIServer, "MAX_CONNECTIONS_PER_IP", 8)
self.addCleanup(setattr, APIHandler, "READ_DEADLINE", 30)
self.server = APIServer(("127.0.0.1", 0), self.engine, "m", None, 16, kv_slots=1)
threading.Thread(target=self.server.serve_forever, daemon=True).start()
self.addCleanup(self.server.server_close)
self.addCleanup(self.server.shutdown)
self.addCleanup(self.server.scheduler.close)
self.port = self.server.server_port
self.held = []
self.addCleanup(self._drop_all)
def _drop_all(self):
for sock in self.held:
try:
sock.close()
except OSError:
pass
def _dribble(self, count):
"""Open connections that send a partial request line and never finish."""
for _ in range(count):
try:
sock = socket.create_connection(("127.0.0.1", self.port), 2)
sock.settimeout(2)
sock.sendall(b"GET /health HTTP/1.1\r\n")
self.held.append(sock)
except OSError:
pass
time.sleep(0.4)
def test_slowloris_cannot_grow_threads_without_bound(self):
self._dribble(40)
self.assertLessEqual(self.server._conn_live, APIServer.MAX_CONNECTIONS)
def test_one_address_cannot_take_every_slot(self):
"""A global cap alone only turns exhaustion into starvation."""
self._dribble(40)
self.assertLessEqual(self.server._conn_live,
APIServer.MAX_CONNECTIONS_PER_IP)
self.assertLess(self.server._conn_live, APIServer.MAX_CONNECTIONS,
"one source filled the server cap")
def test_dripping_connections_are_reclaimed(self):
"""The cumulative deadline, which the per-operation timeout is not."""
self._dribble(10)
self.assertGreater(self.server._conn_live, 0)
time.sleep(APIHandler.READ_DEADLINE + 1.5)
self.assertEqual(self.server._conn_live, 0,
"dripping connections were never reclaimed")
class ReasoningEffortTest(unittest.TestCase):
"""render_chat mapped every level except "high" onto Max (#809).
The endpoint accepts none/minimal/low/medium/high/xhigh. Four of the five
that enable thinking rendered Max, so a client asking for `minimal` got
more reasoning than one asking for `high` -- and on a single machine
unrequested reasoning spends the token budget before the answer starts.
"""
MESSAGES = [{"role": "user", "content": "hi"}]
def effort(self, level):
import re
text = render_chat(self.MESSAGES, enable_thinking=True,
reasoning_effort=level)
found = re.search(r"Reasoning Effort: (\w+)", text)
return found.group(1) if found else None
def test_levels_are_distinct_and_ordered(self):
rendered = [self.effort(l) for l in
("minimal", "low", "medium", "high", "xhigh")]
rank = {"Low": 0, "Medium": 1, "High": 2, "Max": 3}
scores = [rank[r] for r in rendered]
self.assertEqual(scores, sorted(scores), rendered)
self.assertLess(scores[0], scores[-1],
"minimal and xhigh render the same effort")
def test_minimal_is_not_max(self):
"""The reported symptom, pinned on its own."""
self.assertNotEqual(self.effort("minimal"), "Max")
self.assertNotEqual(self.effort("low"), "Max")
self.assertNotEqual(self.effort("medium"), "Max")
def test_thinking_off_emits_no_effort_line(self):
text = render_chat(self.MESSAGES, enable_thinking=False,
reasoning_effort="xhigh")
self.assertNotIn("Reasoning Effort", text)
class ImageUrlPathGuard(unittest.TestCase):
"""image_url.url naming a local file is read with the server's rights, so
it is denied unless the operator sets COLI_IMAGE_ROOT, and then only
inside it (GHSA follow-up to #1354, whose guards left every absolute path
readable by default). data: URIs are the client's way in. Errors stay
generic so a reply never confirms a path or its permissions."""
def setUp(self):
self._saved = os.environ.pop("COLI_IMAGE_ROOT", None)
def tearDown(self):
os.environ.pop("COLI_IMAGE_ROOT", None)
if self._saved is not None:
os.environ["COLI_IMAGE_ROOT"] = self._saved
def test_data_uri_is_the_default_way_in(self):
import base64
payload = base64.b64encode(b"\x89PNG\r\n").decode()
self.assertEqual(_image_bytes_from_url("data:image/png;base64," + payload), b"\x89PNG\r\n")
def test_local_paths_are_denied_by_default(self):
with tempfile.TemporaryDirectory() as root:
img = Path(root) / "pic.png"
img.write_bytes(b"\x89PNG\r\n")
for url in (str(img), "file://" + str(img), "/etc/passwd", "file:///etc/passwd", "relative.png"):
with self.assertRaises(APIError) as caught:
_image_bytes_from_url(url)
self.assertEqual(caught.exception.status, 400)
self.assertIn("COLI_IMAGE_ROOT", str(caught.exception))
self.assertNotIn("passwd", str(caught.exception))
def test_image_root_allows_inside_and_refuses_outside(self):
with tempfile.TemporaryDirectory() as root, \
tempfile.NamedTemporaryFile(suffix=".png") as outside:
os.environ["COLI_IMAGE_ROOT"] = root
inside = Path(root) / "ok.png"
inside.write_bytes(b"ok")
self.assertEqual(_image_bytes_from_url(str(inside)), b"ok")
self.assertEqual(_image_bytes_from_url("file://" + str(inside)), b"ok")
for url in (outside.name, "/etc/passwd", str(Path(root) / ".." / Path(outside.name).name)):
with self.assertRaises(APIError) as caught:
_image_bytes_from_url(url)
self.assertNotIn(Path(outside.name).name, str(caught.exception))
def test_a_symlink_escaping_the_root_is_refused(self):
if not hasattr(os, "symlink"):
self.skipTest("no symlinks here")
with tempfile.TemporaryDirectory() as root, \
tempfile.NamedTemporaryFile(suffix=".png") as outside:
os.environ["COLI_IMAGE_ROOT"] = root
link = Path(root) / "link.png"
try:
os.symlink(outside.name, link)
except OSError:
self.skipTest("cannot create symlinks here")
with self.assertRaises(APIError):
_image_bytes_from_url(str(link))
def test_a_missing_or_file_root_denies_everything(self):
with tempfile.TemporaryDirectory() as root:
img = Path(root) / "pic.png"
img.write_bytes(b"x")
os.environ["COLI_IMAGE_ROOT"] = str(Path(root) / "nowhere")
with self.assertRaises(APIError):
_image_bytes_from_url(str(img))
os.environ["COLI_IMAGE_ROOT"] = str(img)
with self.assertRaises(APIError):
_image_bytes_from_url(str(img))
def test_error_does_not_leak_the_path(self):
os.environ["COLI_IMAGE_ROOT"] = tempfile.gettempdir()
with self.assertRaises(APIError) as caught:
_image_bytes_from_url("/no/such/secret-name.png")
self.assertNotIn("secret-name", str(caught.exception))
class ContextExceededMessageTest(unittest.TestCase):
"""#1376: the engine writes `CONTEXT_EXCEEDED prompt_tokens=N requested=M
capacity=C`. The message took fields[2] ("requested=M", the completion
budget) as the limit and printed the raw key=value token, so a user with a
9000-token prompt on an 8192 ceiling read "maximum context length is
requested=4 tokens". The number that mattered, capacity, appeared nowhere."""
def test_limit_is_capacity_and_used_is_prompt_tokens(self):
from openai_server import _engine_error
err = _engine_error(["CONTEXT_EXCEEDED", "prompt_tokens=9000", "requested=4",
"capacity=8192"], "ignored")
text = str(err)
self.assertIn("8192", text)
self.assertIn("9000", text)
self.assertNotIn("requested=", text)
self.assertNotIn("prompt_tokens=", text)
self.assertNotIn("capacity=", text)
def test_the_positional_spelling_of_colibri_and_deepseek_still_reads(self):
from openai_server import _engine_error
text = str(_engine_error(["CONTEXT_EXCEEDED", "8321", "4094"], "ignored"))
self.assertIn("4094", text)
self.assertIn("8321", text)
def test_missing_fields_do_not_crash_the_message(self):
from openai_server import _engine_error
text = str(_engine_error(["CONTEXT_EXCEEDED"], "ignored"))
self.assertIn("the context", text)
class _ShortWritingStream:
"""A fake stdin that hands back at most `chunk` bytes per write() call,
forcing `_write_all` to loop -- the production pipe does this on a
signal landing mid-write or a full pipe buffer on a large IMAGE frame."""
def __init__(self, chunk=3):
self.chunk = chunk
self.received = bytearray()
def write(self, data):
piece = bytes(data)[:self.chunk]
self.received.extend(piece)
return len(piece)
class _ScriptedStream:
"""A fake stdin whose write() answers each entry in `script` in turn --
an int number of bytes actually taken, or `None` for "took nothing" --
then takes everything it is offered once the script runs out."""
def __init__(self, script):
self.script = list(script)
self.received = bytearray()
self.calls = []
def write(self, data):
data = bytes(data)
self.calls.append(data)
answer = self.script.pop(0) if self.script else len(data)
if answer is None:
return None
taken = data[:answer]
self.received.extend(taken)
return len(taken)
class _CountingLock:
"""A `threading.Lock`-alike that counts `with` acquisitions -- used to
pin that IMAGE and SUBMIT share exactly one `write_lock` acquisition."""
def __init__(self):
self._lock = threading.Lock()
self.acquisitions = 0
def __enter__(self):
self.acquisitions += 1
return self._lock.__enter__()
def __exit__(self, *exc_info):
return self._lock.__exit__(*exc_info)
class WriteAllTest(unittest.TestCase):
"""`_write_all` (extracted from the inline server->engine stdin writes):
loop on a short write until the whole frame is sent, and fail closed --
never spin -- on the two shapes that are not progress.
`_write_all` is imported locally in each test method here, not at module
scope: it does not exist on base, and a module-level import of it would
make this whole test file fail to collect when overlaid on base product
code (the procedure the base-pin tests below rely on)."""
def test_short_writes_reassemble_to_the_full_frame(self):
from openai_server import _write_all
stream = _ShortWritingStream(chunk=3)
frame = b"SUBMIT 1 0 5 3 0.25 0.9\nHello\n"
_write_all(stream, frame, "SUBMIT")
self.assertEqual(bytes(stream.received), frame)
def test_a_short_first_write_still_receives_the_correct_remainder(self):
from openai_server import _write_all
# The first call must not be special-cased to the unsliced buffer:
# every call, including the first, offers a memoryview starting at
# the bytes not yet sent.
stream = _ScriptedStream([2])
_write_all(stream, b"CANCEL 42\n", "CANCEL")
self.assertEqual(bytes(stream.received), b"CANCEL 42\n")
self.assertEqual(len(stream.calls), 2) # the short first write forced a second
def test_none_return_fails_closed_instead_of_spinning(self):
from openai_server import _write_all
stream = _ScriptedStream([None])
with self.assertRaisesRegex(RuntimeError, "failed to write SUBMIT to the engine"):
_write_all(stream, b"SUBMIT 1 0 1 1 1 1\nx\n", "SUBMIT")
def test_zero_return_fails_closed_instead_of_spinning(self):
from openai_server import _write_all
stream = _ScriptedStream([0])
with self.assertRaisesRegex(RuntimeError, "failed to write STOP to the engine"):
_write_all(stream, b"STOP 1\n", "STOP")
class WriteFailureHTTPTest(unittest.TestCase):
"""A checked engine-stdin write that fails must reach the client as the
named 500 engine_error while the response is still uncommitted, and must
never splice anything into a stream that has already committed -- it
just ends (#597 item 3's `_fail`, exercised here by a genuine broken
pipe on the engine's stdin, through the real `Engine`, rather than by a
fake that raises `RuntimeError` directly -- a real `BrokenPipeError` is
a `ConnectionError`, and it is exactly that unwrapped subclass that a
correct checked write must keep away from do_POST's client-hangup
handler)."""
def _serve(self, process):
with patch("openai_server.ARCH", "glm"), \
patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
server = APIServer(("127.0.0.1", 0), engine, "test-model", None, 16, kv_slots=1)
# A short poll_interval means server.shutdown() (addCleanup, below)
# returns almost immediately instead of paying up to the default
# 0.5s poll -- these tests do no waiting of their own, so that 0.5s
# would be pure teardown overhead, not a wait under test.
thread = threading.Thread(target=server.serve_forever, args=(0.01,), daemon=True)
thread.start()
self.addCleanup(thread.join, 2)
self.addCleanup(server.server_close)
self.addCleanup(server.shutdown)
self.addCleanup(server.scheduler.close)
self.addCleanup(engine.close)
return server
def _raw_post(self, server, path, body):
"""POST over a plain socket and read until the peer closes, so the
literal status line(s) on the wire are visible -- `urlopen` strips
the status line into `response.status` before handing back `.read()`,
so asserting on `.read()` alone cannot tell "one status line" from
"a second one spliced into the body"."""
payload = json.dumps(body).encode()
request = (f"POST {path} HTTP/1.1\r\nHost: 127.0.0.1\r\n"
f"Content-Type: application/json\r\nContent-Length: {len(payload)}\r\n"
f"Connection: close\r\n\r\n").encode() + payload
sock = socket.create_connection(("127.0.0.1", server.server_port), 2)
sock.sendall(request)
sock.settimeout(2)
chunks = []
try:
while True:
chunk = sock.recv(4096)
if not chunk:
break
chunks.append(chunk)
except socket.timeout:
pass
sock.close()
return b"".join(chunks)
def test_dead_engine_submit_is_a_named_500_engine_error_not_silence(self):
def respond(process, frame):
if frame.split()[0] == b"SUBMIT":
raise BrokenPipeError("broken pipe")
server = self._serve(FakeProcess(respond))
body = json.dumps({"model": "test-model", "prompt": "hi"}).encode()
request = Request(f"http://127.0.0.1:{server.server_port}/v1/completions",
data=body, headers={"Content-Type": "application/json"})
with self.assertRaises(HTTPError) as caught:
urlopen(request, timeout=2)
self.assertEqual(caught.exception.code, 500)
payload = json.load(caught.exception)
self.assertEqual(payload["error"]["code"], "engine_error")
def test_dead_engine_submit_is_a_named_500_on_the_chat_streaming_path(self):
# Same failure, but through /v1/chat/completions with stream: true --
# the tool-sideband/ThinkingStreamSplit wrapping chat streaming builds
# around engine.generate() must not swallow or reshape a RuntimeError
# that fires before anything is committed (nothing here ever reaches
# that wrapping: the SUBMIT write fails before the first ACCEPT/DATA).
def respond(process, frame):
if frame.split()[0] == b"SUBMIT":
raise BrokenPipeError("broken pipe")
server = self._serve(FakeProcess(respond))
body = json.dumps({"model": "test-model", "stream": True,
"messages": [{"role": "user", "content": "hi"}]}).encode()
request = Request(f"http://127.0.0.1:{server.server_port}/v1/chat/completions",
data=body, headers={"Content-Type": "application/json"})
with self.assertRaises(HTTPError) as caught:
urlopen(request, timeout=2)
self.assertEqual(caught.exception.code, 500)
payload = json.load(caught.exception)
self.assertEqual(payload["error"]["code"], "engine_error")
def test_uncommitted_stop_write_failure_is_a_named_500_engine_error(self):
# The dead-SUBMIT tests above cover the write _write_frame never gets
# a chance to make (the connection is refused before the first
# request even lands). This is the case _write_frame's own docstring
# is actually about: a CANCEL/STOP write that fails while the
# response is still uncommitted -- non-streaming, so nothing commits
# until the whole answer is assembled, and the STOP write happens
# well before that.
request_id = None
def respond(process, frame):
nonlocal request_id
fields = frame.split()
if fields[0] == b"SUBMIT":
request_id = fields[1]
process.stdout.feed(b"DATA " + request_id + b" 3\nhi \n")
process.stdout.feed(b"DATA " + request_id + b" 4\nSTOP\n")
elif fields[0] == b"STOP":
raise BrokenPipeError("broken pipe")
server = self._serve(FakeProcess(respond))
body = json.dumps({"model": "test-model", "prompt": "hi",
"stop": "STOP"}).encode()
request = Request(f"http://127.0.0.1:{server.server_port}/v1/completions",
data=body, headers={"Content-Type": "application/json"})
with self.assertRaises(HTTPError) as caught:
urlopen(request, timeout=2)
self.assertEqual(caught.exception.code, 500)
payload = json.load(caught.exception)
self.assertEqual(payload["error"]["code"], "engine_error")
def test_write_failure_reaching_the_committed_stream_ends_it_cleanly(self):
request_id = None
def respond(process, frame):
nonlocal request_id
fields = frame.split()
if fields[0] == b"SUBMIT":
request_id = fields[1]
# "hi " clears StopFilter's hold and reaches the client; the
# stop sequence itself never does -- feeding them as two
# frames pins the STOP write to the SECOND one, after the
# stream has already committed on the first.
process.stdout.feed(b"DATA " + request_id + b" 3\nhi \n")
process.stdout.feed(b"DATA " + request_id + b" 4\nSTOP\n")
elif fields[0] == b"STOP":
raise BrokenPipeError("broken pipe")
server = self._serve(FakeProcess(respond))
raw = self._raw_post(server, "/v1/completions",
{"model": "test-model", "prompt": "hi", "stream": True,
"stop": "STOP"})
# Exactly one status line on the whole wire -- the original 200. A
# response that spliced a second status line (or any framed error)
# into the already-committed SSE body would show up here as 2.
self.assertEqual(raw.count(b"HTTP/1.1"), 1)
self.assertIn(b"200", raw.split(b"\r\n", 1)[0])
# The committed token reached the client...
self.assertIn(b'"hi "', raw)
# ...and nothing else did: no error object, no terminal [DONE]
# (generate() never returned normally).
self.assertNotIn(b"engine_error", raw)
self.assertNotIn(b"data: [DONE]", raw)
class PendingMapCleanupTest(unittest.TestCase):
"""The pending-map entry is dropped on every failed engine write --
SUBMIT, IMAGE, CANCEL or STOP -- including one forced by a checked
CANCEL/STOP write that fails closed with no `OSError` (a `None`/zero
return). A write that never reached the engine gets no DONE/ERROR back,
so nothing else would ever clear the slot; without this it sits behind
for `close()`/`_fail_pending` to find stale.
This is narrower than "every exit": a raise from inside a decode
callback (`on_text`/`on_accept`/`on_tool`/`on_echo`), or the duplicate-
ACCEPT guard, still leaves the entry behind, as on `dev`; that is not
changed here."""
def test_a_failed_cancel_write_does_not_leave_the_pending_entry_behind(self):
request_id = None
def respond(process, frame):
nonlocal request_id
fields = frame.split()
if fields[0] == b"SUBMIT":
request_id = fields[1]
process.stdout.feed(b"DATA " + request_id + b" 1\nx\n")
elif fields[0] == b"CANCEL":
raise BrokenPipeError("broken pipe")
process = FakeProcess(respond)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
with self.assertRaisesRegex(RuntimeError, "failed to write CANCEL to the engine"):
engine.generate("hello", 8, 0.7, 0.9, lambda _: None, cancelled=lambda: True)
with engine.pending_lock:
self.assertEqual(engine.pending, {})
engine.close()
def test_a_failed_stop_write_does_not_leave_the_pending_entry_behind(self):
request_id = None
def respond(process, frame):
nonlocal request_id
fields = frame.split()
if fields[0] == b"SUBMIT":
request_id = fields[1]
process.stdout.feed(b"DATA " + request_id + b" 1\nx\n")
elif fields[0] == b"STOP":
raise BrokenPipeError("broken pipe")
process = FakeProcess(respond)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
output = []
with self.assertRaisesRegex(RuntimeError, "failed to write STOP to the engine"):
engine.generate("hello", 8, 0.7, 0.9, output.append,
stopped=lambda: output == ["x"])
with engine.pending_lock:
self.assertEqual(engine.pending, {})
engine.close()
def test_a_cancel_write_that_takes_no_bytes_still_drops_the_pending_entry(self):
# Not an OSError: the pipe accepts the call and answers 0, the same
# "took nothing" shape _write_all treats as a fail-closed write. If
# _write_frame's cleanup only ran for OSError, this entry would
# survive -- the pop has to be keyed on failure of the write, not on
# which failure shape it took.
class ZeroOnCancelProcess(FakeProcess):
def write(self, data):
text = bytes(data)
fields = text.split()
if fields[:1] == [b"CANCEL"]:
self.writes.append(text)
# Base ignores write()'s return value entirely, so a bare
# `return 0` here would leave base's generate() waiting
# forever for an ERROR/DONE it never actually asked for
# (the CANCEL it believes it sent never reached the
# engine) -- a hang, not a failure, on the base-overlay
# procedure. Feeding the terminal frame here means base
# fails in seconds instead; at head the RuntimeError from
# the zero-byte write fires first, so this frame arrives
# after the pending entry is already gone and is a no-op.
self.stdout.feed(b"ERROR " + fields[1] + b" CANCELLED\n")
return 0
return super().write(data)
request_id = None
def respond(process, frame):
nonlocal request_id
fields = frame.split()
if fields[0] == b"SUBMIT":
request_id = fields[1]
process.stdout.feed(b"DATA " + request_id + b" 1\nx\n")
process = ZeroOnCancelProcess(respond)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
with self.assertRaisesRegex(RuntimeError, "failed to write CANCEL to the engine"):
engine.generate("hello", 8, 0.7, 0.9, lambda _: None, cancelled=lambda: True)
with engine.pending_lock:
self.assertEqual(engine.pending, {})
engine.close()
def test_a_failed_image_write_names_the_image_frame_and_drops_the_pending_entry(self):
# generate()'s outer handler wraps both the IMAGE and the SUBMIT
# write with one except OSError -- an OSError on the IMAGE write
# specifically must still name "IMAGE", not fall through to the
# handler's own default "SUBMIT" label, or one frame kind would
# report under two different names depending on failure shape
# (_write_all's own None/zero path already says "IMAGE"; an OSError
# is the likelier real failure and must match).
class BrokenPipeOnImageProcess(FakeProcess):
def write(self, data):
if bytes(data).split()[:1] == [b"IMAGE"]:
raise BrokenPipeError("broken pipe")
return super().write(data)
process = BrokenPipeOnImageProcess(lambda _process, _frame: None)
with patch("openai_server.ARCH", "glm53"), \
patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm53", "model")
class FakePatches:
def tobytes(self):
return bytes(range(8))
with self.assertRaisesRegex(RuntimeError, "failed to write IMAGE to the engine"):
engine.generate("hello", 8, 0.7, 0.9, lambda _: None, image=(FakePatches(), 2, 2))
with engine.pending_lock:
self.assertEqual(engine.pending, {})
engine.close()
class PlainRequestFrameOrderTest(unittest.TestCase):
"""A request using none of the checked-write machinery's new surface
(no image, no grammar, no logprobs, no pin) must still put byte-identical
SUBMIT/STOP and SUBMIT/CANCEL frames on the wire, in the same order, as
the base module -- literals captured by running the base module's own
Engine.generate() against this same FakeProcess harness."""
def test_stop_flow_matches_base(self):
request_id = None
def respond(process, frame):
nonlocal request_id
fields = frame.split()
if fields[0] == b"SUBMIT":
request_id = fields[1]
process.stdout.feed(b"DATA " + request_id + b" 1\nx\n")
elif fields[0] == b"STOP":
process.stdout.feed(b"DONE " + request_id + b" STAT 1 1 0 1 2 0\n")
process = FakeProcess(respond)
with patch("openai_server.ARCH", "glm"), \
patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
output = []
engine.generate("hello", 8, 0.7, 0.9, output.append,
stopped=lambda: output == ["x"])
engine.close()
self.assertEqual(process.writes, [b"SUBMIT 1 0 5 8 0.7 0.9\nhello\n", b"STOP 1\n"])
def test_cancel_flow_matches_base(self):
request_id = None
def respond(process, frame):
nonlocal request_id
fields = frame.split()
if fields[0] == b"SUBMIT":
request_id = fields[1]
process.stdout.feed(b"DATA " + request_id + b" 1\nx\n")
elif fields[0] == b"CANCEL":
process.stdout.feed(b"ERROR " + request_id + b" CANCELLED\n")
process = FakeProcess(respond)
with patch("openai_server.ARCH", "glm"), \
patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
disconnected = False
def sink(text):
nonlocal disconnected
disconnected = True
with self.assertRaises(ClientCancelled):
engine.generate("hello", 8, 0.7, 0.9, sink, cancelled=lambda: disconnected)
engine.close()
self.assertEqual(process.writes, [b"SUBMIT 1 0 5 8 0.7 0.9\nhello\n", b"CANCEL 1\n"])
def test_image_and_submit_frames_match_base_under_one_lock_acquisition(self):
request_id = None
def respond(process, frame):
nonlocal request_id
fields = frame.split()
if fields[0] == b"SUBMIT":
request_id = fields[1]
process.stdout.feed(b"DATA " + request_id + b" 1\nx\n")
process.stdout.feed(b"DONE " + request_id + b" STAT 1 1 0 1 2 0\n")
process = FakeProcess(respond)
with patch("openai_server.ARCH", "glm53"), \
patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm53", "model")
counting_lock = _CountingLock()
engine.write_lock = counting_lock
blob = bytes(range(12))
class FakePatches:
def tobytes(self):
return blob
output = []
engine.generate("hello", 8, 0.7, 0.9, output.append, image=(FakePatches(), 2, 2))
engine.close()
self.assertEqual(process.writes, [
b"IMAGE 1 12 2 2\n" + blob + b"\n",
b"SUBMIT 1 0 5 8 0.7 0.9\nhello\n",
])
# IMAGE must reach the wire before SUBMIT, and both under the SAME
# lock acquisition -- another request's IMAGE could otherwise land
# between this one's IMAGE and its SUBMIT.
self.assertEqual(counting_lock.acquisitions, 1)
class LogprobsOptionsTest(unittest.TestCase):
"""logprobs_options(): pure validation and translation, no HTTP and no engine."""
def test_completions_valid_integer_logprobs(self):
self.assertEqual(logprobs_options({"logprobs": 3}, False, True), (3, False, 3))
self.assertEqual(logprobs_options({"logprobs": 3, "echo": True}, False, True),
(3, True, 3))
self.assertEqual(logprobs_options({"logprobs": 1}, False, True), (1, False, 1))
def test_completions_zero_false_null_mean_no_logprobs(self):
# Explicit, never a truthiness accident and never a channel floored on at k=1.
for off in ({"logprobs": 0}, {"logprobs": False}, {"logprobs": None}, {}):
with self.subTest(off=off):
self.assertEqual(logprobs_options(off, False, True), (0, False, 0))
# ...and `echo: true` on top of an off `logprobs` is a named 400, not a no-op.
with self.assertRaises(APIError) as caught:
logprobs_options({"logprobs": 0, "echo": True}, False, True)
self.assertEqual((caught.exception.param, caught.exception.code),
("echo", "unsupported_parameter"))
def test_completions_true_is_a_named_400(self):
# The legacy completions field is an integer count; `true` carries no count.
with self.assertRaises(APIError) as caught:
logprobs_options({"logprobs": True}, False, True)
self.assertEqual((caught.exception.status, caught.exception.param,
caught.exception.code), (400, "logprobs", "invalid_value"))
def test_completions_break_it_battery_non_integer_negative_huge(self):
for bad in (1.5, "5", -1, 33):
with self.subTest(bad=bad):
with self.assertRaises(APIError) as caught:
logprobs_options({"logprobs": bad}, False, True)
self.assertEqual((caught.exception.status, caught.exception.param,
caught.exception.code),
(400, "logprobs", "invalid_value"))
def test_chat_logprobs_requires_a_boolean(self):
with self.assertRaises(APIError) as caught:
logprobs_options({"logprobs": 1}, True, True)
self.assertEqual((caught.exception.param, caught.exception.code),
("logprobs", "invalid_value"))
def test_chat_false_and_null_mean_no_logprobs(self):
for off in ({"logprobs": False}, {"logprobs": None}, {}):
with self.subTest(off=off):
self.assertEqual(logprobs_options(off, True, True), (0, False, 0))
def test_chat_echo_is_always_rejected(self):
# Chat has no echo concept at all: a named 400, not a silent ignore, whether or
# not logprobs was also requested.
for body in ({"echo": True}, {"echo": True, "logprobs": True}):
with self.subTest(body=body):
with self.assertRaises(APIError) as caught:
logprobs_options(body, True, True)
self.assertEqual(caught.exception.param, "echo")
def test_chat_top_logprobs_default_and_cap(self):
self.assertEqual(logprobs_options({"logprobs": True}, True, True), (1, False, 0))
self.assertEqual(
logprobs_options({"logprobs": True, "top_logprobs": 5}, True, True),
(5, False, 5))
with self.assertRaises(APIError) as caught:
logprobs_options({"logprobs": True, "top_logprobs": 33}, True, True)
self.assertEqual(caught.exception.param, "top_logprobs")
def test_cap_boundary_is_thirty_two_on_both_endpoints(self):
# Literal 32/33 rather than the constant plus or minus one, so the boundary stays
# meaningful if the constant itself ever drifts.
self.assertEqual(logprobs_options({"logprobs": 32}, False, True), (32, False, 32))
self.assertEqual(
logprobs_options({"logprobs": True, "top_logprobs": 32}, True, True),
(32, False, 32))
for body, chat, param in (({"logprobs": 33}, False, "logprobs"),
({"logprobs": True, "top_logprobs": 33}, True,
"top_logprobs")):
with self.subTest(param=param):
with self.assertRaises(APIError) as caught:
logprobs_options(body, chat, True)
self.assertEqual((caught.exception.param, caught.exception.code),
(param, "invalid_value"))
self.assertEqual(LOGPROBS_TOP_K_CAP, 32)
def test_capability_gate_rejects_an_engine_that_is_not_asked_for_the_channel(self):
# Never a silent downgrade to "no logprobs".
for body, chat in (({"logprobs": 1}, False), ({"logprobs": True}, True)):
with self.subTest(chat=chat):
with self.assertRaises(APIError) as caught:
logprobs_options(body, chat, False)
self.assertEqual((caught.exception.status, caught.exception.code),
(400, "unsupported_parameter"))
# Absent or zero logprobs never reaches the capability check at all.
self.assertEqual(logprobs_options({}, False, False), (0, False, 0))
self.assertEqual(logprobs_options({"logprobs": 0}, False, False), (0, False, 0))
def test_range_error_precedes_capability_error_and_names_the_cap(self):
with self.assertRaises(APIError) as caught:
logprobs_options({"logprobs": 999}, False, False)
self.assertEqual(caught.exception.code, "invalid_value")
self.assertIn("32", caught.exception.message)
def test_chat_top_logprobs_validated_even_when_logprobs_is_off(self):
# A malformed `top_logprobs` is a named 400 whether or not the gate that would use
# it is open; a valid one with `logprobs` off stays a documented no-op.
for bad in ("x", -1, 999):
with self.subTest(bad=bad):
with self.assertRaises(APIError) as caught:
logprobs_options({"top_logprobs": bad}, True, True)
self.assertEqual((caught.exception.status, caught.exception.param,
caught.exception.code),
(400, "top_logprobs", "invalid_value"))
self.assertEqual(logprobs_options({"top_logprobs": 5}, True, True), (0, False, 0))
def test_chat_top_logprobs_null_normalizes_to_absent(self):
# On both sides of the `logprobs` gate, and it never itself reaches the
# type/range check.
self.assertEqual(
logprobs_options({"logprobs": False, "top_logprobs": None}, True, True),
(0, False, 0))
self.assertEqual(
logprobs_options({"logprobs": True, "top_logprobs": None}, True, True),
(1, False, 0))
def test_echo_null_normalizes_to_absent_both_endpoints(self):
# `echo: null` normalises to absent before the type check, exactly like `logprobs`
# and `top_logprobs`. An SDK that serialises its whole request model with nulls
# must not get a 400 for a field it never meant to set.
self.assertEqual(logprobs_options({"echo": None}, False, True), (0, False, 0))
self.assertEqual(logprobs_options({"echo": None}, True, True), (0, False, 0))
def test_echo_non_bool_rejected_the_same_way_on_both_endpoints(self):
for bad in (1, "true"):
for chat in (False, True):
with self.subTest(bad=bad, chat=chat):
with self.assertRaises(APIError) as caught:
logprobs_options({"echo": bad}, chat, True)
self.assertEqual((caught.exception.param, caught.exception.code),
("echo", "invalid_value"))
def test_echo_refusals_stay_distinct(self):
# The type refusal and chat's capability refusal must not collapse into each
# other: a non-bool `echo` always gets the type message, and only an actual `True`
# on chat draws the unsupported one.
with self.assertRaises(APIError) as caught:
logprobs_options({"echo": "yes"}, True, True)
self.assertEqual(caught.exception.code, "invalid_value")
self.assertIn("must be a boolean", caught.exception.message)
with self.assertRaises(APIError) as caught:
logprobs_options({"echo": True}, True, True)
self.assertEqual(caught.exception.code, "unsupported_parameter")
self.assertIn("not supported for chat completions", caught.exception.message)
def test_completions_ignores_top_logprobs_entirely(self):
# A chat-only field in the OpenAI request shape; completions never reads it.
self.assertEqual(
logprobs_options({"logprobs": 2, "top_logprobs": 999}, False, True),
(2, False, 2))
class LogprobsHTTPTest(unittest.TestCase):
"""End-to-end behaviour of `logprobs`/`echo`/`top_logprobs` against a real APIServer,
with FakeEngine standing in for the engine subprocess."""
@classmethod
def setUpClass(cls):
cls.engine = FakeEngine()
cls.server = APIServer(("127.0.0.1", 0), cls.engine, "test-model", "secret", 16,
kv_slots=2)
cls.thread = threading.Thread(target=cls.server.serve_forever, args=(0.01,),
daemon=True)
cls.thread.start()
cls.base = f"http://127.0.0.1:{cls.server.server_port}"
@classmethod
def tearDownClass(cls):
cls.server.scheduler.close()
cls.server.shutdown()
cls.server.server_close()
cls.thread.join(timeout=2)
def chat(self, body):
return _post_chat(self.base, dict({"model": "test-model",
"messages": [{"role": "user", "content": "Hi"}]},
**body))
def completions(self, body):
return _post_completions(self.base, dict({"model": "test-model", "prompt": "Hi"},
**body))
# ---- shape -------------------------------------------------------------
def test_chat_logprobs_content_shape(self):
with self.chat({"logprobs": True, "top_logprobs": 2}) as response:
body = json.load(response)
choice = body["choices"][0]
content = choice["logprobs"]["content"]
self.assertEqual([entry["token"] for entry in content], ["H", "é", "llo"])
self.assertIsNone(choice["logprobs"]["refusal"])
for entry in content:
self.assertEqual(set(entry), {"token", "logprob", "bytes", "top_logprobs"})
for alternative in entry["top_logprobs"]:
self.assertEqual(set(alternative), {"token", "logprob", "bytes"})
self.assertNotIn("echo", choice)
def test_chat_content_joins_back_to_the_message(self):
with self.chat({"logprobs": True, "top_logprobs": 1}) as response:
choice = json.load(response)["choices"][0]
self.assertEqual("".join(entry["token"] for entry in choice["logprobs"]["content"]),
choice["message"]["content"])
def test_chat_top_logprobs_omitted_yields_empty_alternatives(self):
with self.chat({"logprobs": True}) as response:
content = json.load(response)["choices"][0]["logprobs"]["content"]
self.assertTrue(content)
for entry in content:
self.assertEqual(entry["top_logprobs"], [])
def test_the_chosen_tokens_logprob_comes_from_its_own_record(self):
# The label is found by exact float match against the position's own logprob, and
# the table is unsorted on the wire, so this can never be "whatever came first".
with self.chat({"logprobs": True, "top_logprobs": 2}) as response:
content = json.load(response)["choices"][0]["logprobs"]["content"]
# Against the fixture's declared per-record values, not against the alternative
# the entry itself labelled -- that comparison holds however the value was sourced.
self.assertEqual([entry["logprob"] for entry in content], [-0.2, -0.4, -0.6])
for entry in content:
own = [a for a in entry["top_logprobs"] if a["token"] == entry["token"]]
self.assertEqual(len(own), 1)
self.assertEqual(own[0]["logprob"], entry["logprob"])
self.assertEqual(own[0]["bytes"], entry["bytes"])
def test_the_chosen_token_is_found_by_value_not_by_table_rank(self):
# The table is unsorted on the wire, so the position's own token can sit anywhere
# in it. Labelling by rank would call the first entry the chosen token.
base = _spawn_test_server(self, ScriptedEngine(
chunks=("cat",), records=[(b"cat", -0.8, [(999, -0.05), (42, -0.8)])]))
with _post_chat(base, {"model": "test-model", "logprobs": True, "top_logprobs": 2,
"messages": [{"role": "user", "content": "hi"}]}) as response:
entry = json.load(response)["choices"][0]["logprobs"]["content"][0]
self.assertEqual([alternative["token"] for alternative in entry["top_logprobs"]],
["<token_id:999>", "cat"])
self.assertEqual(entry["logprob"], -0.8)
def test_a_tie_on_the_printed_value_is_broken_deterministically(self):
# The engine prints six decimal digits, so two candidates can share the chosen
# token's printed value. Only the first match in wire order is labelled as the
# chosen token; a later one is labelled by its id like any other candidate, which
# is also what keeps the two labels distinct.
base = _spawn_test_server(self, ScriptedEngine(
chunks=("cat",),
records=[(b"cat", -0.223144, [(1, -0.223144), (2, -0.223144)])]))
with _post_chat(base, {"model": "test-model", "logprobs": True, "top_logprobs": 2,
"messages": [{"role": "user", "content": "hi"}]}) as response:
entry = json.load(response)["choices"][0]["logprobs"]["content"][0]
self.assertEqual([alternative["token"] for alternative in entry["top_logprobs"]],
["cat", "<token_id:2>"])
def test_legacy_token_logprobs_come_from_the_records_own_value(self):
# The completions twin of the chat test above: the table is unsorted on the wire,
# so `token_logprobs` must be the record's own `lp`, never the first table entry.
base = _spawn_test_server(self, ScriptedEngine(
chunks=("cat",), records=[(b"cat", -0.8, [(999, -0.05), (42, -0.8)])]))
with _post_completions(base, {"model": "test-model", "prompt": "hi",
"logprobs": 2, "max_tokens": 1}) as response:
logprobs = json.load(response)["choices"][0]["logprobs"]
self.assertEqual(logprobs["tokens"], ["cat"])
self.assertEqual(logprobs["token_logprobs"], [-0.8])
self.assertEqual(logprobs["top_logprobs"], [{"<token_id:999>": -0.05, "cat": -0.8}])
def test_completions_logprobs_object_shape_and_text_offsets(self):
with self.completions({"logprobs": 2}) as response:
choice = json.load(response)["choices"][0]
logprobs = choice["logprobs"]
self.assertEqual(set(logprobs),
{"tokens", "token_logprobs", "top_logprobs", "text_offset"})
self.assertEqual(logprobs["tokens"], ["H", "é", "llo"])
# Characters into `text` from 0, not bytes: "é" is one character and two bytes.
self.assertEqual(logprobs["text_offset"], [0, 1, 2])
self.assertEqual("".join(logprobs["tokens"]), choice["text"])
def test_echo_reconstructs_the_prompt_and_counts_offsets_from_zero(self):
with self.completions({"logprobs": 2, "echo": True}) as response:
choice = json.load(response)["choices"][0]
logprobs = choice["logprobs"]
self.assertEqual(logprobs["tokens"], ["H", "é", "H", "é", "llo"])
self.assertEqual(logprobs["text_offset"], [0, 1, 2, 3, 4])
self.assertEqual("".join(logprobs["tokens"]), choice["text"])
# The engine's own echo position 0 carries a non-finite sentinel: there is nothing
# to condition the first prompt token on.
self.assertIsNone(logprobs["token_logprobs"][0])
def test_nan_logprob_serializes_as_json_null_over_the_wire(self):
# Not the invalid-JSON NaN literal, and not clamped to a made-up finite number.
with self.completions({"logprobs": 1, "echo": True}) as response:
raw = response.read().decode()
self.assertNotIn("NaN", raw)
self.assertIsNone(json.loads(raw)["choices"][0]["logprobs"]["token_logprobs"][0])
# ---- normalisation and refusals ----------------------------------------
def test_zero_false_null_normalize_to_absent(self):
# Each endpoint's own shape: `0` is a completions value, `false`/`null` are
# accepted on both, and none of them is ever an error.
for field, value in (("logprobs", 0), ("logprobs", False), ("logprobs", None),
("echo", False), ("echo", None)):
with self.subTest(endpoint="completions", field=field, value=value):
with self.completions({field: value}) as response:
choice = json.load(response)["choices"][0]
self.assertIsNone(choice["logprobs"])
self.assertEqual(choice["text"], "Héllo")
for field, value in (("logprobs", False), ("logprobs", None), ("echo", None),
("top_logprobs", None), ("top_logprobs", 0)):
with self.subTest(endpoint="chat", field=field, value=value):
with self.chat({field: value}) as response:
choice = json.load(response)["choices"][0]
self.assertIsNone(choice["logprobs"])
def test_echo_without_logprobs_is_refused_by_name(self):
# The prompt echo is built out of the engine's per-token records, so without them
# there is nothing to echo. A 200 that quietly drops a requested field is the shape
# this surface exists to stop.
for body in ({"echo": True}, {"echo": True, "logprobs": 0},
{"echo": True, "logprobs": None}):
with self.subTest(body=body):
status, error = _error_body(self, lambda: self.completions(body))
self.assertEqual(status, 400)
self.assertEqual(error["param"], "echo")
self.assertEqual(error["code"], "unsupported_parameter")
def test_echo_false_and_null_stay_absent_without_logprobs(self):
# The control: only `true` needs a channel, and the two absent spellings must keep
# serving the request they always served.
for value in (False, None):
with self.subTest(value=value):
with self.completions({"echo": value}) as response:
choice = json.load(response)["choices"][0]
self.assertIsNone(choice["logprobs"])
self.assertEqual(choice["text"], "Héllo")
def test_each_endpoint_refuses_the_others_request_shape_by_name(self):
for post, body, param in ((self.completions, {"logprobs": True}, "logprobs"),
(self.chat, {"logprobs": 2}, "logprobs"),
(self.chat, {"echo": True}, "echo")):
with self.subTest(body=body):
status, error = _error_body(self, lambda: post(body))
self.assertEqual(status, 400)
self.assertEqual(error["param"], param)
def test_break_it_logprobs_out_of_range(self):
status, error = _error_body(self, lambda: self.completions({"logprobs": 99}))
self.assertEqual(status, 400)
self.assertEqual(error["code"], "invalid_value")
def test_break_it_streaming_plus_logprobs_is_named_400(self):
for post, body in ((self.completions, {"logprobs": 1, "stream": True}),
(self.chat, {"logprobs": True, "stream": True})):
with self.subTest(body=body):
status, error = _error_body(self, lambda: post(body))
self.assertEqual(status, 400)
self.assertEqual(error["param"], "logprobs")
self.assertEqual(error["code"], "unsupported_parameter")
def test_chat_echo_false_normalises_to_absent_over_http(self):
# The completions side of this is covered; chat is the half that was only true by
# construction. `false` is not `true`, so it never reaches chat's echo refusal:
# the request is served exactly as one that never named the field.
with self.chat({"echo": False}) as response:
choice = json.load(response)["choices"][0]
self.assertIsNone(choice["logprobs"])
self.assertEqual(choice["message"]["content"], "Héllo")
with self.chat({"echo": False, "logprobs": True, "top_logprobs": 1}) as response:
choice = json.load(response)["choices"][0]
self.assertEqual([entry["token"] for entry in choice["logprobs"]["content"]],
["H", "é", "llo"])
def test_a_continued_turn_is_prompt_and_the_arrays_describe_only_what_was_generated(self):
"""A trailing assistant turn is prefill, not output: the prompt ends inside it. The
text the client sent must therefore reach the engine as prompt and must not appear
in the arrays, which describe the generated text alone.
`enable_thinking` is on because that is the axis where the two features meet. A
continued turn opens no reasoning block, so the split starts in text mode -- and the
span map the arrays are projected through comes from that same split. Primed the
other way the split files the whole answer as reasoning, which leaves `content`
empty and the arrays describing a string that is no longer there."""
opening = "The capital of France is Par"
with self.chat({"logprobs": True, "top_logprobs": 2, "enable_thinking": True,
"messages": [{"role": "user", "content": "Where is it?"},
{"role": "assistant", "content": opening}]}) as response:
choice = json.load(response)["choices"][0]
self.assertTrue(self.engine.calls[-1][0].endswith(opening),
self.engine.calls[-1][0][-60:])
self.assertEqual(choice["message"]["content"], "Héllo")
self.assertNotIn("reasoning_content", choice["message"])
tokens = [entry["token"] for entry in choice["logprobs"]["content"]]
self.assertEqual(tokens, ["H", "é", "llo"])
self.assertEqual("".join(tokens), choice["message"]["content"])
def test_a_continued_turn_does_not_give_chat_an_echo(self):
# `echo` is the legacy completions shape and chat refuses it by name. A continued
# turn puts client text in the prompt, which is the one thing that could look like
# a reason to answer one here; it is not, and the refusal is unchanged.
status, error = _error_body(self, lambda: _post_chat(self.base, {
"model": "test-model", "logprobs": True, "echo": True,
"messages": [{"role": "user", "content": "Where is it?"},
{"role": "assistant", "content": "The capital of France is Par"}]}))
self.assertEqual(status, 400)
self.assertEqual(error["param"], "echo")
def test_the_capability_refusal_names_the_gate_not_the_engines_arch(self):
# The refusal reads the flag it failed, not the module's ARCH: in production the
# two agree, so naming the arch would be a guess that happens to be right, and
# under the test module's own ARCH ("glm") it would read "not supported by the glm
# engine" for an engine whose flag is off -- which is the wrong sentence.
base = _spawn_test_server(self, ScriptedEngine(supports_logprobs_echo=False))
status, error = _error_body(self, lambda: _post_completions(base, {
"model": "test-model", "prompt": "hi", "logprobs": 1}))
self.assertEqual(status, 400)
self.assertEqual(
error["message"],
"Log probabilities are not requested from this engine by these endpoints.")
self.assertNotIn(openai_server.ARCH, error["message"])
self.assertNotIn("glm", error["message"])
def test_break_it_logprobs_rejected_for_non_glm_engine(self):
# A named 400, never a silent no-op, on an engine these endpoints do not request
# the channel from.
base = _spawn_test_server(self, ScriptedEngine(supports_logprobs_echo=False))
for path, body in (("/v1/completions", {"model": "test-model", "prompt": "hi",
"logprobs": 1}),
("/v1/chat/completions",
{"model": "test-model", "logprobs": True,
"messages": [{"role": "user", "content": "hi"}]})):
with self.subTest(path=path):
status, error = _error_body(self, lambda: _post_json(base, path, body))
self.assertEqual(status, 400)
self.assertEqual(error["param"], "logprobs")
self.assertEqual(error["code"], "unsupported_parameter")
def test_a_plain_request_never_asks_the_engine_for_the_channel(self):
with self.completions({}) as response:
choice = json.load(response)["choices"][0]
self.assertIsNone(choice["logprobs"])
self.assertEqual(self.engine.last_logprobs, 0)
self.assertFalse(self.engine.last_echo)
self.assertFalse(self.engine.last_gbytes)
def test_the_echo_sink_is_passed_only_when_echo_was_asked_for(self):
# The engine sends an ECHO frame for every prompt position of every opted-in
# request, so passing a sink is the retention opt-in.
with self.completions({"logprobs": 1}) as response:
response.read()
self.assertEqual(self.engine.last_logprobs, 1)
self.assertFalse(self.engine.last_echo)
with self.completions({"logprobs": 1, "echo": True}) as response:
response.read()
self.assertTrue(self.engine.last_echo)
class LogprobsRawStreamAlignmentTest(unittest.TestCase):
"""The arrays describe the characters the client received, exactly. Every expectation
here is written from the engine's own frames and the stop policy, and each test names
inline what a build without the span layer returns for it."""
def _chat(self, base, **extra):
body = {"model": "test-model", "max_tokens": 8, "logprobs": True,
"top_logprobs": 1, "messages": [{"role": "user", "content": "hi"}]}
body.update(extra)
with _post_chat(base, body) as response:
return json.load(response)["choices"][0]
def test_swallowed_leading_marker_keeps_the_other_two_records(self):
# WITHOUT THE SPAN LAYER: logprobs.content == [] with a 200 and a full message.content. Two of
# three, not three: the role marker was genuinely not emitted, so an entry for it
# would describe a character the client never received.
choice = self._chat(_spawn_test_server(self, _ignored_leading_marker_engine()))
self.assertEqual(choice["message"]["content"], "Hello world")
entries = choice["logprobs"]["content"]
self.assertEqual([entry["token"] for entry in entries], ["Hello", " world"])
self.assertEqual([entry["logprob"] for entry in entries],
[-0.2, -0.30000000000000004])
self.assertEqual("".join(entry["token"] for entry in entries),
choice["message"]["content"])
def test_swallowed_leading_marker_offsets_index_the_returned_text(self):
# The legacy shape's arm of the same defect: "Hello" at 0 and " world" at 5, both
# indexing "Hello world".
base = _spawn_test_server(self, _ignored_leading_marker_engine())
with _post_completions(base, {"model": "test-model", "prompt": "hi",
"max_tokens": 8, "logprobs": 1,
"stop": ["<|user|>"],
"x_colibri_ignore_leading_stop": True}) as response:
choice = json.load(response)["choices"][0]
self.assertEqual(choice["text"], "Hello world")
self.assertEqual(choice["logprobs"]["tokens"], ["Hello", " world"])
self.assertEqual(choice["logprobs"]["text_offset"], [0, 5])
self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])
def test_a_record_the_stop_cut_in_half_keeps_the_emitted_characters(self):
# WITHOUT THE SPAN LAYER: text "Hel" with tokens []. The boundary entry carries the emitted
# span, so `tokens` is ["Hel"] -- not ["Hello"], and not [].
base = _spawn_test_server(self, _mid_token_stop_echo_engine())
with _post_completions(base, {"model": "test-model", "prompt": "prompt",
"max_tokens": 4, "logprobs": 1,
"stop": ["lo"]}) as response:
choice = json.load(response)["choices"][0]
self.assertEqual(choice["text"], "Hel")
self.assertEqual(choice["logprobs"]["tokens"], ["Hel"])
self.assertEqual(choice["logprobs"]["text_offset"], [0])
self.assertEqual(choice["logprobs"]["token_logprobs"], [-0.5])
self.assertEqual(choice["finish_reason"], "stop")
self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])
def test_echo_keeps_the_prompt_and_the_emitted_prefix(self):
# WITHOUT THE SPAN LAYER: text collapses to "prompt" -- the "Hel" the client earned is deleted
# by the echo rebuild.
base = _spawn_test_server(self, _mid_token_stop_echo_engine())
with _post_completions(base, {"model": "test-model", "prompt": "prompt",
"max_tokens": 4, "logprobs": 1, "echo": True,
"stop": ["lo"]}) as response:
choice = json.load(response)["choices"][0]
self.assertEqual(choice["text"], "promptHel")
self.assertEqual(choice["logprobs"]["tokens"], ["pr", "ompt", "Hel"])
self.assertEqual(choice["logprobs"]["text_offset"], [0, 2, 6])
self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])
def test_a_truncated_entrys_bytes_do_not_describe_the_withheld_text(self):
# `bytes` is `token`'s sibling and is held to the same rule: the record's payload
# decodes to "Hello", so leaving it whole would hand the client the "lo" the stop
# withheld, one field over.
choice = self._chat(_spawn_test_server(self, _mid_token_stop_echo_engine()),
max_tokens=4, stop=["lo"])
self.assertEqual(choice["message"]["content"], "Hel")
entry = choice["logprobs"]["content"][0]
self.assertEqual(entry["token"], "Hel")
self.assertEqual(entry["bytes"], [72, 101, 108])
self.assertEqual(bytes(entry["bytes"]).decode("utf-8"), entry["token"])
for alternative in entry["top_logprobs"]:
if alternative["bytes"] is not None:
self.assertEqual(bytes(alternative["bytes"]).decode("utf-8"),
alternative["token"])
def test_a_matched_stop_drops_its_own_record_and_every_later_one(self):
base = _spawn_test_server(self, _stop_token_engine())
with _post_completions(base, {"model": "test-model", "prompt": "hi",
"max_tokens": 8, "stop": ["STOP"],
"logprobs": 1}) as response:
choice = json.load(response)["choices"][0]
self.assertEqual(choice["text"], "ok ")
self.assertEqual(choice["logprobs"]["tokens"], ["ok "])
self.assertEqual(len(choice["logprobs"]["token_logprobs"]), 1)
self.assertEqual(len(choice["logprobs"]["top_logprobs"]), 1)
def test_a_frame_inside_suppressed_text_leaves_no_phantom_entry(self):
# A frame that decodes to "" owns no span, so "was it emitted?" has to be asked of
# the character its bytes went into, not of its position, which sits on the
# boundary of the emitted region and tests true at both ends. A phantom entry here
# carries a byte of the suppressed stop sequence out in the one field that is
# bytes.
base = _spawn_test_server(self, _suppressed_stop_byte_engine())
with _post_completions(base, {"model": "test-model", "prompt": "hi",
"max_tokens": 8, "logprobs": 1,
"stop": ["é"]}) as response:
choice = json.load(response)["choices"][0]
self.assertEqual(choice["text"], "ok ")
self.assertEqual(choice["logprobs"]["tokens"], ["ok "])
self.assertEqual(choice["logprobs"]["text_offset"], [0])
self.assertEqual(choice["logprobs"]["token_logprobs"], [-0.1])
def test_a_frame_that_completes_nothing_at_the_seam_reports_nothing(self):
# Both rows of the two-frame seam case. Frame one's bytes finish no character, so
# its token is "": reporting the raw stream's replacement character instead puts a
# U+FFFD in front of the real one and shifts every later offset.
for stop, text, tokens, offsets in (
(None, "A€Hello", ["A", "", "", "€Hello"], [0, 1, 1, 1]),
(["lo"], "A€Hel", ["A", "", "", "€Hel"], [0, 1, 1, 1])):
with self.subTest(stop=stop):
base = _spawn_test_server(self, _seam_split_across_two_frames_engine())
body = {"model": "test-model", "prompt": "A€", "max_tokens": 8,
"logprobs": 1, "echo": True}
if stop is not None:
body["stop"] = stop
with _post_completions(base, body) as response:
choice = json.load(response)["choices"][0]
self.assertEqual(choice["text"], text)
self.assertNotIn("�", choice["text"])
self.assertEqual(choice["logprobs"]["tokens"], tokens)
self.assertEqual(choice["logprobs"]["text_offset"], offsets)
self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])
def test_seam_codepoint_and_stop_on_the_same_record(self):
# The raw-stream decode opens with two replacement characters where the seam
# decode has one Euro sign; the stop cuts past that head, so the real character
# splices back on and the withheld "lo" stays withheld.
base = _spawn_test_server(self, _seam_split_same_record_stop_engine())
with _post_completions(base, {"model": "test-model", "prompt": "A€",
"max_tokens": 4, "logprobs": 1, "echo": True,
"stop": ["lo"]}) as response:
choice = json.load(response)["choices"][0]
self.assertEqual(choice["text"], "A€Hel")
self.assertNotIn("�", choice["text"])
self.assertEqual(choice["logprobs"]["tokens"], ["A", "", "€Hel"])
self.assertEqual(choice["logprobs"]["text_offset"], [0, 1, 1])
self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])
def test_seam_codepoint_survives_a_later_stop_truncation(self):
# The Euro sign is split across the prompt/generated seam and must decode as one
# character while a later frame is cut by the stop. Four positions for four
# frames: the echo position carrying the lead byte resolves nothing on its own and
# reports "", and the character lands on the frame that completed it.
base = _spawn_test_server(self, _seam_split_stop_engine())
with _post_completions(base, {"model": "test-model", "prompt": "A€",
"max_tokens": 4, "logprobs": 1, "echo": True,
"stop": ["STOP"]}) as response:
choice = json.load(response)["choices"][0]
self.assertEqual(choice["text"], "A€x")
self.assertEqual(choice["text"].count("€"), 1)
self.assertNotIn("�", choice["text"])
self.assertEqual(choice["logprobs"]["tokens"], ["A", "", "€", "x"])
self.assertEqual(choice["logprobs"]["text_offset"], [0, 1, 1, 2])
self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])
def test_an_engine_that_sends_no_records_is_refused_not_half_answered(self):
# The opt-in was accepted and the channel came back empty. Answering with the
# completion and an empty array is the shape the arrays exist to prevent.
base = _spawn_test_server(self, ScriptedEngine(
chunks=("AA", "BB"), records=None,
echoes=[(b"P", NAN, []), (b"Q", -0.1, [(1, -0.1)])]))
status, error = _error_body(self, lambda: _post_completions(base, {
"model": "test-model", "prompt": "PQ", "max_tokens": 4, "logprobs": 1,
"echo": True}))
self.assertEqual(status, 500)
self.assertEqual(error["code"], "engine_logprob_records_incomplete")
def test_a_length_limited_turn_reports_finish_reason_length(self):
base = _spawn_test_server(self, ScriptedEngine(chunks=("ok",),
length_limited=True))
with _post_completions(base, {"model": "test-model", "prompt": "hi",
"max_tokens": 1, "logprobs": 1}) as response:
choice = json.load(response)["choices"][0]
self.assertEqual(choice["finish_reason"], "length")
self.assertEqual(choice["logprobs"]["tokens"], ["ok"])
def test_every_response_joins_its_tokens_back_to_the_text_it_returned(self):
# The invariant as a test rather than a comment, over every fixture in this file
# that can produce a boundary.
for name, build in (("leading marker", _ignored_leading_marker_engine),
("mid-token stop", _mid_token_stop_echo_engine),
("stop token", _stop_token_engine),
("seam and stop", _seam_split_stop_engine)):
for echo in (False, True):
with self.subTest(engine=name, echo=echo):
base = _spawn_test_server(self, build())
with _post_completions(base, {
"model": "test-model", "prompt": "A€", "max_tokens": 8,
"logprobs": 1, "echo": echo,
"stop": ["lo", "STOP", "<|user|>"]}) as response:
choice = json.load(response)["choices"][0]
tokens = choice["logprobs"]["tokens"]
offsets = choice["logprobs"]["text_offset"]
self.assertEqual("".join(tokens), choice["text"])
self.assertEqual(offsets, sorted(offsets))
for offset in offsets:
self.assertLessEqual(offset, len(choice["text"]))
class PlainPathSpanWorkTest(unittest.TestCase):
"""What a request that never asked for per-token logprobs actually pays for the span
layer, stage by stage. Counted at a seam -- the primitives each stage would call --
rather than against a timing or a golden number, so the assertion is about work done.
The thinking split builds nothing. The tool-call stage composes and exports nothing,
which is the part that was worth gating; it still walks its own cut lists, because
those ARE the deletion the content is sliced from rather than a map kept for a later
reader. The exact residue is asserted below rather than described, so it cannot drift
upward unnoticed."""
def _counted(self, name):
"""Replace one span primitive with a counting pass-through for the duration of a
request, and return the list its calls are recorded in."""
calls = []
original = getattr(openai_server, name)
def counting(*args, **kwargs):
calls.append(1)
return original(*args, **kwargs)
patcher = patch.object(openai_server, name, counting)
patcher.start()
self.addCleanup(patcher.stop)
return calls
def _chat(self, engine, **extra):
base = _spawn_test_server(self, engine)
body = {"model": "test-model", "max_tokens": 16,
"messages": [{"role": "user", "content": "hi"}]}
body.update(extra)
with _post_chat(base, body) as response:
return json.load(response)["choices"][0]
THINKING = ScriptedEngine(
chunks=("<think>", "why", "</think>", "the ", "answer"),
records=[(b"<think>", -0.1, [(1, -0.1)]), (b"why", -0.2, [(2, -0.2)]),
(b"</think>", -0.3, [(3, -0.3)]), (b"the ", -0.4, [(4, -0.4)]),
(b"answer", -0.5, [(5, -0.5)])])
TOOLS = [{"type": "function",
"function": {"name": "search",
"parameters": {"type": "object",
"properties": {"q": {"type": "string"}}}}}]
def test_the_thinking_split_records_nothing_on_a_plain_request(self):
# No tools, so the thinking split is the only stage that could append a span.
appends = self._counted("_append_span")
choice = self._chat(self.THINKING, enable_thinking=True)
self.assertEqual(choice["message"]["content"], "the answer")
self.assertEqual(choice["message"]["reasoning_content"], "why")
self.assertEqual(sum(appends), 0)
def test_the_thinking_split_records_when_logprobs_are_asked_for(self):
# The control: the same request with the opt-in builds the map it needs, so the
# test above cannot be satisfied by a stage that never records at all.
appends = self._counted("_append_span")
choice = self._chat(self.THINKING, enable_thinking=True, logprobs=True,
top_logprobs=1)
self.assertEqual(choice["message"]["content"], "the answer")
self.assertGreater(sum(appends), 0)
def test_the_tool_call_parse_composes_and_exports_nothing_on_a_plain_request(self):
# The composition and the exported maps are what the gate removes. The per-step
# cut lists remain: `_apply_cuts` returns the survivors the content is built by
# slicing, so removing them would mean a second implementation of the deletion.
# Every count is asserted, including the ones that are not zero, so "no span map"
# can never quietly become "some span map".
engine = ScriptedEngine(
chunks=("before ", "<tool_call>search<arg_key>q</arg_key>"
"<arg_value>x</arg_value></tool_call>", " after"))
counts = {name: self._counted(name) for name in
("_compose_span_maps", "_append_span", "_cut_span_map", "_apply_cuts",
"_marker_cuts", "_project_span")}
choice = self._chat(engine, tools=self.TOOLS)
self.assertEqual(choice["message"]["content"], "before after")
self.assertEqual(len(choice["message"]["tool_calls"]), 1)
self.assertEqual({name: sum(calls) for name, calls in counts.items()},
{"_compose_span_maps": 0, # nothing composed
"_project_span": 0, # nothing read back
"_append_span": 5, # inside the cut lists below
"_cut_span_map": 4,
"_apply_cuts": 4,
"_marker_cuts": 2})
# ...and no map leaves the stage for anyone to read.
_content, _calls, box_map, content_map = openai_server._parse_tool_calls(
"before <tool_call>search</tool_call> after", self.TOOLS, False)
self.assertIsNone(box_map)
self.assertIsNone(content_map)
def test_the_tool_call_parse_composes_when_logprobs_are_asked_for(self):
engine = ScriptedEngine(
chunks=("before ", "<tool_call>search<arg_key>q</arg_key>"
"<arg_value>x</arg_value></tool_call>", " after"),
records=[(b"before ", -0.1, [(1, -0.1)]),
("<tool_call>search<arg_key>q</arg_key>"
"<arg_value>x</arg_value></tool_call>".encode(), -0.2, [(2, -0.2)]),
(b" after", -0.3, [(3, -0.3)])])
composes = self._counted("_compose_span_maps")
choice = self._chat(engine, tools=self.TOOLS, logprobs=True, top_logprobs=1)
self.assertEqual(choice["message"]["content"], "before after")
self.assertGreater(sum(composes), 0)
class LogprobsStageCompositionTest(unittest.TestCase):
"""The chat path runs the stop filter, then the thinking split, then the tool-call
parse. `logprobs.content` must describe `message.content` after all three."""
THINKING = ScriptedEngine(
chunks=("<think>", "why", "</think>", "the ", "answer"),
records=[(b"<think>", -0.1, [(1, -0.1)]),
(b"why", -0.2, [(2, -0.2)]),
(b"</think>", -0.3, [(3, -0.3)]),
(b"the ", -0.4, [(4, -0.4)]),
(b"answer", -0.5, [(5, -0.5)])])
def _chat(self, engine, **extra):
base = _spawn_test_server(self, engine)
body = {"model": "test-model", "max_tokens": 16, "logprobs": True,
"top_logprobs": 1, "messages": [{"role": "user", "content": "hi"}]}
body.update(extra)
with _post_chat(base, body) as response:
return json.load(response)["choices"][0]
def test_content_is_a_projection_of_the_stream_not_a_subset(self):
# Cardinality first, then absence: a build that returns an empty array satisfies
# every "X is not described" assertion trivially, so the count has to be asserted
# before anything about what is missing.
choice = self._chat(self.THINKING, enable_thinking=True)
entries = choice["logprobs"]["content"]
self.assertEqual(choice["message"]["content"], "the answer")
self.assertEqual(choice["message"]["reasoning_content"], "why")
self.assertEqual(len(entries), 2)
self.assertEqual([entry["token"] for entry in entries], ["the ", "answer"])
# Each entry's logprob is its OWN record's, not the one at the same index in the
# raw stream: a chain composed one stage short keeps the count and the text and
# still attaches the reasoning tokens' numbers.
self.assertEqual([entry["logprob"] for entry in entries], [-0.4, -0.5])
self.assertEqual("".join(entry["token"] for entry in entries),
choice["message"]["content"])
# ...and the reasoning and the markers are not described as content.
self.assertNotIn("why", [entry["token"] for entry in entries])
def test_tool_call_syntax_is_not_described_as_content(self):
engine = ScriptedEngine(
chunks=("before ", "<tool_call>search<arg_key>q</arg_key>"
"<arg_value>x</arg_value></tool_call>", " after"),
records=[(b"before ", -0.1, [(1, -0.1)]),
("<tool_call>search<arg_key>q</arg_key>"
"<arg_value>x</arg_value></tool_call>".encode(), -0.2, [(2, -0.2)]),
(b" after", -0.3, [(3, -0.3)])])
choice = self._chat(engine, tools=[{
"type": "function",
"function": {"name": "search",
"parameters": {"type": "object",
"properties": {"q": {"type": "string"}}}}}])
entries = choice["logprobs"]["content"]
self.assertEqual(choice["message"]["content"], "before after")
self.assertEqual(len(choice["message"]["tool_calls"]), 1)
self.assertEqual(len(entries), 2)
self.assertEqual([entry["token"] for entry in entries], ["before ", " after"])
self.assertEqual([entry["logprob"] for entry in entries], [-0.1, -0.3])
self.assertEqual("".join(entry["token"] for entry in entries),
choice["message"]["content"])
def test_all_three_stages_compose_in_one_turn(self):
# Thinking and tools together: the stop filter, the thinking split and the
# tool-call parse each cut something, and the chain has to be composed in that
# order. Composed the other way round, or stopped one stage early, the entries
# stop joining back to `message.content`.
engine = ScriptedEngine(
chunks=("<think>", "why", "</think>", "before ",
"<tool_call>search<arg_key>q</arg_key>"
"<arg_value>x</arg_value></tool_call>", " after"),
records=[(b"<think>", -0.1, [(1, -0.1)]),
(b"why", -0.2, [(2, -0.2)]),
(b"</think>", -0.3, [(3, -0.3)]),
(b"before ", -0.4, [(4, -0.4)]),
("<tool_call>search<arg_key>q</arg_key>"
"<arg_value>x</arg_value></tool_call>".encode(), -0.5, [(5, -0.5)]),
(b" after", -0.6, [(6, -0.6)])])
choice = self._chat(engine, enable_thinking=True, tools=[{
"type": "function",
"function": {"name": "search",
"parameters": {"type": "object",
"properties": {"q": {"type": "string"}}}}}])
entries = choice["logprobs"]["content"]
self.assertEqual(choice["message"]["reasoning_content"], "why")
self.assertEqual(choice["message"]["content"], "before after")
self.assertEqual(len(choice["message"]["tool_calls"]), 1)
self.assertEqual(len(entries), 2)
self.assertEqual([entry["token"] for entry in entries], ["before ", " after"])
self.assertEqual([entry["logprob"] for entry in entries], [-0.4, -0.6])
self.assertEqual("".join(entry["token"] for entry in entries),
choice["message"]["content"])
def test_mimo_tool_call_syntax_is_not_described_as_content(self):
# MiMo's call form, through the same three stages on a mimo gateway: the
# reasoning and the call block are cut, and the entries join back to the
# content the message returns.
call = ("<tool_call>\n<function=search>\n<parameter=q>x</parameter>\n"
"</function>\n</tool_call>")
engine = ScriptedEngine(
chunks=("why", "</think>", "before ", call, " after"),
records=[(b"why", -0.1, [(1, -0.1)]), (b"</think>", -0.2, [(2, -0.2)]),
(b"before ", -0.3, [(3, -0.3)]), (call.encode(), -0.4, [(4, -0.4)]),
(b" after", -0.5, [(5, -0.5)])])
with patch("openai_server.ARCH", "mimo"):
choice = self._chat(engine, enable_thinking=True, tools=[{
"type": "function",
"function": {"name": "search",
"parameters": {"type": "object",
"properties": {"q": {"type": "string"}}}}}])
entries = choice["logprobs"]["content"]
self.assertEqual(choice["message"]["reasoning_content"], "why")
self.assertEqual(choice["message"]["content"], "before after")
self.assertEqual(len(choice["message"]["tool_calls"]), 1)
self.assertEqual([entry["token"] for entry in entries], ["before ", " after"])
self.assertEqual([entry["logprob"] for entry in entries], [-0.3, -0.5])
self.assertEqual("".join(entry["token"] for entry in entries),
choice["message"]["content"])
def test_a_plain_chat_turn_describes_every_token(self):
# The control: with no stage cutting anything, nothing is dropped.
choice = self._chat(ScriptedEngine(chunks=("the ", "answer")))
entries = choice["logprobs"]["content"]
self.assertEqual(len(entries), 2)
self.assertEqual("".join(entry["token"] for entry in entries),
choice["message"]["content"])
class EchoCoverageTest(unittest.TestCase):
"""`echo: true` rebuilds `text` from the logprob records, because `text` and
`text_offset` have to describe the same reconstruction. If the records stop short of
the generated stream that rebuild silently drops the uncovered tail and the client gets
a truncated completion with a 200 -- the worst shape a fault here can take, because
nothing in the response says anything is missing. It is refused by name instead."""
BODY = {"model": "test-model", "prompt": "PQ", "echo": True,
"logprobs": 1, "max_tokens": 3}
def _post(self, engine, body):
return _post_completions(_spawn_test_server(self, engine), body)
def test_records_short_of_the_stream_are_refused_not_truncated(self):
# Unrefused, this is a 200 carrying "PQAA" with the client's "BB" and "CC" gone.
status, error = _error_body(
self, lambda: self._post(_partial_coverage_engine(covered=1), dict(self.BODY)))
self.assertEqual(status, 500)
self.assertEqual(error["code"], "engine_logprob_records_incomplete")
def test_full_coverage_is_served_normally(self):
# The control, and it is not optional: without it the check above is satisfied
# just as well by refusing every echo request.
with self._post(_partial_coverage_engine(covered=3), dict(self.BODY)) as response:
choice = json.load(response)["choices"][0]
self.assertEqual(choice["text"], "PQAABBCC")
self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])
self.assertEqual(choice["logprobs"]["text_offset"], [0, 1, 2, 4, 6])
def test_a_generation_ending_mid_codepoint_is_served(self):
# Control 1 of 2. A coverage check written as a string comparison reads an
# unflushed incremental decoder as a short generation and turns this valid
# response into a 500. Counting in raw-stream coordinates cannot.
engine = _partial_coverage_engine(
covered=3, chunks=("AA", "BB", "�"),
record_bytes=(b"AA", b"BB", b"\xc3"))
with self._post(engine, dict(self.BODY)) as response:
choice = json.load(response)["choices"][0]
self.assertEqual(choice["text"], "PQAABB�")
self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])
def test_a_codepoint_split_across_the_seam_is_served(self):
# Control 2 of 2. With `echo` on, the reconstruction decodes the generated half
# with the decoder that already consumed the prompt bytes, so a codepoint
# straddling the seam reads as one character there and as replacement characters
# in the raw stream, on purpose.
engine = _partial_coverage_engine(
covered=1, chunks=("��",), record_bytes=(b"\x82\xac",),
echo_bytes=(b"A", b"\xe2"))
with self._post(engine, dict(self.BODY)) as response:
choice = json.load(response)["choices"][0]
self.assertEqual(choice["text"], "A€")
self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])
def test_echo_false_is_refused_on_the_same_coverage_gap(self):
# The arrays have to describe the whole text on every shape that carries them, not
# only the one that rebuilds `text` out of them. Without this, `echo: false`
# returns "AABBCC" with `tokens == ["AA"]` and a 200.
status, error = _error_body(
self, lambda: self._post(_partial_coverage_engine(covered=1),
dict(self.BODY, echo=False)))
self.assertEqual(status, 500)
self.assertEqual(error["code"], "engine_logprob_records_incomplete")
self.assertEqual(error["param"], "logprobs")
def test_echo_false_full_coverage_is_served(self):
# The control for the refusal above: same engine, same shape, every record present.
with self._post(_partial_coverage_engine(covered=3),
dict(self.BODY, echo=False)) as response:
choice = json.load(response)["choices"][0]
self.assertEqual(choice["text"], "AABBCC")
self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])
def test_chat_is_refused_on_the_same_coverage_gap(self):
# The worst shape this surface can take, reached by the other door: a 200 with a
# full message.content and logprobs.content describing a prefix of it.
for covered, tokens in ((1, "short"), (0, "absent")):
with self.subTest(records=tokens):
base = _spawn_test_server(self, _partial_coverage_engine(covered=covered))
status, error = _error_body(self, lambda: _post_chat(base, {
"model": "test-model", "max_tokens": 3, "logprobs": True,
"top_logprobs": 1,
"messages": [{"role": "user", "content": "hi"}]}))
self.assertEqual(status, 500)
self.assertEqual(error["code"], "engine_logprob_records_incomplete")
self.assertEqual(error["param"], "logprobs")
def test_chat_full_coverage_is_served(self):
base = _spawn_test_server(self, _partial_coverage_engine(covered=3))
with _post_chat(base, {"model": "test-model", "max_tokens": 3, "logprobs": True,
"top_logprobs": 1,
"messages": [{"role": "user", "content": "hi"}]}) as response:
choice = json.load(response)["choices"][0]
entries = choice["logprobs"]["content"]
self.assertEqual("".join(entry["token"] for entry in entries),
choice["message"]["content"])
self.assertEqual(choice["message"]["content"], "AABBCC")
def test_echo_with_no_prompt_echo_records_is_refused_by_name(self):
# The engine answered half the opt-in: every generated record, no ECHO frame. The
# prompt echo is the half `echo` asked for.
base = _spawn_test_server(self, ScriptedEngine(chunks=("Hi",), echoes=()))
status, error = _error_body(self, lambda: _post_completions(base, {
"model": "test-model", "prompt": "Hi", "max_tokens": 2, "logprobs": 1,
"echo": True}))
self.assertEqual(status, 500)
self.assertEqual(error["code"], "engine_logprob_records_incomplete")
self.assertEqual(error["param"], "echo")
class PinnedPrefixEchoTest(unittest.TestCase):
"""A KV slot holding a pin snapshot echoes prompt positions from the snapshot's length
rather than from 0. Re-basing that run would attach every logprob to the wrong prompt
token, so it is refused by name."""
def test_a_pinned_prefix_is_refused_by_name(self):
base = _spawn_test_server(self, _pinned_prefix_engine())
status, error = _error_body(self, lambda: _post_completions(base, {
"model": "test-model", "prompt": "abc", "logprobs": 1, "echo": True,
"max_tokens": 2}))
self.assertEqual(status, 503)
self.assertEqual(error["code"], "engine_pinned_prefix_not_echoed")
self.assertEqual(error["param"], "echo")
def test_an_unpinned_prompt_control_is_served(self):
base = _spawn_test_server(self, ScriptedEngine(
chunks=("ok",),
echoes=[(b"a", NAN, []), (b"b", -0.1, [(1, -0.1)])]))
with _post_completions(base, {"model": "test-model", "prompt": "ab",
"logprobs": 1, "echo": True,
"max_tokens": 2}) as response:
choice = json.load(response)["choices"][0]
self.assertEqual(choice["text"], "abok")
class ClientVisibleEngineFaultCodeTest(unittest.TestCase):
"""Each engine-fault path fails closed AND tells the client which fault. Every test
reads the code and param out of the response body, because an exception with a good
name that never reaches the wire is the defect rather than the fix. Each has a control
that must come back 200, so "names the fault" can never be satisfied by a server that
refuses everything.
The codes say what the ENGINE did, never what this gateway did about it, so they
survive a change of reaction."""
def test_duplicate_candidate_names_itself_with_the_callers_own_param(self):
# The same engine table is requested by two different parameters, and a client
# branching on the code is told which of its own fields went unanswered.
for post, body, param in (
(_post_completions,
{"model": "test-model", "prompt": "hi", "logprobs": 2, "max_tokens": 1},
"logprobs"),
(_post_chat,
{"model": "test-model", "messages": [{"role": "user", "content": "hi"}],
"logprobs": True, "top_logprobs": 2, "max_tokens": 1},
"top_logprobs")):
with self.subTest(param=param):
base = _spawn_test_server(self, _duplicate_candidate_engine())
status, error = _error_body(self, lambda: post(base, body))
self.assertEqual(status, 500)
self.assertEqual(error["code"], "engine_duplicate_logprob_candidate")
self.assertEqual(error["param"], param)
def test_distinct_candidates_control_is_served_on_both_surfaces(self):
base = _spawn_test_server(self, _duplicate_candidate_engine(duplicate=False))
with _post_completions(base, {"model": "test-model", "prompt": "hi",
"logprobs": 2, "max_tokens": 1}) as response:
body = json.load(response)
self.assertEqual(len(body["choices"][0]["logprobs"]["top_logprobs"][0]), 2)
base = _spawn_test_server(self, _duplicate_candidate_engine(duplicate=False))
with _post_chat(base, {"model": "test-model",
"messages": [{"role": "user", "content": "hi"}],
"logprobs": True, "top_logprobs": 2,
"max_tokens": 1}) as response:
body = json.load(response)
self.assertEqual(
len(body["choices"][0]["logprobs"]["content"][0]["top_logprobs"]), 2)
def _tail_engine(self, frame):
"""A real Engine over a fake process, so the frame is parsed by the dispatcher
exactly as it is in production and the named error has to travel from the
dispatcher thread to the waiting request thread and out onto the wire."""
def respond(process, written):
request_id = written.split()[1]
process.stdout.feed(frame.replace(b"{id}", request_id))
process.stdout.feed(b"DONE " + request_id + b" STAT 1 1 0 1 1 0\n")
process = FakeProcess(respond)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
self.addCleanup(engine.close)
return engine
def test_malformed_tail_names_itself_to_the_client(self):
base = _spawn_test_server(
self, self._tail_engine(b"DATA {id} 1 -0.5 1 3\nx\n"))
status, error = _error_body(self, lambda: _post_completions(base, {
"model": "test-model", "prompt": "hi", "logprobs": 1, "max_tokens": 1}))
self.assertEqual(status, 500)
self.assertEqual(error["code"], "engine_logprob_tail_malformed")
self.assertEqual(error["param"], "logprobs")
def test_well_formed_tail_control_is_served(self):
# The control that matters most here: the relay must carry a good frame through
# untouched, or "names the fault" is satisfied by a build that fails every
# logprobs request.
base = _spawn_test_server(
self, self._tail_engine(b"DATA {id} 1 -0.5 1 3 -0.5\nx\n"))
with _post_completions(base, {"model": "test-model", "prompt": "hi",
"logprobs": 1, "max_tokens": 1}) as response:
body = json.load(response)
self.assertNotIn("error", body)
self.assertEqual(body["choices"][0]["text"], "x")
def test_an_unsupported_engine_names_its_refusal_in_the_body(self):
base = _spawn_test_server(self, ScriptedEngine(supports_logprobs_echo=False))
status, error = _error_body(self, lambda: _post_completions(base, {
"model": "test-model", "prompt": "hi", "logprobs": 1}))
self.assertEqual(status, 400)
self.assertEqual(error["code"], "unsupported_parameter")
self.assertEqual(error["param"], "logprobs")
def test_the_same_engine_serves_a_plain_request(self):
base = _spawn_test_server(self, ScriptedEngine(supports_logprobs_echo=False))
with _post_completions(base, {"model": "test-model", "prompt": "hi"}) as response:
body = json.load(response)
self.assertEqual(body["choices"][0]["text"], "ok")
self.assertIsNone(body["choices"][0]["logprobs"])
class CapabilitySplitIndependenceTest(unittest.TestCase):
"""`supports_logprobs_echo` and `supports_tok_ids` gate two unrelated SUBMIT extension
keys and must be settable one without the other. A mimo engine has the first and not
the second; the doubles below also set them apart by hand, so the split is shown to be
real rather than cosmetic."""
def test_logprobs_is_refused_when_only_token_id_intake_is_available(self):
base = _spawn_test_server(self, ScriptedEngine(supports_logprobs_echo=False,
supports_tok_ids=True))
status, error = _error_body(self, lambda: _post_completions(base, {
"model": "test-model", "prompt": "hi", "logprobs": 1}))
self.assertEqual(status, 400)
self.assertEqual(error["param"], "logprobs")
def test_logprobs_is_served_when_only_the_numeric_channel_is_available(self):
base = _spawn_test_server(self, ScriptedEngine(supports_logprobs_echo=True,
supports_tok_ids=False))
with _post_completions(base, {"model": "test-model", "prompt": "hi",
"logprobs": 1}) as response:
choice = json.load(response)["choices"][0]
self.assertEqual(choice["logprobs"]["tokens"], ["ok"])
def test_an_engine_sets_the_two_flags_independently_of_each_other(self):
# Set from the same arch check today, but each read site checks the one it means:
# one flag forced off must not take the other with it.
process = FakeProcess(lambda process, written: None)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
self.addCleanup(engine.close)
self.assertTrue(engine.supports_logprobs_echo)
self.assertTrue(engine.supports_tok_ids)
engine.supports_logprobs_echo = False
self.assertTrue(engine.supports_tok_ids)
def test_a_mimo_engine_has_the_numeric_channel_and_not_token_id_intake(self):
# mimo.c keeps the channel's whole contract (an ECHO for every prompt position
# unless a pin photo covers the prefix); it reads its frames with serve_codec.h,
# which has no `ids=`.
process = FakeProcess(lambda process, written: None)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("mimo", "model", cap=8, family=family_by_id("mimo"))
self.addCleanup(engine.close)
self.assertTrue(engine.supports_logprobs_echo)
self.assertFalse(engine.supports_tok_ids)
class ChatFlavorLogprobsGateTest(unittest.TestCase):
"""The logprobs/echo gate is keyed on the engine family (`supports_logprobs_echo`). A chat
flavor (#1757) selects the chat template, the tool-call parser and the default thinking
effort; it does not change what the engine can do. Under the qwen38 flavor a qwen36
engine parses tool calls with the qwen38 parser, which reports no span maps, so that
engine must never be asked for the numeric channel. Today detect_chat_flavor flavors
only qwen36, and only for a checkpoint that ships Qwen3.8's template. The invariant test
below feeds that template to every registry family and requires that no family it
flavors has the channel."""
def _engine(self, arch, flavor):
# Every SUBMIT gets its terminal frame, so a request that wrongly passes the gate
# completes at once and fails on an assertion, not on the client's timeout.
def respond(process, written):
if written.startswith(b"SUBMIT "):
request_id = written.split()[1]
process.stdout.feed(b"DONE " + request_id + b" STAT 1 1 0 1 1 0\n")
process = FakeProcess(respond)
with patch("openai_server.ARCH", arch), patch("openai_server.CHAT_FLAVOR", flavor), \
patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("engine", "model")
self.addCleanup(engine.close)
return engine
def test_no_family_with_the_numeric_channel_is_ever_given_a_chat_flavor(self):
# Both sides come from the sources the server reads: the registry's families, the
# capability each one's Engine sets, and detect_chat_flavor over a checkpoint that
# ships Qwen3.8's template -- the one input that flavors anything today.
families = family_ids()
self.assertGreater(len(families), 0)
with tempfile.TemporaryDirectory() as model_dir:
(Path(model_dir) / "chat_template.jinja").write_text(
"{%- set resolved_reasoning_effort = reasoning_effort|default('xhigh') %}",
encoding="utf-8")
flavored = {family_id for family_id in families
if detect_chat_flavor(family_id, model_dir) is not None}
channel = set()
for family_id in families:
process = FakeProcess(lambda _process, _frame: None)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("engine", "model", cap=1, family=family_by_id(family_id))
self.addCleanup(engine.close)
if engine.supports_logprobs_echo:
channel.add(family_id)
self.assertIn("qwen36", flavored)
self.assertIn("glm", channel)
self.assertEqual(flavored & channel, set())
def test_logprobs_is_refused_on_a_qwen36_engine_with_the_qwen38_template(self):
engine = self._engine("qwen36", "qwen38")
self.assertFalse(engine.supports_logprobs_echo)
base = _spawn_test_server(self, engine)
with patch("openai_server.ARCH", "qwen36"), patch("openai_server.CHAT_FLAVOR", "qwen38"):
status, error = _error_body(self, lambda: _post_chat(base, {
"model": "test-model", "logprobs": True,
"messages": [{"role": "user", "content": "hi"}]}))
self.assertEqual(status, 400)
self.assertEqual(error["param"], "logprobs")
self.assertEqual(error["code"], "unsupported_parameter")
self.assertEqual(
error["message"],
"Log probabilities are not requested from this engine by these endpoints.")
# Refused before anything reached the engine, not after a turn it then discarded.
self.assertEqual([w for w in engine.process.writes if w.startswith(b"SUBMIT ")], [])
def test_the_same_request_passes_the_gate_on_a_glm_engine(self):
engine = self._engine("glm", None)
self.assertTrue(engine.supports_logprobs_echo)
body = {"model": "test-model", "logprobs": True,
"messages": [{"role": "user", "content": "hi"}]}
self.assertEqual(logprobs_options(body, True, engine.supports_logprobs_echo),
(1, False, 0))
class SubmitHeaderExtensionTest(unittest.TestCase):
"""The SUBMIT header, byte for byte. A request that opts into nothing must produce the
header it produced before this channel existed, and an opted-in one must carry the
extension keys behind the seventh numeric field."""
def _engine(self, expected, frames):
def respond(process, frame):
self.assertEqual(frame, expected)
process.stdout.feed(frames)
process = FakeProcess(respond)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
return engine, process
def test_a_plain_request_submit_header_is_byte_identical(self):
expected = b"SUBMIT 1 0 5 4 0.25 0.9\n" + b"hello\n"
engine, process = self._engine(
expected, b"DATA 1 2\nok\nDONE 1 STAT 1 2.5 0 1.0 5 0\n")
engine.generate("hello", 4, 0.25, 0.9, [].append)
engine.close()
self.assertEqual(process.writes, [expected])
def test_an_opted_in_request_carries_the_extension_behind_the_seventh_field(self):
# coli_submit_parse reaches its key=value arm only after seven numeric fields, and
# sending the extension on every request would make the plain-header test above
# fail.
expected = b"SUBMIT 1 0 5 4 0.25 0.9 0 logprobs=2\n" + b"hello\n"
engine, process = self._engine(
expected,
b"ACCEPT 1 5\nECHO 1 1 0 nan 0\nh\n"
b"DATA 1 2 -0.223144 1 3 -0.223144\nok\nDONE 1 STAT 1 2.5 0 1.0 5 0\n")
stats = engine.generate("hello", 4, 0.25, 0.9, [].append, logprobs=2,
gbytes_before_ext=True)
engine.close()
self.assertEqual(process.writes, [expected])
self.assertEqual(stats["logprobs"]["generated"],
[(b"ok", {"lp": -0.223144, "topk": [(3, -0.223144)]})])
def test_the_seventh_field_is_sent_only_when_the_caller_asks_for_it(self):
# It is a request, not a rule: engines that parse a header without that field
# reject or mis-read one carrying it, so only a caller whose engine uses
# coli_submit_parse asks for it. `pin` rides the same request.
expected = b"SUBMIT 1 0 5 0 0.25 0.9 logprobs=1 pin=1\n" + b"hello\n"
engine, process = self._engine(
expected, b"ACCEPT 1 5\nECHO 1 1 0 nan 0\nh\nDONE 1 STAT 0 2.5 0 1.0 5 0\n")
engine.generate("hello", 0, 0.25, 0.9, lambda _chunk: None, logprobs=1, pin=True,
on_echo=lambda _record: None)
engine.close()
self.assertEqual(process.writes, [expected])
class EngineExtensionCallSiteTest(unittest.TestCase):
"""Each of this server's call sites into Engine.generate() applies the same extension
profile, so none can be left behind. For each: an opted-in request emits the extended
header, and a plain request emits the base header."""
def _header(self, path, body, engine_frames=None):
"""The SUBMIT header one HTTP request produces, taken off a fake engine process."""
headers = []
def respond(process, frame):
headers.append(frame.split(b"\n", 1)[0])
request_id = frame.split()[1]
process.stdout.feed(engine_frames.replace(b"{id}", request_id)
if engine_frames else
b"ACCEPT " + request_id + b" 1\n"
b"DATA " + request_id + b" 2\nok\n"
b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
process = FakeProcess(respond)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
self.addCleanup(engine.close)
base = _spawn_test_server(self, engine)
with _post_json(base, path, body) as response:
response.read()
return headers[0]
OPTED_IN_FRAMES = (b"ACCEPT {id} 1\n"
b"DATA {id} 2 -0.5 1 3 -0.5\nok\n"
b"DONE {id} STAT 1 2.5 0 1.0 5 0\n")
def test_non_streaming_completions_site(self):
plain = self._header("/v1/completions", {"model": "test-model", "prompt": "hi"})
self.assertNotIn(b"logprobs=", plain)
opted = self._header("/v1/completions",
{"model": "test-model", "prompt": "hi", "logprobs": 2},
self.OPTED_IN_FRAMES)
self.assertIn(b" 0 logprobs=2", opted)
def test_non_streaming_chat_site(self):
messages = [{"role": "user", "content": "hi"}]
plain = self._header("/v1/chat/completions",
{"model": "test-model", "messages": messages})
self.assertNotIn(b"logprobs=", plain)
opted = self._header("/v1/chat/completions",
{"model": "test-model", "messages": messages,
"logprobs": True, "top_logprobs": 1},
self.OPTED_IN_FRAMES)
self.assertIn(b" 0 logprobs=1", opted)
def test_streaming_plain_site_keeps_the_base_header(self):
# Streaming with logprobs is a named 400, so this site can only ever be reached
# without the extension -- and must never grow it.
header = self._header("/v1/completions",
{"model": "test-model", "prompt": "hi", "stream": True})
self.assertNotIn(b"logprobs=", header)
status, error = _error_body(self, lambda: self._header(
"/v1/completions",
{"model": "test-model", "prompt": "hi", "stream": True, "logprobs": 1}))
self.assertEqual((status, error["param"]), (400, "logprobs"))
def test_streaming_chat_with_tools_site_keeps_the_base_header(self):
messages = [{"role": "user", "content": "hi"}]
tools = [{"type": "function",
"function": {"name": "search",
"parameters": {"type": "object", "properties": {}}}}]
header = self._header("/v1/chat/completions",
{"model": "test-model", "messages": messages,
"tools": tools, "stream": True})
self.assertNotIn(b"logprobs=", header)
status, error = _error_body(self, lambda: self._header(
"/v1/chat/completions",
{"model": "test-model", "messages": messages, "tools": tools,
"stream": True, "logprobs": True}))
self.assertEqual((status, error["param"]), (400, "logprobs"))
def test_the_messages_endpoint_ignores_a_logprobs_field(self):
# `logprobs` is not in the Anthropic request shape, and this endpoint builds its
# own translated request rather than forwarding the client's body, so the field
# never reaches the options parser. A 200 with no logprobs anywhere, for either
# endpoint's spelling of the field.
for value in (True, 1):
with self.subTest(value=value):
base = _spawn_test_server(self, ScriptedEngine(chunks=("Hello",)))
with _post_json(base, "/v1/messages",
{"model": "test-model", "max_tokens": 4,
"messages": [{"role": "user", "content": "hi"}],
"logprobs": value}) as response:
body = json.load(response)
self.assertEqual(body["content"], [{"type": "text", "text": "Hello"}])
self.assertNotIn("logprobs", json.dumps(body))
def test_the_messages_endpoint_keeps_the_base_header(self):
# /v1/messages sends no extension key, and nothing about this change may give it
# one.
header = self._header("/v1/messages",
{"model": "test-model", "max_tokens": 4,
"messages": [{"role": "user", "content": "hi"}]})
self.assertEqual(header.split(b" ")[:2], [b"SUBMIT", b"1"])
self.assertNotIn(b"logprobs=", header)
self.assertNotIn(b"=", header)
class DispatcherLogprobTailTest(unittest.TestCase):
"""A numeric tail is parsed only for the request that asked for the channel. A
malformed one fails that request alone; a frame carrying fields a request never asked
for is tolerated, as it was before the channel existed."""
def _engine(self, respond):
process = FakeProcess(respond)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
self.addCleanup(engine.close)
return engine, process
def test_extra_fields_on_a_plain_requests_frame_are_tolerated(self):
# The request never asked for the channel, so the tail is not parsed -- not even
# to reject it. This is the base behaviour, restored.
def respond(process, frame):
request_id = frame.split()[1]
process.stdout.feed(b"DATA " + request_id + b" 2 garbage fields here\nok\n"
b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
engine, _process = self._engine(respond)
chunks = []
stats = engine.generate("hi", 4, 0.25, 0.9, chunks.append)
self.assertEqual(chunks, ["ok"])
self.assertNotIn("logprobs", stats)
def _concurrent_fault(self, bad_frame):
"""Drive one plain request and one opted-in request over the same engine, with the
opted-in request's fault arriving while the plain one is still in flight.
The plain request is submitted first and left waiting for its DONE. The opted-in
request's SUBMIT then triggers the bad frame, that request's terminal frame, and
finally the plain request's. Returns `(plain result, opted-in exception)`."""
first_submitted = threading.Event()
def respond(process, frame):
request_id = frame.split()[1]
if b"logprobs=" in frame.split(b"\n", 1)[0]:
process.stdout.feed(bad_frame.replace(b"{id}", request_id))
process.stdout.feed(b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
process.stdout.feed(b"DONE 1 STAT 1 2.5 0 1.0 5 0\n")
else:
process.stdout.feed(b"DATA " + request_id + b" 2\nok\n")
first_submitted.set()
engine, _process = self._engine(respond)
plain = {}
def run_plain():
chunks = []
try:
plain["stats"] = engine.generate("hi", 4, 0.25, 0.9, chunks.append)
plain["chunks"] = chunks
except Exception as error: # recorded, asserted by the caller
plain["error"] = error
thread = threading.Thread(target=run_plain, daemon=True)
thread.start()
self.assertTrue(first_submitted.wait(2))
with self.assertRaises(APIError) as caught:
engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=1,
gbytes_before_ext=True)
thread.join(timeout=2)
self.assertFalse(thread.is_alive())
return plain, caught.exception
def test_a_malformed_tail_spares_a_request_that_is_in_flight(self):
# The containment rule's whole point: not that the dispatcher survived the fault,
# but that the request which never asked for the channel is untouched by it.
# Failing every pending request instead leaves the plain request holding an error
# it did not earn.
plain, error = self._concurrent_fault(b"DATA {id} 2 -0.5 1 3\nok\n")
self.assertEqual(error.status, 500)
self.assertEqual(error.code, "engine_logprob_tail_malformed")
self.assertNotIn("error", plain)
self.assertEqual(plain["chunks"], ["ok"])
self.assertEqual(plain["stats"]["completion_tokens"], 1)
def test_a_malformed_echo_logprob_on_a_request_that_never_opted_in(self):
# The `lp` field is read for EVERY ECHO frame, opted in or not, because the
# closed-set scorer's own `logprob` key comes from it -- so unlike the tail, this
# one is reachable by a request with no numeric channel at all. That is the row of
# the four that was still reaching the dispatcher's blanket handler, which fails
# every request in flight and stops the dispatcher for good. Here the bad frame
# goes to the NON-opted-in request and a second request is in flight beside it.
second_submitted = threading.Event()
def respond(process, frame):
request_id = frame.split()[1]
if b"logprobs=" in frame.split(b"\n", 1)[0]:
process.stdout.feed(b"DATA " + request_id + b" 2 -0.5 1 3 -0.5\nok\n")
second_submitted.set()
else:
process.stdout.feed(b"ECHO " + request_id + b" 1 0 zz 0\nh\n")
# The fault is delivered on the turn's terminal frame, so the engine has
# to end the turn for the request to be answered at all.
process.stdout.feed(
b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
engine, _process = self._engine(respond)
opted = {}
def run_opted():
chunks = []
try:
opted["stats"] = engine.generate(
"hi", 4, 0.25, 0.9, chunks.append, logprobs=1,
gbytes_before_ext=True, on_echo=lambda _r: None)
opted["chunks"] = chunks
except Exception as error: # recorded, asserted below
opted["error"] = error
thread = threading.Thread(target=run_opted, daemon=True)
thread.start()
self.assertTrue(second_submitted.wait(2))
with self.assertRaises(APIError) as caught:
engine.generate("hi", 4, 0.25, 0.9, [].append, on_echo=lambda _r: None)
self.assertEqual(caught.exception.status, 500)
self.assertEqual(caught.exception.code, "engine_logprob_tail_malformed")
self.assertIsNone(engine.dispatcher_error)
# The request beside it never saw the frame and is still being served.
self.assertNotIn("error", opted)
self.assertTrue(thread.is_alive())
thread.join(timeout=0.1)
def test_a_malformed_echo_position_spares_a_request_that_is_in_flight(self):
# The ECHO `pos` field rides the same opted-in frame as the numeric tail and gets
# the same containment. Raising inside the dispatcher instead kills it, and every
# concurrent request on the engine fails with that request's fault.
plain, error = self._concurrent_fault(b"ECHO {id} 1 x nan 0\nh\n")
self.assertEqual(error.status, 500)
# Its own code: the position and the numeric tail are different fields of the
# frame, and a client branching on the code is told which one was unreadable.
self.assertEqual(error.code, "engine_echo_position_malformed")
self.assertEqual(error.param, "echo")
self.assertNotIn("error", plain)
self.assertEqual(plain["chunks"], ["ok"])
def _poll(self, predicate, what):
deadline = time.monotonic() + 2
while time.monotonic() < deadline:
if predicate():
return
time.sleep(0.005)
self.fail(what)
def test_a_recorded_fault_keeps_the_turn_open_until_its_terminal_frame(self):
# The fault is found mid-turn and raised at the end of it. Delivering it when it
# is found would unwind the request thread, and with it the scheduler admission,
# while the engine is still generating and no CANCEL has been written -- the next
# request would then submit into a pipe nobody is reading. The observable form of
# "the turn is still the engine's" is that the pending entry is still there.
release = threading.Event()
def respond(process, frame):
request_id = frame.split()[1]
process.stdout.feed(b"DATA " + request_id + b" 2 -0.5 1 3\nok\n")
def terminal():
release.wait(2)
process.stdout.feed(
b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
threading.Thread(target=terminal, daemon=True).start()
engine, process = self._engine(respond)
result = {}
def run():
try:
engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=1,
gbytes_before_ext=True)
except APIError as error:
result["error"] = error
thread = threading.Thread(target=run, daemon=True)
thread.start()
self._poll(lambda: (engine.pending.get("1") is not None
and engine.pending["1"].failed is not None),
"the fault was never recorded")
# Recorded, held, and not yet delivered.
self.assertIn("1", engine.pending)
self.assertNotIn("error", result)
self.assertTrue(thread.is_alive())
release.set()
thread.join(timeout=2)
self.assertFalse(thread.is_alive())
self.assertEqual(result["error"].code, "engine_logprob_tail_malformed")
self.assertNotIn("1", engine.pending)
# No CANCEL was written: the turn was never abandoned.
self.assertEqual([w for w in process.writes if w.startswith(b"CANCEL")], [])
def test_frames_for_a_failed_or_finished_request_are_ignored(self):
# While the fault is recorded the entry stays, and that request's later frames are
# routed to it and discarded; once its terminal frame has been delivered the entry
# is gone, and anything further for that id belongs to no request. Neither may be
# treated as a fault of its own: the dispatcher keeps running and the next request
# on the same engine completes normally.
def respond(process, frame):
request_id = frame.split()[1]
if b"logprobs=" in frame.split(b"\n", 1)[0]:
process.stdout.feed(b"DATA " + request_id + b" 2 -0.5 1 3\nok\n")
process.stdout.feed(b"DATA " + request_id + b" 2\nxx\n")
process.stdout.feed(b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
# ...and two frames the engine sends after the turn it already ended,
# which belong to no pending entry at all.
process.stdout.feed(b"DATA " + request_id + b" 2\nyy\n")
process.stdout.feed(b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
else:
process.stdout.feed(b"DATA " + request_id + b" 2\nok\n"
b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
engine, _process = self._engine(respond)
with self.assertRaises(APIError) as caught:
engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=1,
gbytes_before_ext=True)
self.assertEqual(caught.exception.code, "engine_logprob_tail_malformed")
chunks = []
stats = engine.generate("hi", 4, 0.25, 0.9, chunks.append)
self.assertEqual(chunks, ["ok"])
self.assertEqual(stats["completion_tokens"], 1)
self.assertIsNone(engine.dispatcher_error)
def _faulted_turn(self, terminal, cancelled=None, settle=1.0):
"""One opted-in request whose numeric tail is malformed, with `terminal` delivered
only once the server answers the fault -- so the turn stays open long enough for
the request thread's idle poll to run, which is where that answer is written.
A timer delivers `terminal` anyway after `settle` seconds, so a build that answers
nothing fails an assertion instead of hanging the suite. Returns `(raised exception
or None, the frames the server wrote)`."""
answered = threading.Event()
def respond(process, frame):
if not frame.startswith(b"SUBMIT"):
answered.set() # STOP or CANCEL: the turn can end now
process.stdout.feed(terminal.replace(b"{id}", frame.split()[1]))
return
request_id = frame.split()[1]
process.stdout.feed(b"DATA " + request_id + b" 2 -0.5 1 3\nok\n")
def fallback():
if not answered.wait(settle):
process.stdout.feed(terminal.replace(b"{id}", request_id))
threading.Thread(target=fallback, daemon=True).start()
engine, process = self._engine(respond)
raised = None
try:
engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=1,
gbytes_before_ext=True, cancelled=cancelled)
except Exception as error: # the class is what the test asserts
raised = error
return raised, process.writes
def test_a_cancellation_outranks_the_recorded_fault_on_a_terminal_done(self):
# The engine may end a cancelled turn with DONE rather than ERROR CANCELLED. The
# client left either way, so the turn is a cancellation, not a server failure --
# which is what the scheduler books when anything but ClientCancelled comes out,
# and what the metrics then count.
raised, writes = self._faulted_turn(b"DONE {id} STAT 1 2.5 0 1.0 5 0\n",
cancelled=lambda: True)
self.assertIsInstance(raised, ClientCancelled)
self.assertEqual([w for w in writes if w.startswith(b"CANCEL")], [b"CANCEL 1\n"])
self.assertEqual([w for w in writes if w.startswith(b"STOP")], [])
def test_a_cancellation_outranks_the_recorded_fault(self):
# The client left mid-turn on a request that also carries a fault of its own. That
# is answered the way it is answered on a healthy turn -- ClientCancelled, not a
# 500 written to a socket that is already closed.
raised, writes = self._faulted_turn(b"ERROR {id} CANCELLED\n",
cancelled=lambda: True)
self.assertIsInstance(raised, ClientCancelled)
# The disconnect is what the engine was told about: CANCEL, not STOP.
self.assertEqual([w for w in writes if w.startswith(b"CANCEL")], [b"CANCEL 1\n"])
self.assertEqual([w for w in writes if w.startswith(b"STOP")], [])
def test_any_other_terminal_error_keeps_the_fault_and_carries_the_engines_text(self):
# The fault happened first and names the framing defect, so it is what the client
# is told; the engine's own parting word is appended rather than dropped. Reporting
# the engine's error instead would tell a client with an over-long prompt to
# shorten it when the real defect was a frame this server could not read.
raised, _writes = self._faulted_turn(b"ERROR {id} CONTEXT_EXCEEDED 9 8\n")
self.assertIsInstance(raised, APIError)
self.assertEqual(raised.status, 500)
self.assertEqual(raised.code, "engine_logprob_tail_malformed")
self.assertIn("malformed per-token logprob tail", raised.message)
self.assertIn("The engine then reported:", raised.message)
self.assertIn("CONTEXT_EXCEEDED", raised.message)
def test_a_faulted_turn_is_stopped_once_and_then_drained(self):
# The turn is doomed but still the engine's: the admission is held to its terminal
# frame either way, and one STOP keeps it from spending the rest of the budget on a
# response nobody will receive. Exactly one, and no CANCEL.
raised, writes = self._faulted_turn(b"DONE {id} STAT 1 2.5 0 1.0 5 0\n")
self.assertIsInstance(raised, APIError)
self.assertEqual(raised.code, "engine_logprob_tail_malformed")
self.assertEqual([w for w in writes if w.startswith(b"STOP")], [b"STOP 1\n"])
self.assertEqual([w for w in writes if w.startswith(b"CANCEL")], [])
def test_a_healthy_slow_turn_is_not_stopped(self):
# The control for the fault guard, and it has to IDLE to be one: a fixture that
# feeds every frame back to back never reaches the `queue.Empty` branch, which is
# the only place the guard lives. This one leaves a gap wider than the 0.05 s poll
# between DATA and DONE -- an ordinary inter-token interval on a large model -- so
# a build that stops on every idle tick writes a STOP here and also skips the
# decode for the rest of the turn, returning a 200 with an empty completion.
def respond(process, frame):
request_id = frame.split()[1]
process.stdout.feed(b"DATA " + request_id + b" 2 -0.5 1 3 -0.5\nok\n")
def terminal():
time.sleep(0.15) # three idle polls, well under the ceiling
process.stdout.feed(
b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
threading.Thread(target=terminal, daemon=True).start()
engine, process = self._engine(respond)
chunks = []
stats = engine.generate("hi", 4, 0.25, 0.9, chunks.append, logprobs=1,
gbytes_before_ext=True)
self.assertEqual([w for w in process.writes if w.startswith(b"STOP")], [])
self.assertEqual([w for w in process.writes if w.startswith(b"CANCEL")], [])
# ...and the turn still delivered, which is what a stray STOP would cost.
self.assertEqual(chunks, ["ok"])
self.assertEqual(stats["completion_tokens"], 1)
def test_a_failed_fault_stop_write_answers_by_name_and_keeps_the_connection(self):
# The STOP written for a recorded fault can itself fail at the pipe. An engine this
# server cannot write to outranks a fault in the stream it cannot read: the request
# is answered with a named error rather than having its connection dropped, which
# is what an unhandled write error does. Written against this tree's own STOP write
# so the pin survives the composition with the checked writer.
stop_tried = threading.Event()
class DeadOnStop:
"""The engine's stdin: accepts the SUBMIT, refuses the STOP."""
def __init__(self, process):
self.process = process
def write(self, data):
# The checked writer (#1721) hands every write a `memoryview`
# slice, so normalise before pattern-matching the frame bytes --
# the same normalisation `FakeProcess.write` above does.
data = bytes(data)
if data.startswith(b"STOP"):
stop_tried.set()
raise BrokenPipeError(32, "Broken pipe")
return self.process.real_write(data)
def flush(self):
pass
def respond(process, frame):
request_id = frame.split()[1]
process.stdout.feed(b"DATA " + request_id + b" 2 -0.5 1 3\nok\n")
def terminal():
# DONE only once the STOP was tried, as `_faulted_turn` does. A fixed
# 0.15 s let a slow runner read DATA and DONE back to back before the
# idle poll wrote the STOP (macOS CI on dev 1f9a8523): the turn then
# ended on the recorded fault and no STOP was ever written. The timeout
# still ends the turn on a build that never writes it, which fails below.
stop_tried.wait(1.0)
process.stdout.feed(
b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
threading.Thread(target=terminal, daemon=True).start()
engine, process = self._engine(respond)
process.real_write = process.write
process.stdin = DeadOnStop(process)
with self.assertRaises(Exception) as caught:
engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=1,
gbytes_before_ext=True)
# The turn is over and nothing will ever answer it: the entry must not be left for
# the dispatcher to deliver to a caller that has already unwound, nor to hold a
# request id the next turn could be given.
self.assertNotIn("1", engine.pending)
self.assertEqual(engine.pending, {})
# Named, and reaching the caller: an unraised write error would instead unwind
# through the handler and close the socket with no status line on it.
raised = caught.exception
self.assertNotIsInstance(raised, BrokenPipeError)
named = raised.code if isinstance(raised, APIError) else str(raised)
self.assertTrue(named, "the failed STOP write must be reported by name")
self.assertIn("STOP", str(raised).upper() + (raised.message.upper()
if isinstance(raised, APIError) else ""))
def test_a_failed_fault_stop_write_by_zero_progress_answers_by_name_and_keeps_the_connection(self):
# The sibling test above fails the STOP write with a BrokenPipeError
# -- an OSError, which is exactly the shape the pre-#1721 inline form
# this branch replaced already caught (`except OSError`). #1721's
# checked writer (`_write_all`) treats a write that returns 0 -- the
# pipe accepts the call, takes nothing, and raises no exception at
# all -- as failing the same way (a fail-closed RuntimeError, per
# `WriteAllTest.test_zero_return_fails_closed_instead_of_spinning`).
# The raw inline form has no such check: it discards write()'s
# return value, so it does not see this shape as a failure at all.
# `stop_sent` is set, the branch never fires again, and nothing else
# is left to solicit a terminal frame for this turn -- the request
# hangs until the engine (or, here, `close()`'s `_fail_pending`)
# ends it. That regression -- a named 500 `engine_error` at the head
# (do_POST's generic `except Exception` names any uncaught failure
# this way, #597 item 3) becoming a silent hang on the raw form -- is
# deep audit S1 (AUDIT_S3_devmerge_5a071274.md), and this pins it.
# Reuses the sibling's fixture, request shape and assertions; only
# the write failure's shape changes.
class ZeroOnStop:
"""The engine's stdin: accepts the SUBMIT, takes 0 bytes on the STOP."""
def __init__(self, process):
self.process = process
def write(self, data):
# Same memoryview normalisation the sibling's DeadOnStop and
# FakeProcess.write both do.
data = bytes(data)
if data.startswith(b"STOP"):
return 0
return self.process.real_write(data)
def flush(self):
pass
def respond(process, frame):
request_id = frame.split()[1]
process.stdout.feed(b"DATA " + request_id + b" 2 -0.5 1 3\nok\n")
engine, process = self._engine(respond)
process.real_write = process.write
process.stdin = ZeroOnStop(process)
result = {}
def run():
try:
engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=1,
gbytes_before_ext=True)
except Exception as error: # recorded, asserted below
result["error"] = error
# Wait under test: at the head, the idle poll (0.05s) finds the
# recorded fault and the zero-progress STOP write answers within a
# tick or two -- well under the 0.3s per-test budget. Under the
# reverted (pre-#1721 inline) form there is nothing left to end the
# turn, so this call would hang indefinitely; running it on a daemon
# thread and bounding the join is what turns that hang into a
# prompt, visible test failure instead of stalling the suite.
thread = threading.Thread(target=run, daemon=True)
thread.start()
thread.join(timeout=2)
self.assertFalse(thread.is_alive(),
"the fault-STOP write must answer, not hang")
# The turn is over and nothing will ever answer it: the entry must not be
# left for the dispatcher to deliver to a caller that has already unwound,
# nor to hold a request id the next turn could be given.
self.assertNotIn("1", engine.pending)
self.assertEqual(engine.pending, {})
# Named, and reaching the caller: an unraised write error would instead
# leave the request thread blocked with no answer at all.
self.assertIn("error", result, "the failed STOP write must answer, not hang")
raised = result["error"]
self.assertNotIsInstance(raised, BrokenPipeError)
named = raised.code if isinstance(raised, APIError) else str(raised)
self.assertTrue(named, "the failed STOP write must be reported by name")
self.assertIn("STOP", str(raised).upper() + (raised.message.upper()
if isinstance(raised, APIError) else ""))
def test_the_malformed_tail_battery(self):
# A required field absent or non-numeric, a k outside the engine's interface, or a
# negative token id. Every one of them is a missing or unreadable field, never a
# frame that merely carries more than the grammar names.
for tail in (b"-0.5", b"-0.5 1 3", b"x 1 3 -0.5", b"-0.5 99 3 -0.5",
b"-0.5 1 -3 -0.5", b"-0.5 x 3 -0.5"):
with self.subTest(tail=tail):
def respond(process, frame, tail=tail):
request_id = frame.split()[1]
process.stdout.feed(b"DATA " + request_id + b" 2 " + tail + b"\nok\n"
b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
engine, _process = self._engine(respond)
with self.assertRaises(APIError) as caught:
engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=1,
gbytes_before_ext=True)
self.assertEqual(caught.exception.code, "engine_logprob_tail_malformed")
def test_extra_trailing_fields_on_an_opted_in_frame_are_served(self):
# The parser consumes the fields the grammar defines and ignores what follows. The
# base tolerated a longer frame for every caller, and this parser now runs for
# internal consumers of the channel that read only part of the record.
def respond(process, frame):
request_id = frame.split()[1]
process.stdout.feed(
b"DATA " + request_id + b" 2 -0.5 1 3 -0.5 9 -9.0\nok\n"
b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
engine, _process = self._engine(respond)
chunks = []
stats = engine.generate("hi", 4, 0.25, 0.9, chunks.append, logprobs=1,
gbytes_before_ext=True)
self.assertEqual(chunks, ["ok"])
self.assertEqual(stats["logprobs"]["generated"],
[(b"ok", {"lp": -0.5, "topk": [(3, -0.5)]})])
def test_the_closed_set_scoring_path_keeps_the_bases_frame_tolerance(self):
# The other caller of the numeric channel reads `pos`/`logprob`/`text` and nothing
# else, and its engine's ECHO frames may carry more than this parser names. Driven
# through a real Engine and the real dispatcher, in that caller's own call shape,
# because a FakeEngine cannot see a framing change at all.
for label, echo_frame in (
("exact", b"ECHO {id} 1 0 nan 0\nh\n"),
("one extra trailing field", b"ECHO {id} 1 0 nan 0 7\nh\n"),
("a populated table plus an extra", b"ECHO {id} 1 0 -0.5 1 3 -0.5 7\nh\n")):
with self.subTest(frame=label):
def respond(process, frame, echo_frame=echo_frame):
request_id = frame.split()[1]
process.stdout.feed(echo_frame.replace(b"{id}", request_id))
process.stdout.feed(b"DONE " + request_id + b" STAT 0 2.5 0 1.0 5 0\n")
engine, _process = self._engine(respond)
echoes = []
engine.generate("hi", 0, 0.0, 1.0, lambda _chunk: None,
logprobs=1, pin=True, on_echo=echoes.append)
self.assertEqual(len(echoes), 1)
self.assertEqual(echoes[0]["pos"], 0)
self.assertEqual(echoes[0]["text"], "h")
def test_a_well_formed_tail_reaches_the_caller(self):
def respond(process, frame):
request_id = frame.split()[1]
process.stdout.feed(
b"ACCEPT " + request_id + b" 1\n"
b"ECHO " + request_id + b" 1 0 nan 0\nh\n"
b"DATA " + request_id + b" 2 -0.5 2 3 -0.5 4 -1.25\nok\n"
b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
engine, _process = self._engine(respond)
echoes = []
stats = engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=2,
gbytes_before_ext=True, on_echo=echoes.append)
self.assertEqual(stats["logprobs"]["generated"],
[(b"ok", {"lp": -0.5, "topk": [(3, -0.5), (4, -1.25)]})])
# The echo record keeps the scorer's three keys and their meanings, and gains the
# payload and the numeric tail the logprobs surface needs.
self.assertEqual(echoes[0]["pos"], 0)
self.assertIsNone(echoes[0]["logprob"])
self.assertEqual(echoes[0]["text"], "h")
self.assertEqual(echoes[0]["bytes"], b"h")
self.assertTrue(math.isnan(echoes[0]["lp"]))
self.assertEqual(echoes[0]["topk"], [])
class EngineCapsTest(unittest.TestCase):
def test_engine_learns_its_vision_tower_from_the_handshake(self):
# CAPS vision=<0|1> sits between READY and STAT; an engine that predates the
# line (or has no tower to speak of) leaves the flag unknown, not False.
for line, expected in ((b"CAPS vision=1\n", True), (b"CAPS vision=0\n", False),
(b"", None)):
with self.subTest(line=line):
process = FakeProcess(lambda _process, _frame: None)
process.stdout = BlockingStream(READY + line + b"STAT 0 0 0 0\n")
with patch("openai_server.ARCH", "glm53"), \
patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm53", "model")
try:
self.assertIs(engine.vision, expected)
finally:
engine.close()
class EngineLoadFailTest(unittest.TestCase):
"""An engine that cannot load says `LOAD_FAIL kind=<kind> <detail>` and exits
before READY; the gateway raises the kind, not "exited unexpectedly"."""
def _engine(self, stdout_bytes):
process = FakeProcess(lambda _process, _frame: None)
process.stdout = BlockingStream(stdout_bytes)
process.stdout.close() # the engine is gone: EOF right after what it said
with patch("openai_server.ARCH", "glm53"), \
patch("openai_server.subprocess.Popen", return_value=process):
return Engine("glm53", "model")
def test_load_fail_line_is_raised_as_its_kind(self):
detail = "model-00007.safetensors: short read at EOF (off 8, 0/16 bytes)"
for kind in ("nomem", "io", "format", "unsupported"):
with self.subTest(kind=kind):
with self.assertRaises(openai_server.EngineLoadError) as caught:
self._engine(f"LOAD_FAIL kind={kind} {detail}\n".encode())
self.assertEqual(caught.exception.kind, kind)
self.assertEqual(caught.exception.detail, detail)
self.assertIn(f"kind={kind}", str(caught.exception))
self.assertNotIn("unexpectedly", str(caught.exception))
def test_engine_that_says_nothing_is_still_exited_unexpectedly(self):
with self.assertRaisesRegex(RuntimeError, "exited unexpectedly") as caught:
self._engine(b"")
self.assertNotIsInstance(caught.exception, openai_server.EngineLoadError)
def test_parse_load_fail_takes_the_last_line_and_keeps_the_detail_whole(self):
parse = openai_server.parse_load_fail
self.assertIsNone(parse(b"HWINFO 8 64 32 0 0 cpu|none\n"))
self.assertEqual(parse(b"noise\nLOAD_FAIL kind=nomem malloc 12 bytes for tensor a.b failed\n"),
("nomem", "malloc 12 bytes for tensor a.b failed"))
self.assertEqual(parse(b"LOAD_FAIL kind=io\n"), ("io", ""))
def test_serve_logs_the_kind_and_exits(self):
error = openai_server.EngineLoadError("nomem", "malloc 12 bytes for tensor a.b failed")
stderr = io.StringIO()
# serve() sets the module's ARCH / CHAT_FLAVOR for the family it resolves;
# patched here so the rest of the suite keeps its defaults.
with patch("openai_server.ARCH", openai_server.ARCH), \
patch("openai_server.CHAT_FLAVOR", openai_server.CHAT_FLAVOR), \
patch("openai_server.Engine", side_effect=error), \
patch("openai_server.resolve_model") as resolve, \
patch("openai_server.default_engine", return_value="colibri"), \
patch("openai_server.detect_chat_flavor", return_value=None), \
patch("openai_server.APIServer") as api_server, \
patch("sys.stderr", stderr):
resolve.return_value.descriptor = openai_server.family_by_id("glm53")
with self.assertRaises(SystemExit) as caught:
openai_server.serve("model", "127.0.0.1", 8000, "id", None)
self.assertEqual(caught.exception.code, 1)
self.assertIn("[gateway] engine load failed: kind=nomem malloc 12 bytes for tensor a.b failed",
stderr.getvalue())
api_server.return_value.server_close.assert_called_once()
class ServedModalityTest(unittest.TestCase):
"""What /v1/models says it accepts is what the engine loaded. The tower is a
property of the checkpoint (a glm53 export can carry vision_config and no
model.visual.* tensors), not of the family, so the card follows the engine's
handshake, and a picture sent to an engine that announced no tower is refused
by name at the gateway instead of dying in the engine as a bare BAD_REQUEST."""
def serve(self, vision):
engine = FakeEngine()
if vision is not None:
engine.vision = vision
server = APIServer(("127.0.0.1", 0), engine, "test-model", "secret", 16, kv_slots=1)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
def stop():
server.scheduler.close()
server.shutdown()
server.server_close()
thread.join(timeout=2)
self.addCleanup(stop)
return engine, f"http://127.0.0.1:{server.server_port}"
def request(self, base, path, body=None):
headers = {"Authorization": "Bearer secret"}
data = None
if body is not None:
data = json.dumps(body).encode()
headers["Content-Type"] = "application/json"
return urlopen(Request(base + path, data=data, headers=headers), timeout=2)
def test_card_and_health_report_the_live_tower(self):
with patch("openai_server.ARCH", "glm53"):
for vision, expected in ((True, ["text", "image"]), (False, ["text"]),
(None, ["text"])):
with self.subTest(vision=vision):
_engine, base = self.serve(vision)
with self.request(base, "/v1/models") as response:
card = json.load(response)["data"][0]
self.assertEqual(card["input_modalities"], expected)
with self.request(base, "/v1/models/test-model") as response:
self.assertEqual(json.load(response)["input_modalities"], expected)
with self.request(base, "/health") as response:
self.assertEqual(json.load(response)["input_modalities"], expected)
def test_a_family_without_an_image_path_never_claims_images(self):
# Even an engine that announces a tower is text-only to its clients when
# the gateway has no placeholder expansion for the family.
with patch("openai_server.ARCH", "glm"):
_engine, base = self.serve(True)
with self.request(base, "/v1/models") as response:
self.assertEqual(json.load(response)["data"][0]["input_modalities"], ["text"])
def test_tower_less_engine_refuses_a_picture_by_name(self):
picture = {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}
with patch("openai_server.ARCH", "glm53"):
engine, base = self.serve(False)
with self.assertRaises(HTTPError) as caught:
self.request(base, "/v1/chat/completions", {
"model": "test-model",
"messages": [{"role": "user", "content": [
{"type": "text", "text": "what is this?"}, picture]}]})
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
error = json.load(caught.exception)["error"]
self.assertIn("vision tower", error["message"])
self.assertEqual(error["param"], "messages")
self.assertEqual(engine.calls, []) # refused before the engine
# the same engine still serves text
with self.request(base, "/v1/chat/completions", {
"model": "test-model",
"messages": [{"role": "user", "content": "hello"}]}) as response:
self.assertEqual(response.status, 200)
self.assertEqual(len(engine.calls), 1)
if __name__ == "__main__":
unittest.main()