Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Merge dev: take the seed acceptance (#1720) and the checked engine writes (#1721) under the logprobs work

monotophic5a071274485b
diff --cc c/openai_server.py
index 729ded8c,3c49576b..57e8b53e
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@@ -3355,10 -3083,12 +3369,10 @@@ def generation_options(body, limit)
if choice != "none" and not (body.get("tools") or body.get("functions")):
raise APIError(400, "`tool_choice` requires `tools`.", "tool_choice", "invalid_value")
stop_sequences = parse_stop_sequences(body)
- if body.get("logprobs"):
- raise APIError(400, "Log probabilities are not supported yet.", "logprobs", "unsupported_parameter")
if body.get("frequency_penalty", 0) or body.get("presence_penalty", 0):
raise APIError(400, "Token penalties are not supported yet.", None, "unsupported_parameter")
- if body.get("seed") is not None:
- raise APIError(400, "Per-request seeds are not supported yet.", "seed", "unsupported_parameter")
+ # `seed` is accepted for request-shape compatibility and silently discarded:
+ # this server puts no per-request seed on the wire, at any temperature.
# response_format -> optional per-request grammar for the engine's grammar-forced
# draft source (#70/#148). NEVER a sampling constraint: drafts are verified, so a
# schema the engine cannot compile degrades to "no speedup", not to an error and
@@@ -4125,28 -3367,35 +4139,57 @@@ def _win_kill_on_close_job(pid)
return None # never let process bookkeeping break starting the engine
+ def _write_all(stream, data, frame):
+ """Write every byte of `data` to `stream`, looping on short writes.
+
+ The production engine stdin is a raw, unbuffered pipe (bufsize=0 ->
+ io.FileIO), whose write() is a single os.write() and may transfer fewer
+ bytes than it was given (a signal landing mid-write, a full pipe buffer
+ on a large IMAGE frame). Discarding the return value would leave the
+ tail of a frame unsent and desynchronize the engine's stdin framing, so
+ the remainder is re-offered until it is all consumed.
+
+ Neither `None` nor 0 is progress. `RawIOBase.write` answers `None` when
+ the stream is non-blocking and could not take a single byte, and 0 says
+ the same thing with a count; re-offering the buffer after either would
+ spin forever, so both fail closed as the named engine-write error a
+ broken pipe raises."""
+ written = 0
+ total = len(data)
+ view = memoryview(data)
+ while written < total:
+ sent = stream.write(view[written:])
+ # None is RawIOBase's "not one byte went out", not an uncounted
+ # full write, so it fails closed exactly as a zero count does.
+ if sent is None or sent <= 0:
+ raise RuntimeError(
+ f"failed to write {frame} to the engine "
+ f"(stdin took {written} of {total} bytes)")
+ written += sent
+
+
+class _Pending:
+ """One in-flight engine request: the queue its frames are delivered on, whether it asked
+ for the per-token numeric channel, and any fault the dispatcher has recorded against it.
+
+ The dispatcher parses a frame's numeric tail only for a request that asked for it, so a
+ frame carrying fields a request never requested is tolerated exactly as it was before
+ the channel existed, and a fault in one request's frames can only fail that request.
+
+ `failed` is how that stays true without abandoning the engine's turn. The entry is kept
+ rather than dropped: the request's later frames are routed here and discarded, and the
+ error is delivered on its terminal frame, so the thread holding the scheduler admission
+ keeps holding it until the engine says the turn is over. Dropping the entry instead
+ would free the admission while the engine is still generating, and the next request
+ would submit into a pipe nobody is reading."""
+ __slots__ = ("events", "logprob_channel", "failed")
+
+ def __init__(self, logprob_channel=False):
+ self.events = queue.Queue()
+ self.logprob_channel = logprob_channel
+ self.failed = None
+
+
class Engine:
# cap=None = "not explicitly set": a glm-arch model's engine resolves the
# 0 sentinel (8 historically, 1 on Metal+darwin+fast SSD -- colibri.c
@@@ -4227,9 -3469,35 +4270,35 @@@
with self.pending_lock:
requests = list(self.pending.values())
self.pending.clear()
- for events in requests:
- events.put(("error", error))
+ for entry in requests:
+ entry.events.put(("error", error))
+ def _write_frame(self, request_id, data, frame):
+ """Checked server->engine protocol write for CANCEL/STOP: the write
+ and its flush happen under one write_lock acquisition. Any failure
+ here -- an OSError from the pipe itself, or _write_all's own
+ fail-closed RuntimeError on a None/zero-progress write -- drops this
+ request's pending-map entry: the dispatcher only does that on this
+ id's own DONE/ERROR frame, and neither arrives when the write that
+ would have solicited one never reached the engine. An OSError is
+ additionally re-raised as a named RuntimeError rather than left as
+ itself: BrokenPipeError is a ConnectionError subclass, so an
+ unwrapped failure here would fall into do_POST's client-hangup
+ handler (`except ConnectionError: pass`) and the client would see a
+ silent connection close instead of the 500 engine_error the failure
+ actually is. _write_all's own RuntimeError is already the named
+ error this raises for an OSError, so it is re-raised as-is."""
+ try:
+ with self.write_lock:
+ _write_all(self.process.stdin, data, frame)
+ self.process.stdin.flush()
+ except Exception as error:
+ with self.pending_lock:
+ self.pending.pop(request_id, None)
+ if isinstance(error, OSError):
+ raise RuntimeError(f"failed to write {frame} to the engine ({error})") from error
+ raise
+
def _read_exact(self, size):
chunks = []
remaining = size
@@@ -4617,31 -3772,7 +4693,22 @@@
# DONE frame; ClientCancelled is raised when it arrives.
if not cancel_sent and not stop_sent and cancelled and cancelled():
cancel_sent = True
- with self.write_lock:
- self.process.stdin.write(f"CANCEL {request_id}\n".encode())
- self.process.stdin.flush()
+ self._write_frame(request_id, f"CANCEL {request_id}\n".encode(), "CANCEL")
+ elif not cancel_sent and not stop_sent and pending.failed is not None:
+ # A fault was recorded for this request, so nothing it generates from
+ # here can reach the client. The turn is still the engine's -- the
+ # admission is held to its terminal frame either way -- but it should
+ # not run the rest of the budget for a response nobody will get.
++ #
++ # The STOP goes out through the checked frame writer (#1721), which
++ # drops this request's pending-map entry and re-raises an OSError as
++ # the named engine_error -- exactly the handling this branch carried
++ # inline before that writer existed. Keeping the entry would be worse
++ # than losing it: a STOP that never reached the engine is never
++ # answered with a terminal frame, so the admission would be held for
++ # the process's lifetime.
+ stop_sent = True
- try:
- with self.write_lock:
- self.process.stdin.write(f"STOP {request_id}\n".encode())
- self.process.stdin.flush()
- except OSError as error:
- # An engine this server cannot write to outranks a fault in the
- # stream it cannot read: the request is answered by name instead
- # of having the write error unwind through the handler, which
- # closes the client's connection with no status line on it. The
- # entry goes too -- a STOP that never reached the engine will
- # never be answered with a terminal frame, so waiting for one
- # would hold the admission for the process's lifetime.
- with self.pending_lock:
- self.pending.pop(request_id, None)
- raise RuntimeError(
- f"failed to write STOP to the engine ({error})") from error
++ self._write_frame(request_id, f"STOP {request_id}\n".encode(), "STOP")
continue
if kind == "accept":
if accepted:
@@@ -4649,18 -3780,11 +4716,16 @@@
_accept(value)
elif kind == "data":
_accept({"prompt_tokens": None})
+ # The dispatcher wraps the payload in a tuple only when a logprob record
+ # rides along; bare bytes is the non-opted-in shape.
+ data, record = value if isinstance(value, tuple) else (value, None)
+ if record is not None:
+ generated_logprobs.append((data, record))
if not cancel_sent and not stop_sent:
- decode(value)
+ decode(data)
if stopped and stopped():
stop_sent = True
- with self.write_lock:
- self.process.stdin.write(f"STOP {request_id}\n".encode())
- self.process.stdin.flush()
+ self._write_frame(request_id, f"STOP {request_id}\n".encode(), "STOP")
elif cancelled and cancelled():
# Same admission-holding rule as the idle branch above:
# send CANCEL, then keep consuming frames until the
diff --cc c/tests/golden_fixture_capture.py
index f966662f,3633c738..8e2aac30
--- a/c/tests/golden_fixture_capture.py
+++ b/c/tests/golden_fixture_capture.py
@@@ -21,11 -23,16 +23,18 @@@ Usage
# 3. compare (exit 0 = byte-identical modulo normalized fields):
python3 tests/golden_fixture_capture.py diff fixtures_pre fixtures_post
-Battery: chat, chat+tools, chat streaming, completions without logprobs, and
-the error cases whose behavior must not move (array-prompt 400, logprobs 400,
-out-of-range temperature 400), plus one case that is a no-op from this change
-on (seed accepted-and-ignored), plus /v1/models.
+Battery: chat, chat+tools, chat streaming, completions without logprobs, the
- logprobs case now served on the glm engine (which a pre-change capture answered
- with a 400, so the two differ there by construction), and the error cases whose
- behavior must not move (seed 400, array-prompt 400, out-of-range temperature
- 400) plus /v1/models.
++seed case that is a no-op from #1720 on (accepted-and-ignored, formerly a 400),
++the logprobs case now served on the glm engine (formerly a 400 too), and the
++error cases whose behavior must not move (array-prompt 400, out-of-range
++temperature 400) plus /v1/models.
+
-A pre/post diff spanning this change prints `err_seed.json: MISSING on one
-side` and `seed_accepted.json: MISSING on one side`, then exits 1: the
-rename means the two captures carry different keys for this one case, so
-`diff()` cannot pair them under either name. That pair is expected across
-this change and is the only expected difference.
++A pre/post diff spanning these changes prints four MISSING-on-one-side lines
++and exits 1: `err_seed.json` / `seed_accepted.json` for the seed rename, and
++`err_logprobs.json` / `logprobs_served.json` for the logprobs one. Each rename
++means the two captures carry different keys for that one case, so `diff()`
++cannot pair them under either name. Those two pairs are expected across these
++changes and are the only expected differences.
Normalization: every "id"/"created" field (recursively, and per SSE event) is
replaced with a constant; nothing else is touched. Generation-bearing requests
@@@ -78,11 -85,11 +87,11 @@@ def battery(model)
("completions_stop", "POST", "/v1/completions",
{"model": model, "prompt": "Count: one, two,",
"max_tokens": 16, "temperature": 0, "stop": ["five"]}),
- ("err_seed", "POST", "/v1/completions",
- {"model": model, "prompt": "hello", "max_tokens": 1, "seed": 1234}),
+ ("seed_accepted", "POST", "/v1/completions",
+ {"model": model, "prompt": "hello", "max_tokens": 1, "temperature": 0, "seed": 1234}),
("err_array_prompt", "POST", "/v1/completions",
{"model": model, "prompt": [1, 2, 3], "max_tokens": 1}),
- ("err_logprobs", "POST", "/v1/completions",
+ ("logprobs_served", "POST", "/v1/completions",
{"model": model, "prompt": "hello", "max_tokens": 1, "logprobs": 1}),
("err_bad_temperature", "POST", "/v1/chat/completions",
{"model": model, "messages": chat_messages,
diff --cc c/tests/test_openai_server.py
index 4f06ed8e,dd5d3b6f..303fe30c
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@@ -15,13 -15,12 +15,14 @@@ from urllib.error import HTTPErro
from urllib.request import Request, urlopen
from pathlib import Path
+import openai_server
from openai_server import (APIError, APIHandler, APIServer, ClientCancelled,
- CONTINUATION_FAMILIES,
+ CONTINUATION_FAMILIES, _marker_cuts,
DEFAULT_CHAT_STOP_SEQUENCES, END, GenerationScheduler,
+ LOGPROBS_TOP_K_CAP, logprobs_options,
READY, Engine, InklingStreamSplit, StopFilter, ThinkingStreamSplit,
- _engine_error, _image_bytes_from_url, cap_for_arch, conversation_cache_slot, model_arch,
+ _engine_error, _image_bytes_from_url, cap_for_arch,
+ conversation_cache_slot, model_arch,
generation_options, parse_tool_calls, parse_dsv4_tool_calls,
parse_arch_tool_calls, parse_k3_tool_calls, parse_qwen38_tool_calls,
read_engine_turn, render_chat, render_chat_for_arch,
@@@ -4090,1758 -3753,475 +4256,2232 @@@ class ContextExceededMessageTest(unitte
self.assertIn("the context", text)
+ class _ShortWritingStream:
+ """A fake stdin that hands back at most `chunk` bytes per write() call,
+ forcing `_write_all` to loop -- the production pipe does this on a
+ signal landing mid-write or a full pipe buffer on a large IMAGE frame."""
+
+ def __init__(self, chunk=3):
+ self.chunk = chunk
+ self.received = bytearray()
+
+ def write(self, data):
+ piece = bytes(data)[:self.chunk]
+ self.received.extend(piece)
+ return len(piece)
+
+
+ class _ScriptedStream:
+ """A fake stdin whose write() answers each entry in `script` in turn --
+ an int number of bytes actually taken, or `None` for "took nothing" --
+ then takes everything it is offered once the script runs out."""
+
+ def __init__(self, script):
+ self.script = list(script)
+ self.received = bytearray()
+ self.calls = []
+
+ def write(self, data):
+ data = bytes(data)
+ self.calls.append(data)
+ answer = self.script.pop(0) if self.script else len(data)
+ if answer is None:
+ return None
+ taken = data[:answer]
+ self.received.extend(taken)
+ return len(taken)
+
+
+ class _CountingLock:
+ """A `threading.Lock`-alike that counts `with` acquisitions -- used to
+ pin that IMAGE and SUBMIT share exactly one `write_lock` acquisition."""
+
+ def __init__(self):
+ self._lock = threading.Lock()
+ self.acquisitions = 0
+
+ def __enter__(self):
+ self.acquisitions += 1
+ return self._lock.__enter__()
+
+ def __exit__(self, *exc_info):
+ return self._lock.__exit__(*exc_info)
+
+
+ class WriteAllTest(unittest.TestCase):
+ """`_write_all` (extracted from the inline server->engine stdin writes):
+ loop on a short write until the whole frame is sent, and fail closed --
+ never spin -- on the two shapes that are not progress.
+
+ `_write_all` is imported locally in each test method here, not at module
+ scope: it does not exist on base, and a module-level import of it would
+ make this whole test file fail to collect when overlaid on base product
+ code (the procedure the base-pin tests below rely on)."""
+
+ def test_short_writes_reassemble_to_the_full_frame(self):
+ from openai_server import _write_all
+ stream = _ShortWritingStream(chunk=3)
+ frame = b"SUBMIT 1 0 5 3 0.25 0.9\nHello\n"
+ _write_all(stream, frame, "SUBMIT")
+ self.assertEqual(bytes(stream.received), frame)
+
+ def test_a_short_first_write_still_receives_the_correct_remainder(self):
+ from openai_server import _write_all
+ # The first call must not be special-cased to the unsliced buffer:
+ # every call, including the first, offers a memoryview starting at
+ # the bytes not yet sent.
+ stream = _ScriptedStream([2])
+ _write_all(stream, b"CANCEL 42\n", "CANCEL")
+ self.assertEqual(bytes(stream.received), b"CANCEL 42\n")
+ self.assertEqual(len(stream.calls), 2) # the short first write forced a second
+
+ def test_none_return_fails_closed_instead_of_spinning(self):
+ from openai_server import _write_all
+ stream = _ScriptedStream([None])
+ with self.assertRaisesRegex(RuntimeError, "failed to write SUBMIT to the engine"):
+ _write_all(stream, b"SUBMIT 1 0 1 1 1 1\nx\n", "SUBMIT")
+
+ def test_zero_return_fails_closed_instead_of_spinning(self):
+ from openai_server import _write_all
+ stream = _ScriptedStream([0])
+ with self.assertRaisesRegex(RuntimeError, "failed to write STOP to the engine"):
+ _write_all(stream, b"STOP 1\n", "STOP")
+
+
+ class WriteFailureHTTPTest(unittest.TestCase):
+ """A checked engine-stdin write that fails must reach the client as the
+ named 500 engine_error while the response is still uncommitted, and must
+ never splice anything into a stream that has already committed -- it
+ just ends (#597 item 3's `_fail`, exercised here by a genuine broken
+ pipe on the engine's stdin, through the real `Engine`, rather than by a
+ fake that raises `RuntimeError` directly -- a real `BrokenPipeError` is
+ a `ConnectionError`, and it is exactly that unwrapped subclass that a
+ correct checked write must keep away from do_POST's client-hangup
+ handler)."""
+
+ def _serve(self, process):
+ with patch("openai_server.ARCH", "glm"), \
+ patch("openai_server.subprocess.Popen", return_value=process):
+ engine = Engine("glm", "model")
+ server = APIServer(("127.0.0.1", 0), engine, "test-model", None, 16, kv_slots=1)
+ # A short poll_interval means server.shutdown() (addCleanup, below)
+ # returns almost immediately instead of paying up to the default
+ # 0.5s poll -- these tests do no waiting of their own, so that 0.5s
+ # would be pure teardown overhead, not a wait under test.
+ thread = threading.Thread(target=server.serve_forever, args=(0.01,), daemon=True)
+ thread.start()
+ self.addCleanup(thread.join, 2)
+ self.addCleanup(server.server_close)
+ self.addCleanup(server.shutdown)
+ self.addCleanup(server.scheduler.close)
+ self.addCleanup(engine.close)
+ return server
+
+ def _raw_post(self, server, path, body):
+ """POST over a plain socket and read until the peer closes, so the
+ literal status line(s) on the wire are visible -- `urlopen` strips
+ the status line into `response.status` before handing back `.read()`,
+ so asserting on `.read()` alone cannot tell "one status line" from
+ "a second one spliced into the body"."""
+ payload = json.dumps(body).encode()
+ request = (f"POST {path} HTTP/1.1\r\nHost: 127.0.0.1\r\n"
+ f"Content-Type: application/json\r\nContent-Length: {len(payload)}\r\n"
+ f"Connection: close\r\n\r\n").encode() + payload
+ sock = socket.create_connection(("127.0.0.1", server.server_port), 2)
+ sock.sendall(request)
+ sock.settimeout(2)
+ chunks = []
+ try:
+ while True:
+ chunk = sock.recv(4096)
+ if not chunk:
+ break
+ chunks.append(chunk)
+ except socket.timeout:
+ pass
+ sock.close()
+ return b"".join(chunks)
+
+ def test_dead_engine_submit_is_a_named_500_engine_error_not_silence(self):
+ def respond(process, frame):
+ if frame.split()[0] == b"SUBMIT":
+ raise BrokenPipeError("broken pipe")
+
+ server = self._serve(FakeProcess(respond))
+ body = json.dumps({"model": "test-model", "prompt": "hi"}).encode()
+ request = Request(f"http://127.0.0.1:{server.server_port}/v1/completions",
+ data=body, headers={"Content-Type": "application/json"})
+ with self.assertRaises(HTTPError) as caught:
+ urlopen(request, timeout=2)
+ self.assertEqual(caught.exception.code, 500)
+ payload = json.load(caught.exception)
+ self.assertEqual(payload["error"]["code"], "engine_error")
+
+ def test_dead_engine_submit_is_a_named_500_on_the_chat_streaming_path(self):
+ # Same failure, but through /v1/chat/completions with stream: true --
+ # the tool-sideband/ThinkingStreamSplit wrapping chat streaming builds
+ # around engine.generate() must not swallow or reshape a RuntimeError
+ # that fires before anything is committed (nothing here ever reaches
+ # that wrapping: the SUBMIT write fails before the first ACCEPT/DATA).
+ def respond(process, frame):
+ if frame.split()[0] == b"SUBMIT":
+ raise BrokenPipeError("broken pipe")
+
+ server = self._serve(FakeProcess(respond))
+ body = json.dumps({"model": "test-model", "stream": True,
+ "messages": [{"role": "user", "content": "hi"}]}).encode()
+ request = Request(f"http://127.0.0.1:{server.server_port}/v1/chat/completions",
+ data=body, headers={"Content-Type": "application/json"})
+ with self.assertRaises(HTTPError) as caught:
+ urlopen(request, timeout=2)
+ self.assertEqual(caught.exception.code, 500)
+ payload = json.load(caught.exception)
+ self.assertEqual(payload["error"]["code"], "engine_error")
+
+ def test_uncommitted_stop_write_failure_is_a_named_500_engine_error(self):
+ # The dead-SUBMIT tests above cover the write _write_frame never gets
+ # a chance to make (the connection is refused before the first
+ # request even lands). This is the case _write_frame's own docstring
+ # is actually about: a CANCEL/STOP write that fails while the
+ # response is still uncommitted -- non-streaming, so nothing commits
+ # until the whole answer is assembled, and the STOP write happens
+ # well before that.
+ request_id = None
+
+ def respond(process, frame):
+ nonlocal request_id
+ fields = frame.split()
+ if fields[0] == b"SUBMIT":
+ request_id = fields[1]
+ process.stdout.feed(b"DATA " + request_id + b" 3\nhi \n")
+ process.stdout.feed(b"DATA " + request_id + b" 4\nSTOP\n")
+ elif fields[0] == b"STOP":
+ raise BrokenPipeError("broken pipe")
+
+ server = self._serve(FakeProcess(respond))
+ body = json.dumps({"model": "test-model", "prompt": "hi",
+ "stop": "STOP"}).encode()
+ request = Request(f"http://127.0.0.1:{server.server_port}/v1/completions",
+ data=body, headers={"Content-Type": "application/json"})
+ with self.assertRaises(HTTPError) as caught:
+ urlopen(request, timeout=2)
+ self.assertEqual(caught.exception.code, 500)
+ payload = json.load(caught.exception)
+ self.assertEqual(payload["error"]["code"], "engine_error")
+
+ def test_write_failure_reaching_the_committed_stream_ends_it_cleanly(self):
+ request_id = None
+
+ def respond(process, frame):
+ nonlocal request_id
+ fields = frame.split()
+ if fields[0] == b"SUBMIT":
+ request_id = fields[1]
+ # "hi " clears StopFilter's hold and reaches the client; the
+ # stop sequence itself never does -- feeding them as two
+ # frames pins the STOP write to the SECOND one, after the
+ # stream has already committed on the first.
+ process.stdout.feed(b"DATA " + request_id + b" 3\nhi \n")
+ process.stdout.feed(b"DATA " + request_id + b" 4\nSTOP\n")
+ elif fields[0] == b"STOP":
+ raise BrokenPipeError("broken pipe")
+
+ server = self._serve(FakeProcess(respond))
+ raw = self._raw_post(server, "/v1/completions",
+ {"model": "test-model", "prompt": "hi", "stream": True,
+ "stop": "STOP"})
+ # Exactly one status line on the whole wire -- the original 200. A
+ # response that spliced a second status line (or any framed error)
+ # into the already-committed SSE body would show up here as 2.
+ self.assertEqual(raw.count(b"HTTP/1.1"), 1)
+ self.assertIn(b"200", raw.split(b"\r\n", 1)[0])
+ # The committed token reached the client...
+ self.assertIn(b'"hi "', raw)
+ # ...and nothing else did: no error object, no terminal [DONE]
+ # (generate() never returned normally).
+ self.assertNotIn(b"engine_error", raw)
+ self.assertNotIn(b"data: [DONE]", raw)
+
+
+ class PendingMapCleanupTest(unittest.TestCase):
+ """The pending-map entry is dropped on every failed engine write --
+ SUBMIT, IMAGE, CANCEL or STOP -- including one forced by a checked
+ CANCEL/STOP write that fails closed with no `OSError` (a `None`/zero
+ return). A write that never reached the engine gets no DONE/ERROR back,
+ so nothing else would ever clear the slot; without this it sits behind
+ for `close()`/`_fail_pending` to find stale.
+
+ This is narrower than "every exit": a raise from inside a decode
+ callback (`on_text`/`on_accept`/`on_tool`/`on_echo`), or the duplicate-
+ ACCEPT guard, still leaves the entry behind, as on `dev`; that is not
+ changed here."""
+
+ def test_a_failed_cancel_write_does_not_leave_the_pending_entry_behind(self):
+ request_id = None
+
+ def respond(process, frame):
+ nonlocal request_id
+ fields = frame.split()
+ if fields[0] == b"SUBMIT":
+ request_id = fields[1]
+ process.stdout.feed(b"DATA " + request_id + b" 1\nx\n")
+ elif fields[0] == b"CANCEL":
+ raise BrokenPipeError("broken pipe")
+
+ process = FakeProcess(respond)
+ with patch("openai_server.subprocess.Popen", return_value=process):
+ engine = Engine("glm", "model")
+ with self.assertRaisesRegex(RuntimeError, "failed to write CANCEL to the engine"):
+ engine.generate("hello", 8, 0.7, 0.9, lambda _: None, cancelled=lambda: True)
+ with engine.pending_lock:
+ self.assertEqual(engine.pending, {})
+ engine.close()
+
+ def test_a_failed_stop_write_does_not_leave_the_pending_entry_behind(self):
+ request_id = None
+
+ def respond(process, frame):
+ nonlocal request_id
+ fields = frame.split()
+ if fields[0] == b"SUBMIT":
+ request_id = fields[1]
+ process.stdout.feed(b"DATA " + request_id + b" 1\nx\n")
+ elif fields[0] == b"STOP":
+ raise BrokenPipeError("broken pipe")
+
+ process = FakeProcess(respond)
+ with patch("openai_server.subprocess.Popen", return_value=process):
+ engine = Engine("glm", "model")
+ output = []
+ with self.assertRaisesRegex(RuntimeError, "failed to write STOP to the engine"):
+ engine.generate("hello", 8, 0.7, 0.9, output.append,
+ stopped=lambda: output == ["x"])
+ with engine.pending_lock:
+ self.assertEqual(engine.pending, {})
+ engine.close()
+
+ def test_a_cancel_write_that_takes_no_bytes_still_drops_the_pending_entry(self):
+ # Not an OSError: the pipe accepts the call and answers 0, the same
+ # "took nothing" shape _write_all treats as a fail-closed write. If
+ # _write_frame's cleanup only ran for OSError, this entry would
+ # survive -- the pop has to be keyed on failure of the write, not on
+ # which failure shape it took.
+ class ZeroOnCancelProcess(FakeProcess):
+ def write(self, data):
+ text = bytes(data)
+ fields = text.split()
+ if fields[:1] == [b"CANCEL"]:
+ self.writes.append(text)
+ # Base ignores write()'s return value entirely, so a bare
+ # `return 0` here would leave base's generate() waiting
+ # forever for an ERROR/DONE it never actually asked for
+ # (the CANCEL it believes it sent never reached the
+ # engine) -- a hang, not a failure, on the base-overlay
+ # procedure. Feeding the terminal frame here means base
+ # fails in seconds instead; at head the RuntimeError from
+ # the zero-byte write fires first, so this frame arrives
+ # after the pending entry is already gone and is a no-op.
+ self.stdout.feed(b"ERROR " + fields[1] + b" CANCELLED\n")
+ return 0
+ return super().write(data)
+
+ request_id = None
+
+ def respond(process, frame):
+ nonlocal request_id
+ fields = frame.split()
+ if fields[0] == b"SUBMIT":
+ request_id = fields[1]
+ process.stdout.feed(b"DATA " + request_id + b" 1\nx\n")
+
+ process = ZeroOnCancelProcess(respond)
+ with patch("openai_server.subprocess.Popen", return_value=process):
+ engine = Engine("glm", "model")
+ with self.assertRaisesRegex(RuntimeError, "failed to write CANCEL to the engine"):
+ engine.generate("hello", 8, 0.7, 0.9, lambda _: None, cancelled=lambda: True)
+ with engine.pending_lock:
+ self.assertEqual(engine.pending, {})
+ engine.close()
+
+ def test_a_failed_image_write_names_the_image_frame_and_drops_the_pending_entry(self):
+ # generate()'s outer handler wraps both the IMAGE and the SUBMIT
+ # write with one except OSError -- an OSError on the IMAGE write
+ # specifically must still name "IMAGE", not fall through to the
+ # handler's own default "SUBMIT" label, or one frame kind would
+ # report under two different names depending on failure shape
+ # (_write_all's own None/zero path already says "IMAGE"; an OSError
+ # is the likelier real failure and must match).
+ class BrokenPipeOnImageProcess(FakeProcess):
+ def write(self, data):
+ if bytes(data).split()[:1] == [b"IMAGE"]:
+ raise BrokenPipeError("broken pipe")
+ return super().write(data)
+
+ process = BrokenPipeOnImageProcess(lambda _process, _frame: None)
+ with patch("openai_server.ARCH", "glm53"), \
+ patch("openai_server.subprocess.Popen", return_value=process):
+ engine = Engine("glm53", "model")
+
+ class FakePatches:
+ def tobytes(self):
+ return bytes(range(8))
+
+ with self.assertRaisesRegex(RuntimeError, "failed to write IMAGE to the engine"):
+ engine.generate("hello", 8, 0.7, 0.9, lambda _: None, image=(FakePatches(), 2, 2))
+ with engine.pending_lock:
+ self.assertEqual(engine.pending, {})
+ engine.close()
+
+
+ class PlainRequestFrameOrderTest(unittest.TestCase):
+ """A request using none of the checked-write machinery's new surface
+ (no image, no grammar, no logprobs, no pin) must still put byte-identical
+ SUBMIT/STOP and SUBMIT/CANCEL frames on the wire, in the same order, as
+ the base module -- literals captured by running the base module's own
+ Engine.generate() against this same FakeProcess harness."""
+
+ def test_stop_flow_matches_base(self):
+ request_id = None
+
+ def respond(process, frame):
+ nonlocal request_id
+ fields = frame.split()
+ if fields[0] == b"SUBMIT":
+ request_id = fields[1]
+ process.stdout.feed(b"DATA " + request_id + b" 1\nx\n")
+ elif fields[0] == b"STOP":
+ process.stdout.feed(b"DONE " + request_id + b" STAT 1 1 0 1 2 0\n")
+
+ process = FakeProcess(respond)
+ with patch("openai_server.ARCH", "glm"), \
+ patch("openai_server.subprocess.Popen", return_value=process):
+ engine = Engine("glm", "model")
+ output = []
+ engine.generate("hello", 8, 0.7, 0.9, output.append,
+ stopped=lambda: output == ["x"])
+ engine.close()
+ self.assertEqual(process.writes, [b"SUBMIT 1 0 5 8 0.7 0.9\nhello\n", b"STOP 1\n"])
+
+ def test_cancel_flow_matches_base(self):
+ request_id = None
+
+ def respond(process, frame):
+ nonlocal request_id
+ fields = frame.split()
+ if fields[0] == b"SUBMIT":
+ request_id = fields[1]
+ process.stdout.feed(b"DATA " + request_id + b" 1\nx\n")
+ elif fields[0] == b"CANCEL":
+ process.stdout.feed(b"ERROR " + request_id + b" CANCELLED\n")
+
+ process = FakeProcess(respond)
+ with patch("openai_server.ARCH", "glm"), \
+ patch("openai_server.subprocess.Popen", return_value=process):
+ engine = Engine("glm", "model")
+ disconnected = False
+
+ def sink(text):
+ nonlocal disconnected
+ disconnected = True
+
+ with self.assertRaises(ClientCancelled):
+ engine.generate("hello", 8, 0.7, 0.9, sink, cancelled=lambda: disconnected)
+ engine.close()
+ self.assertEqual(process.writes, [b"SUBMIT 1 0 5 8 0.7 0.9\nhello\n", b"CANCEL 1\n"])
+
+ def test_image_and_submit_frames_match_base_under_one_lock_acquisition(self):
+ request_id = None
+
+ def respond(process, frame):
+ nonlocal request_id
+ fields = frame.split()
+ if fields[0] == b"SUBMIT":
+ request_id = fields[1]
+ process.stdout.feed(b"DATA " + request_id + b" 1\nx\n")
+ process.stdout.feed(b"DONE " + request_id + b" STAT 1 1 0 1 2 0\n")
+
+ process = FakeProcess(respond)
+ with patch("openai_server.ARCH", "glm53"), \
+ patch("openai_server.subprocess.Popen", return_value=process):
+ engine = Engine("glm53", "model")
+ counting_lock = _CountingLock()
+ engine.write_lock = counting_lock
+
+ blob = bytes(range(12))
+
+ class FakePatches:
+ def tobytes(self):
+ return blob
+
+ output = []
+ engine.generate("hello", 8, 0.7, 0.9, output.append, image=(FakePatches(), 2, 2))
+ engine.close()
+ self.assertEqual(process.writes, [
+ b"IMAGE 1 12 2 2\n" + blob + b"\n",
+ b"SUBMIT 1 0 5 8 0.7 0.9\nhello\n",
+ ])
+ # IMAGE must reach the wire before SUBMIT, and both under the SAME
+ # lock acquisition -- another request's IMAGE could otherwise land
+ # between this one's IMAGE and its SUBMIT.
+ self.assertEqual(counting_lock.acquisitions, 1)
+
+
+class LogprobsOptionsTest(unittest.TestCase):
+ """logprobs_options(): pure validation and translation, no HTTP and no engine."""
+
+ def test_completions_valid_integer_logprobs(self):
+ self.assertEqual(logprobs_options({"logprobs": 3}, False, True), (3, False, 3))
+ self.assertEqual(logprobs_options({"logprobs": 3, "echo": True}, False, True),
+ (3, True, 3))
+ self.assertEqual(logprobs_options({"logprobs": 1}, False, True), (1, False, 1))
+
+ def test_completions_zero_false_null_mean_no_logprobs(self):
+ # Explicit, never a truthiness accident and never a channel floored on at k=1.
+ for off in ({"logprobs": 0}, {"logprobs": False}, {"logprobs": None}, {}):
+ with self.subTest(off=off):
+ self.assertEqual(logprobs_options(off, False, True), (0, False, 0))
+ # ...and `echo: true` on top of an off `logprobs` is a named 400, not a no-op.
+ with self.assertRaises(APIError) as caught:
+ logprobs_options({"logprobs": 0, "echo": True}, False, True)
+ self.assertEqual((caught.exception.param, caught.exception.code),
+ ("echo", "unsupported_parameter"))
+
+ def test_completions_true_is_a_named_400(self):
+ # The legacy completions field is an integer count; `true` carries no count.
+ with self.assertRaises(APIError) as caught:
+ logprobs_options({"logprobs": True}, False, True)
+ self.assertEqual((caught.exception.status, caught.exception.param,
+ caught.exception.code), (400, "logprobs", "invalid_value"))
+
+ def test_completions_break_it_battery_non_integer_negative_huge(self):
+ for bad in (1.5, "5", -1, 33):
+ with self.subTest(bad=bad):
+ with self.assertRaises(APIError) as caught:
+ logprobs_options({"logprobs": bad}, False, True)
+ self.assertEqual((caught.exception.status, caught.exception.param,
+ caught.exception.code),
+ (400, "logprobs", "invalid_value"))
+
+ def test_chat_logprobs_requires_a_boolean(self):
+ with self.assertRaises(APIError) as caught:
+ logprobs_options({"logprobs": 1}, True, True)
+ self.assertEqual((caught.exception.param, caught.exception.code),
+ ("logprobs", "invalid_value"))
+
+ def test_chat_false_and_null_mean_no_logprobs(self):
+ for off in ({"logprobs": False}, {"logprobs": None}, {}):
+ with self.subTest(off=off):
+ self.assertEqual(logprobs_options(off, True, True), (0, False, 0))
+
+ def test_chat_echo_is_always_rejected(self):
+ # Chat has no echo concept at all: a named 400, not a silent ignore, whether or
+ # not logprobs was also requested.
+ for body in ({"echo": True}, {"echo": True, "logprobs": True}):
+ with self.subTest(body=body):
+ with self.assertRaises(APIError) as caught:
+ logprobs_options(body, True, True)
+ self.assertEqual(caught.exception.param, "echo")
+
+ def test_chat_top_logprobs_default_and_cap(self):
+ self.assertEqual(logprobs_options({"logprobs": True}, True, True), (1, False, 0))
+ self.assertEqual(
+ logprobs_options({"logprobs": True, "top_logprobs": 5}, True, True),
+ (5, False, 5))
+ with self.assertRaises(APIError) as caught:
+ logprobs_options({"logprobs": True, "top_logprobs": 33}, True, True)
+ self.assertEqual(caught.exception.param, "top_logprobs")
+
+ def test_cap_boundary_is_thirty_two_on_both_endpoints(self):
+ # Literal 32/33 rather than the constant plus or minus one, so the boundary stays
+ # meaningful if the constant itself ever drifts.
+ self.assertEqual(logprobs_options({"logprobs": 32}, False, True), (32, False, 32))
+ self.assertEqual(
+ logprobs_options({"logprobs": True, "top_logprobs": 32}, True, True),
+ (32, False, 32))
+ for body, chat, param in (({"logprobs": 33}, False, "logprobs"),
+ ({"logprobs": True, "top_logprobs": 33}, True,
+ "top_logprobs")):
+ with self.subTest(param=param):
+ with self.assertRaises(APIError) as caught:
+ logprobs_options(body, chat, True)
+ self.assertEqual((caught.exception.param, caught.exception.code),
+ (param, "invalid_value"))
+ self.assertEqual(LOGPROBS_TOP_K_CAP, 32)
+
+ def test_capability_gate_rejects_an_engine_that_is_not_asked_for_the_channel(self):
+ # Never a silent downgrade to "no logprobs".
+ for body, chat in (({"logprobs": 1}, False), ({"logprobs": True}, True)):
+ with self.subTest(chat=chat):
+ with self.assertRaises(APIError) as caught:
+ logprobs_options(body, chat, False)
+ self.assertEqual((caught.exception.status, caught.exception.code),
+ (400, "unsupported_parameter"))
+ # Absent or zero logprobs never reaches the capability check at all.
+ self.assertEqual(logprobs_options({}, False, False), (0, False, 0))
+ self.assertEqual(logprobs_options({"logprobs": 0}, False, False), (0, False, 0))
+
+ def test_range_error_precedes_capability_error_and_names_the_cap(self):
+ with self.assertRaises(APIError) as caught:
+ logprobs_options({"logprobs": 999}, False, False)
+ self.assertEqual(caught.exception.code, "invalid_value")
+ self.assertIn("32", caught.exception.message)
+
+ def test_chat_top_logprobs_validated_even_when_logprobs_is_off(self):
+ # A malformed `top_logprobs` is a named 400 whether or not the gate that would use
+ # it is open; a valid one with `logprobs` off stays a documented no-op.
+ for bad in ("x", -1, 999):
+ with self.subTest(bad=bad):
+ with self.assertRaises(APIError) as caught:
+ logprobs_options({"top_logprobs": bad}, True, True)
+ self.assertEqual((caught.exception.status, caught.exception.param,
+ caught.exception.code),
+ (400, "top_logprobs", "invalid_value"))
+ self.assertEqual(logprobs_options({"top_logprobs": 5}, True, True), (0, False, 0))
+
+ def test_chat_top_logprobs_null_normalizes_to_absent(self):
+ # On both sides of the `logprobs` gate, and it never itself reaches the
+ # type/range check.
+ self.assertEqual(
+ logprobs_options({"logprobs": False, "top_logprobs": None}, True, True),
+ (0, False, 0))
+ self.assertEqual(
+ logprobs_options({"logprobs": True, "top_logprobs": None}, True, True),
+ (1, False, 0))
+
+ def test_echo_null_normalizes_to_absent_both_endpoints(self):
+ # `echo: null` normalises to absent before the type check, exactly like `logprobs`
+ # and `top_logprobs`. An SDK that serialises its whole request model with nulls
+ # must not get a 400 for a field it never meant to set.
+ self.assertEqual(logprobs_options({"echo": None}, False, True), (0, False, 0))
+ self.assertEqual(logprobs_options({"echo": None}, True, True), (0, False, 0))
+
+ def test_echo_non_bool_rejected_the_same_way_on_both_endpoints(self):
+ for bad in (1, "true"):
+ for chat in (False, True):
+ with self.subTest(bad=bad, chat=chat):
+ with self.assertRaises(APIError) as caught:
+ logprobs_options({"echo": bad}, chat, True)
+ self.assertEqual((caught.exception.param, caught.exception.code),
+ ("echo", "invalid_value"))
+
+ def test_echo_refusals_stay_distinct(self):
+ # The type refusal and chat's capability refusal must not collapse into each
+ # other: a non-bool `echo` always gets the type message, and only an actual `True`
+ # on chat draws the unsupported one.
+ with self.assertRaises(APIError) as caught:
+ logprobs_options({"echo": "yes"}, True, True)
+ self.assertEqual(caught.exception.code, "invalid_value")
+ self.assertIn("must be a boolean", caught.exception.message)
+ with self.assertRaises(APIError) as caught:
+ logprobs_options({"echo": True}, True, True)
+ self.assertEqual(caught.exception.code, "unsupported_parameter")
+ self.assertIn("not supported for chat completions", caught.exception.message)
+
+ def test_completions_ignores_top_logprobs_entirely(self):
+ # A chat-only field in the OpenAI request shape; completions never reads it.
+ self.assertEqual(
+ logprobs_options({"logprobs": 2, "top_logprobs": 999}, False, True),
+ (2, False, 2))
+
+
+class LogprobsHTTPTest(unittest.TestCase):
+ """End-to-end behaviour of `logprobs`/`echo`/`top_logprobs` against a real APIServer,
+ with FakeEngine standing in for the engine subprocess."""
+
+ @classmethod
+ def setUpClass(cls):
+ cls.engine = FakeEngine()
+ cls.server = APIServer(("127.0.0.1", 0), cls.engine, "test-model", "secret", 16,
+ kv_slots=2)
+ cls.thread = threading.Thread(target=cls.server.serve_forever, args=(0.01,),
+ daemon=True)
+ cls.thread.start()
+ cls.base = f"http://127.0.0.1:{cls.server.server_port}"
+
+ @classmethod
+ def tearDownClass(cls):
+ cls.server.scheduler.close()
+ cls.server.shutdown()
+ cls.server.server_close()
+ cls.thread.join(timeout=2)
+
+ def chat(self, body):
+ return _post_chat(self.base, dict({"model": "test-model",
+ "messages": [{"role": "user", "content": "Hi"}]},
+ **body))
+
+ def completions(self, body):
+ return _post_completions(self.base, dict({"model": "test-model", "prompt": "Hi"},
+ **body))
+
+ # ---- shape -------------------------------------------------------------
+
+ def test_chat_logprobs_content_shape(self):
+ with self.chat({"logprobs": True, "top_logprobs": 2}) as response:
+ body = json.load(response)
+ choice = body["choices"][0]
+ content = choice["logprobs"]["content"]
+ self.assertEqual([entry["token"] for entry in content], ["H", "Γ©", "llo"])
+ self.assertIsNone(choice["logprobs"]["refusal"])
+ for entry in content:
+ self.assertEqual(set(entry), {"token", "logprob", "bytes", "top_logprobs"})
+ for alternative in entry["top_logprobs"]:
+ self.assertEqual(set(alternative), {"token", "logprob", "bytes"})
+ self.assertNotIn("echo", choice)
+
+ def test_chat_content_joins_back_to_the_message(self):
+ with self.chat({"logprobs": True, "top_logprobs": 1}) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertEqual("".join(entry["token"] for entry in choice["logprobs"]["content"]),
+ choice["message"]["content"])
+
+ def test_chat_top_logprobs_omitted_yields_empty_alternatives(self):
+ with self.chat({"logprobs": True}) as response:
+ content = json.load(response)["choices"][0]["logprobs"]["content"]
+ self.assertTrue(content)
+ for entry in content:
+ self.assertEqual(entry["top_logprobs"], [])
+
+ def test_the_chosen_tokens_logprob_comes_from_its_own_record(self):
+ # The label is found by exact float match against the position's own logprob, and
+ # the table is unsorted on the wire, so this can never be "whatever came first".
+ with self.chat({"logprobs": True, "top_logprobs": 2}) as response:
+ content = json.load(response)["choices"][0]["logprobs"]["content"]
+ # Against the fixture's declared per-record values, not against the alternative
+ # the entry itself labelled -- that comparison holds however the value was sourced.
+ self.assertEqual([entry["logprob"] for entry in content], [-0.2, -0.4, -0.6])
+ for entry in content:
+ own = [a for a in entry["top_logprobs"] if a["token"] == entry["token"]]
+ self.assertEqual(len(own), 1)
+ self.assertEqual(own[0]["logprob"], entry["logprob"])
+ self.assertEqual(own[0]["bytes"], entry["bytes"])
+
+ def test_the_chosen_token_is_found_by_value_not_by_table_rank(self):
+ # The table is unsorted on the wire, so the position's own token can sit anywhere
+ # in it. Labelling by rank would call the first entry the chosen token.
+ base = _spawn_test_server(self, ScriptedEngine(
+ chunks=("cat",), records=[(b"cat", -0.8, [(999, -0.05), (42, -0.8)])]))
+ with _post_chat(base, {"model": "test-model", "logprobs": True, "top_logprobs": 2,
+ "messages": [{"role": "user", "content": "hi"}]}) as response:
+ entry = json.load(response)["choices"][0]["logprobs"]["content"][0]
+ self.assertEqual([alternative["token"] for alternative in entry["top_logprobs"]],
+ ["<token_id:999>", "cat"])
+ self.assertEqual(entry["logprob"], -0.8)
+
+ def test_a_tie_on_the_printed_value_is_broken_deterministically(self):
+ # The engine prints six decimal digits, so two candidates can share the chosen
+ # token's printed value. Only the first match in wire order is labelled as the
+ # chosen token; a later one is labelled by its id like any other candidate, which
+ # is also what keeps the two labels distinct.
+ base = _spawn_test_server(self, ScriptedEngine(
+ chunks=("cat",),
+ records=[(b"cat", -0.223144, [(1, -0.223144), (2, -0.223144)])]))
+ with _post_chat(base, {"model": "test-model", "logprobs": True, "top_logprobs": 2,
+ "messages": [{"role": "user", "content": "hi"}]}) as response:
+ entry = json.load(response)["choices"][0]["logprobs"]["content"][0]
+ self.assertEqual([alternative["token"] for alternative in entry["top_logprobs"]],
+ ["cat", "<token_id:2>"])
+
+ def test_legacy_token_logprobs_come_from_the_records_own_value(self):
+ # The completions twin of the chat test above: the table is unsorted on the wire,
+ # so `token_logprobs` must be the record's own `lp`, never the first table entry.
+ base = _spawn_test_server(self, ScriptedEngine(
+ chunks=("cat",), records=[(b"cat", -0.8, [(999, -0.05), (42, -0.8)])]))
+ with _post_completions(base, {"model": "test-model", "prompt": "hi",
+ "logprobs": 2, "max_tokens": 1}) as response:
+ logprobs = json.load(response)["choices"][0]["logprobs"]
+ self.assertEqual(logprobs["tokens"], ["cat"])
+ self.assertEqual(logprobs["token_logprobs"], [-0.8])
+ self.assertEqual(logprobs["top_logprobs"], [{"<token_id:999>": -0.05, "cat": -0.8}])
+
+ def test_completions_logprobs_object_shape_and_text_offsets(self):
+ with self.completions({"logprobs": 2}) as response:
+ choice = json.load(response)["choices"][0]
+ logprobs = choice["logprobs"]
+ self.assertEqual(set(logprobs),
+ {"tokens", "token_logprobs", "top_logprobs", "text_offset"})
+ self.assertEqual(logprobs["tokens"], ["H", "Γ©", "llo"])
+ # Characters into `text` from 0, not bytes: "Γ©" is one character and two bytes.
+ self.assertEqual(logprobs["text_offset"], [0, 1, 2])
+ self.assertEqual("".join(logprobs["tokens"]), choice["text"])
+
+ def test_echo_reconstructs_the_prompt_and_counts_offsets_from_zero(self):
+ with self.completions({"logprobs": 2, "echo": True}) as response:
+ choice = json.load(response)["choices"][0]
+ logprobs = choice["logprobs"]
+ self.assertEqual(logprobs["tokens"], ["H", "Γ©", "H", "Γ©", "llo"])
+ self.assertEqual(logprobs["text_offset"], [0, 1, 2, 3, 4])
+ self.assertEqual("".join(logprobs["tokens"]), choice["text"])
+ # The engine's own echo position 0 carries a non-finite sentinel: there is nothing
+ # to condition the first prompt token on.
+ self.assertIsNone(logprobs["token_logprobs"][0])
+
+ def test_nan_logprob_serializes_as_json_null_over_the_wire(self):
+ # Not the invalid-JSON NaN literal, and not clamped to a made-up finite number.
+ with self.completions({"logprobs": 1, "echo": True}) as response:
+ raw = response.read().decode()
+ self.assertNotIn("NaN", raw)
+ self.assertIsNone(json.loads(raw)["choices"][0]["logprobs"]["token_logprobs"][0])
+
+ # ---- normalisation and refusals ----------------------------------------
+
+ def test_zero_false_null_normalize_to_absent(self):
+ # Each endpoint's own shape: `0` is a completions value, `false`/`null` are
+ # accepted on both, and none of them is ever an error.
+ for field, value in (("logprobs", 0), ("logprobs", False), ("logprobs", None),
+ ("echo", False), ("echo", None)):
+ with self.subTest(endpoint="completions", field=field, value=value):
+ with self.completions({field: value}) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertIsNone(choice["logprobs"])
+ self.assertEqual(choice["text"], "HΓ©llo")
+ for field, value in (("logprobs", False), ("logprobs", None), ("echo", None),
+ ("top_logprobs", None), ("top_logprobs", 0)):
+ with self.subTest(endpoint="chat", field=field, value=value):
+ with self.chat({field: value}) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertIsNone(choice["logprobs"])
+
+ def test_echo_without_logprobs_is_refused_by_name(self):
+ # The prompt echo is built out of the engine's per-token records, so without them
+ # there is nothing to echo. A 200 that quietly drops a requested field is the shape
+ # this surface exists to stop.
+ for body in ({"echo": True}, {"echo": True, "logprobs": 0},
+ {"echo": True, "logprobs": None}):
+ with self.subTest(body=body):
+ status, error = _error_body(self, lambda: self.completions(body))
+ self.assertEqual(status, 400)
+ self.assertEqual(error["param"], "echo")
+ self.assertEqual(error["code"], "unsupported_parameter")
+
+ def test_echo_false_and_null_stay_absent_without_logprobs(self):
+ # The control: only `true` needs a channel, and the two absent spellings must keep
+ # serving the request they always served.
+ for value in (False, None):
+ with self.subTest(value=value):
+ with self.completions({"echo": value}) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertIsNone(choice["logprobs"])
+ self.assertEqual(choice["text"], "HΓ©llo")
+
+ def test_each_endpoint_refuses_the_others_request_shape_by_name(self):
+ for post, body, param in ((self.completions, {"logprobs": True}, "logprobs"),
+ (self.chat, {"logprobs": 2}, "logprobs"),
+ (self.chat, {"echo": True}, "echo")):
+ with self.subTest(body=body):
+ status, error = _error_body(self, lambda: post(body))
+ self.assertEqual(status, 400)
+ self.assertEqual(error["param"], param)
+
+ def test_break_it_logprobs_out_of_range(self):
+ status, error = _error_body(self, lambda: self.completions({"logprobs": 99}))
+ self.assertEqual(status, 400)
+ self.assertEqual(error["code"], "invalid_value")
+
+ def test_break_it_streaming_plus_logprobs_is_named_400(self):
+ for post, body in ((self.completions, {"logprobs": 1, "stream": True}),
+ (self.chat, {"logprobs": True, "stream": True})):
+ with self.subTest(body=body):
+ status, error = _error_body(self, lambda: post(body))
+ self.assertEqual(status, 400)
+ self.assertEqual(error["param"], "logprobs")
+ self.assertEqual(error["code"], "unsupported_parameter")
+
+ def test_chat_echo_false_normalises_to_absent_over_http(self):
+ # The completions side of this is covered; chat is the half that was only true by
+ # construction. `false` is not `true`, so it never reaches chat's echo refusal:
+ # the request is served exactly as one that never named the field.
+ with self.chat({"echo": False}) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertIsNone(choice["logprobs"])
+ self.assertEqual(choice["message"]["content"], "HΓ©llo")
+ with self.chat({"echo": False, "logprobs": True, "top_logprobs": 1}) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertEqual([entry["token"] for entry in choice["logprobs"]["content"]],
+ ["H", "Γ©", "llo"])
+
+ def test_a_continued_turn_is_prompt_and_the_arrays_describe_only_what_was_generated(self):
+ """A trailing assistant turn is prefill, not output: the prompt ends inside it. The
+ text the client sent must therefore reach the engine as prompt and must not appear
+ in the arrays, which describe the generated text alone.
+
+ `enable_thinking` is on because that is the axis where the two features meet. A
+ continued turn opens no reasoning block, so the split starts in text mode -- and the
+ span map the arrays are projected through comes from that same split. Primed the
+ other way the split files the whole answer as reasoning, which leaves `content`
+ empty and the arrays describing a string that is no longer there."""
+ opening = "The capital of France is Par"
+ with self.chat({"logprobs": True, "top_logprobs": 2, "enable_thinking": True,
+ "messages": [{"role": "user", "content": "Where is it?"},
+ {"role": "assistant", "content": opening}]}) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertTrue(self.engine.calls[-1][0].endswith(opening),
+ self.engine.calls[-1][0][-60:])
+ self.assertEqual(choice["message"]["content"], "HΓ©llo")
+ self.assertNotIn("reasoning_content", choice["message"])
+ tokens = [entry["token"] for entry in choice["logprobs"]["content"]]
+ self.assertEqual(tokens, ["H", "Γ©", "llo"])
+ self.assertEqual("".join(tokens), choice["message"]["content"])
+
+ def test_a_continued_turn_does_not_give_chat_an_echo(self):
+ # `echo` is the legacy completions shape and chat refuses it by name. A continued
+ # turn puts client text in the prompt, which is the one thing that could look like
+ # a reason to answer one here; it is not, and the refusal is unchanged.
+ status, error = _error_body(self, lambda: _post_chat(self.base, {
+ "model": "test-model", "logprobs": True, "echo": True,
+ "messages": [{"role": "user", "content": "Where is it?"},
+ {"role": "assistant", "content": "The capital of France is Par"}]}))
+ self.assertEqual(status, 400)
+ self.assertEqual(error["param"], "echo")
+
+ def test_the_capability_refusal_names_the_gate_not_the_engines_arch(self):
+ # The refusal reads the flag it failed, not the module's ARCH: in production the
+ # two agree, so naming the arch would be a guess that happens to be right, and
+ # under the test module's own ARCH ("glm") it would read "not supported by the glm
+ # engine" for an engine whose flag is off -- which is the wrong sentence.
+ base = _spawn_test_server(self, ScriptedEngine(supports_logprobs_echo=False))
+ status, error = _error_body(self, lambda: _post_completions(base, {
+ "model": "test-model", "prompt": "hi", "logprobs": 1}))
+ self.assertEqual(status, 400)
+ self.assertEqual(
+ error["message"],
+ "Log probabilities are not requested from this engine by these endpoints.")
+ self.assertNotIn(openai_server.ARCH, error["message"])
+ self.assertNotIn("glm", error["message"])
+
+ def test_break_it_logprobs_rejected_for_non_glm_engine(self):
+ # A named 400, never a silent no-op, on an engine these endpoints do not request
+ # the channel from.
+ base = _spawn_test_server(self, ScriptedEngine(supports_logprobs_echo=False))
+ for path, body in (("/v1/completions", {"model": "test-model", "prompt": "hi",
+ "logprobs": 1}),
+ ("/v1/chat/completions",
+ {"model": "test-model", "logprobs": True,
+ "messages": [{"role": "user", "content": "hi"}]})):
+ with self.subTest(path=path):
+ status, error = _error_body(self, lambda: _post_json(base, path, body))
+ self.assertEqual(status, 400)
+ self.assertEqual(error["param"], "logprobs")
+ self.assertEqual(error["code"], "unsupported_parameter")
+
+ def test_a_plain_request_never_asks_the_engine_for_the_channel(self):
+ with self.completions({}) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertIsNone(choice["logprobs"])
+ self.assertEqual(self.engine.last_logprobs, 0)
+ self.assertFalse(self.engine.last_echo)
+ self.assertFalse(self.engine.last_gbytes)
+
+ def test_the_echo_sink_is_passed_only_when_echo_was_asked_for(self):
+ # The engine sends an ECHO frame for every prompt position of every opted-in
+ # request, so passing a sink is the retention opt-in.
+ with self.completions({"logprobs": 1}) as response:
+ response.read()
+ self.assertEqual(self.engine.last_logprobs, 1)
+ self.assertFalse(self.engine.last_echo)
+ with self.completions({"logprobs": 1, "echo": True}) as response:
+ response.read()
+ self.assertTrue(self.engine.last_echo)
+
+
+class LogprobsRawStreamAlignmentTest(unittest.TestCase):
+ """The arrays describe the characters the client received, exactly. Every expectation
+ here is written from the engine's own frames and the stop policy, and each test names
+ inline what a build without the span layer returns for it."""
+
+ def _chat(self, base, **extra):
+ body = {"model": "test-model", "max_tokens": 8, "logprobs": True,
+ "top_logprobs": 1, "messages": [{"role": "user", "content": "hi"}]}
+ body.update(extra)
+ with _post_chat(base, body) as response:
+ return json.load(response)["choices"][0]
+
+ def test_swallowed_leading_marker_keeps_the_other_two_records(self):
+ # WITHOUT THE SPAN LAYER: logprobs.content == [] with a 200 and a full message.content. Two of
+ # three, not three: the role marker was genuinely not emitted, so an entry for it
+ # would describe a character the client never received.
+ choice = self._chat(_spawn_test_server(self, _ignored_leading_marker_engine()))
+ self.assertEqual(choice["message"]["content"], "Hello world")
+ entries = choice["logprobs"]["content"]
+ self.assertEqual([entry["token"] for entry in entries], ["Hello", " world"])
+ self.assertEqual([entry["logprob"] for entry in entries],
+ [-0.2, -0.30000000000000004])
+ self.assertEqual("".join(entry["token"] for entry in entries),
+ choice["message"]["content"])
+
+ def test_swallowed_leading_marker_offsets_index_the_returned_text(self):
+ # The legacy shape's arm of the same defect: "Hello" at 0 and " world" at 5, both
+ # indexing "Hello world".
+ base = _spawn_test_server(self, _ignored_leading_marker_engine())
+ with _post_completions(base, {"model": "test-model", "prompt": "hi",
+ "max_tokens": 8, "logprobs": 1,
+ "stop": ["<|user|>"],
+ "x_colibri_ignore_leading_stop": True}) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertEqual(choice["text"], "Hello world")
+ self.assertEqual(choice["logprobs"]["tokens"], ["Hello", " world"])
+ self.assertEqual(choice["logprobs"]["text_offset"], [0, 5])
+ self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])
+
+ def test_a_record_the_stop_cut_in_half_keeps_the_emitted_characters(self):
+ # WITHOUT THE SPAN LAYER: text "Hel" with tokens []. The boundary entry carries the emitted
+ # span, so `tokens` is ["Hel"] -- not ["Hello"], and not [].
+ base = _spawn_test_server(self, _mid_token_stop_echo_engine())
+ with _post_completions(base, {"model": "test-model", "prompt": "prompt",
+ "max_tokens": 4, "logprobs": 1,
+ "stop": ["lo"]}) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertEqual(choice["text"], "Hel")
+ self.assertEqual(choice["logprobs"]["tokens"], ["Hel"])
+ self.assertEqual(choice["logprobs"]["text_offset"], [0])
+ self.assertEqual(choice["logprobs"]["token_logprobs"], [-0.5])
+ self.assertEqual(choice["finish_reason"], "stop")
+ self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])
+
+ def test_echo_keeps_the_prompt_and_the_emitted_prefix(self):
+ # WITHOUT THE SPAN LAYER: text collapses to "prompt" -- the "Hel" the client earned is deleted
+ # by the echo rebuild.
+ base = _spawn_test_server(self, _mid_token_stop_echo_engine())
+ with _post_completions(base, {"model": "test-model", "prompt": "prompt",
+ "max_tokens": 4, "logprobs": 1, "echo": True,
+ "stop": ["lo"]}) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertEqual(choice["text"], "promptHel")
+ self.assertEqual(choice["logprobs"]["tokens"], ["pr", "ompt", "Hel"])
+ self.assertEqual(choice["logprobs"]["text_offset"], [0, 2, 6])
+ self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])
+
+ def test_a_truncated_entrys_bytes_do_not_describe_the_withheld_text(self):
+ # `bytes` is `token`'s sibling and is held to the same rule: the record's payload
+ # decodes to "Hello", so leaving it whole would hand the client the "lo" the stop
+ # withheld, one field over.
+ choice = self._chat(_spawn_test_server(self, _mid_token_stop_echo_engine()),
+ max_tokens=4, stop=["lo"])
+ self.assertEqual(choice["message"]["content"], "Hel")
+ entry = choice["logprobs"]["content"][0]
+ self.assertEqual(entry["token"], "Hel")
+ self.assertEqual(entry["bytes"], [72, 101, 108])
+ self.assertEqual(bytes(entry["bytes"]).decode("utf-8"), entry["token"])
+ for alternative in entry["top_logprobs"]:
+ if alternative["bytes"] is not None:
+ self.assertEqual(bytes(alternative["bytes"]).decode("utf-8"),
+ alternative["token"])
+
+ def test_a_matched_stop_drops_its_own_record_and_every_later_one(self):
+ base = _spawn_test_server(self, _stop_token_engine())
+ with _post_completions(base, {"model": "test-model", "prompt": "hi",
+ "max_tokens": 8, "stop": ["STOP"],
+ "logprobs": 1}) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertEqual(choice["text"], "ok ")
+ self.assertEqual(choice["logprobs"]["tokens"], ["ok "])
+ self.assertEqual(len(choice["logprobs"]["token_logprobs"]), 1)
+ self.assertEqual(len(choice["logprobs"]["top_logprobs"]), 1)
+
+ def test_a_frame_inside_suppressed_text_leaves_no_phantom_entry(self):
+ # A frame that decodes to "" owns no span, so "was it emitted?" has to be asked of
+ # the character its bytes went into, not of its position, which sits on the
+ # boundary of the emitted region and tests true at both ends. A phantom entry here
+ # carries a byte of the suppressed stop sequence out in the one field that is
+ # bytes.
+ base = _spawn_test_server(self, _suppressed_stop_byte_engine())
+ with _post_completions(base, {"model": "test-model", "prompt": "hi",
+ "max_tokens": 8, "logprobs": 1,
+ "stop": ["Γ©"]}) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertEqual(choice["text"], "ok ")
+ self.assertEqual(choice["logprobs"]["tokens"], ["ok "])
+ self.assertEqual(choice["logprobs"]["text_offset"], [0])
+ self.assertEqual(choice["logprobs"]["token_logprobs"], [-0.1])
+
+ def test_a_frame_that_completes_nothing_at_the_seam_reports_nothing(self):
+ # Both rows of the two-frame seam case. Frame one's bytes finish no character, so
+ # its token is "": reporting the raw stream's replacement character instead puts a
+ # U+FFFD in front of the real one and shifts every later offset.
+ for stop, text, tokens, offsets in (
+ (None, "A€Hello", ["A", "", "", "€Hello"], [0, 1, 1, 1]),
+ (["lo"], "A€Hel", ["A", "", "", "€Hel"], [0, 1, 1, 1])):
+ with self.subTest(stop=stop):
+ base = _spawn_test_server(self, _seam_split_across_two_frames_engine())
+ body = {"model": "test-model", "prompt": "A€", "max_tokens": 8,
+ "logprobs": 1, "echo": True}
+ if stop is not None:
+ body["stop"] = stop
+ with _post_completions(base, body) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertEqual(choice["text"], text)
+ self.assertNotIn("οΏ½", choice["text"])
+ self.assertEqual(choice["logprobs"]["tokens"], tokens)
+ self.assertEqual(choice["logprobs"]["text_offset"], offsets)
+ self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])
+
+ def test_seam_codepoint_and_stop_on_the_same_record(self):
+ # The raw-stream decode opens with two replacement characters where the seam
+ # decode has one Euro sign; the stop cuts past that head, so the real character
+ # splices back on and the withheld "lo" stays withheld.
+ base = _spawn_test_server(self, _seam_split_same_record_stop_engine())
+ with _post_completions(base, {"model": "test-model", "prompt": "A€",
+ "max_tokens": 4, "logprobs": 1, "echo": True,
+ "stop": ["lo"]}) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertEqual(choice["text"], "A€Hel")
+ self.assertNotIn("οΏ½", choice["text"])
+ self.assertEqual(choice["logprobs"]["tokens"], ["A", "", "€Hel"])
+ self.assertEqual(choice["logprobs"]["text_offset"], [0, 1, 1])
+ self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])
+
+ def test_seam_codepoint_survives_a_later_stop_truncation(self):
+ # The Euro sign is split across the prompt/generated seam and must decode as one
+ # character while a later frame is cut by the stop. Four positions for four
+ # frames: the echo position carrying the lead byte resolves nothing on its own and
+ # reports "", and the character lands on the frame that completed it.
+ base = _spawn_test_server(self, _seam_split_stop_engine())
+ with _post_completions(base, {"model": "test-model", "prompt": "A€",
+ "max_tokens": 4, "logprobs": 1, "echo": True,
+ "stop": ["STOP"]}) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertEqual(choice["text"], "A€x")
+ self.assertEqual(choice["text"].count("€"), 1)
+ self.assertNotIn("οΏ½", choice["text"])
+ self.assertEqual(choice["logprobs"]["tokens"], ["A", "", "€", "x"])
+ self.assertEqual(choice["logprobs"]["text_offset"], [0, 1, 1, 2])
+ self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])
+
+ def test_an_engine_that_sends_no_records_is_refused_not_half_answered(self):
+ # The opt-in was accepted and the channel came back empty. Answering with the
+ # completion and an empty array is the shape the arrays exist to prevent.
+ base = _spawn_test_server(self, ScriptedEngine(
+ chunks=("AA", "BB"), records=None,
+ echoes=[(b"P", NAN, []), (b"Q", -0.1, [(1, -0.1)])]))
+ status, error = _error_body(self, lambda: _post_completions(base, {
+ "model": "test-model", "prompt": "PQ", "max_tokens": 4, "logprobs": 1,
+ "echo": True}))
+ self.assertEqual(status, 500)
+ self.assertEqual(error["code"], "engine_logprob_records_incomplete")
+
+ def test_a_length_limited_turn_reports_finish_reason_length(self):
+ base = _spawn_test_server(self, ScriptedEngine(chunks=("ok",),
+ length_limited=True))
+ with _post_completions(base, {"model": "test-model", "prompt": "hi",
+ "max_tokens": 1, "logprobs": 1}) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertEqual(choice["finish_reason"], "length")
+ self.assertEqual(choice["logprobs"]["tokens"], ["ok"])
+
+ def test_every_response_joins_its_tokens_back_to_the_text_it_returned(self):
+ # The invariant as a test rather than a comment, over every fixture in this file
+ # that can produce a boundary.
+ for name, build in (("leading marker", _ignored_leading_marker_engine),
+ ("mid-token stop", _mid_token_stop_echo_engine),
+ ("stop token", _stop_token_engine),
+ ("seam and stop", _seam_split_stop_engine)):
+ for echo in (False, True):
+ with self.subTest(engine=name, echo=echo):
+ base = _spawn_test_server(self, build())
+ with _post_completions(base, {
+ "model": "test-model", "prompt": "A€", "max_tokens": 8,
+ "logprobs": 1, "echo": echo,
+ "stop": ["lo", "STOP", "<|user|>"]}) as response:
+ choice = json.load(response)["choices"][0]
+ tokens = choice["logprobs"]["tokens"]
+ offsets = choice["logprobs"]["text_offset"]
+ self.assertEqual("".join(tokens), choice["text"])
+ self.assertEqual(offsets, sorted(offsets))
+ for offset in offsets:
+ self.assertLessEqual(offset, len(choice["text"]))
+
+
+class PlainPathSpanWorkTest(unittest.TestCase):
+ """What a request that never asked for per-token logprobs actually pays for the span
+ layer, stage by stage. Counted at a seam -- the primitives each stage would call --
+ rather than against a timing or a golden number, so the assertion is about work done.
+
+ The thinking split builds nothing. The tool-call stage composes and exports nothing,
+ which is the part that was worth gating; it still walks its own cut lists, because
+ those ARE the deletion the content is sliced from rather than a map kept for a later
+ reader. The exact residue is asserted below rather than described, so it cannot drift
+ upward unnoticed."""
+
+ def _counted(self, name):
+ """Replace one span primitive with a counting pass-through for the duration of a
+ request, and return the list its calls are recorded in."""
+ calls = []
+ original = getattr(openai_server, name)
+
+ def counting(*args, **kwargs):
+ calls.append(1)
+ return original(*args, **kwargs)
+
+ patcher = patch.object(openai_server, name, counting)
+ patcher.start()
+ self.addCleanup(patcher.stop)
+ return calls
+
+ def _chat(self, engine, **extra):
+ base = _spawn_test_server(self, engine)
+ body = {"model": "test-model", "max_tokens": 16,
+ "messages": [{"role": "user", "content": "hi"}]}
+ body.update(extra)
+ with _post_chat(base, body) as response:
+ return json.load(response)["choices"][0]
+
+ THINKING = ScriptedEngine(
+ chunks=("<think>", "why", "</think>", "the ", "answer"),
+ records=[(b"<think>", -0.1, [(1, -0.1)]), (b"why", -0.2, [(2, -0.2)]),
+ (b"</think>", -0.3, [(3, -0.3)]), (b"the ", -0.4, [(4, -0.4)]),
+ (b"answer", -0.5, [(5, -0.5)])])
+
+ TOOLS = [{"type": "function",
+ "function": {"name": "search",
+ "parameters": {"type": "object",
+ "properties": {"q": {"type": "string"}}}}}]
+
+ def test_the_thinking_split_records_nothing_on_a_plain_request(self):
+ # No tools, so the thinking split is the only stage that could append a span.
+ appends = self._counted("_append_span")
+ choice = self._chat(self.THINKING, enable_thinking=True)
+ self.assertEqual(choice["message"]["content"], "the answer")
+ self.assertEqual(choice["message"]["reasoning_content"], "why")
+ self.assertEqual(sum(appends), 0)
+
+ def test_the_thinking_split_records_when_logprobs_are_asked_for(self):
+ # The control: the same request with the opt-in builds the map it needs, so the
+ # test above cannot be satisfied by a stage that never records at all.
+ appends = self._counted("_append_span")
+ choice = self._chat(self.THINKING, enable_thinking=True, logprobs=True,
+ top_logprobs=1)
+ self.assertEqual(choice["message"]["content"], "the answer")
+ self.assertGreater(sum(appends), 0)
+
+ def test_the_tool_call_parse_composes_and_exports_nothing_on_a_plain_request(self):
+ # The composition and the exported maps are what the gate removes. The per-step
+ # cut lists remain: `_apply_cuts` returns the survivors the content is built by
+ # slicing, so removing them would mean a second implementation of the deletion.
+ # Every count is asserted, including the ones that are not zero, so "no span map"
+ # can never quietly become "some span map".
+ engine = ScriptedEngine(
+ chunks=("before ", "<tool_call>search<arg_key>q</arg_key>"
+ "<arg_value>x</arg_value></tool_call>", " after"))
+ counts = {name: self._counted(name) for name in
+ ("_compose_span_maps", "_append_span", "_cut_span_map", "_apply_cuts",
+ "_marker_cuts", "_project_span")}
+ choice = self._chat(engine, tools=self.TOOLS)
+ self.assertEqual(choice["message"]["content"], "before after")
+ self.assertEqual(len(choice["message"]["tool_calls"]), 1)
+ self.assertEqual({name: sum(calls) for name, calls in counts.items()},
+ {"_compose_span_maps": 0, # nothing composed
+ "_project_span": 0, # nothing read back
+ "_append_span": 5, # inside the cut lists below
+ "_cut_span_map": 4,
+ "_apply_cuts": 4,
+ "_marker_cuts": 2})
+ # ...and no map leaves the stage for anyone to read.
+ _content, _calls, box_map, content_map = openai_server._parse_tool_calls(
+ "before <tool_call>search</tool_call> after", self.TOOLS, False)
+ self.assertIsNone(box_map)
+ self.assertIsNone(content_map)
+
+ def test_the_tool_call_parse_composes_when_logprobs_are_asked_for(self):
+ engine = ScriptedEngine(
+ chunks=("before ", "<tool_call>search<arg_key>q</arg_key>"
+ "<arg_value>x</arg_value></tool_call>", " after"),
+ records=[(b"before ", -0.1, [(1, -0.1)]),
+ ("<tool_call>search<arg_key>q</arg_key>"
+ "<arg_value>x</arg_value></tool_call>".encode(), -0.2, [(2, -0.2)]),
+ (b" after", -0.3, [(3, -0.3)])])
+ composes = self._counted("_compose_span_maps")
+ choice = self._chat(engine, tools=self.TOOLS, logprobs=True, top_logprobs=1)
+ self.assertEqual(choice["message"]["content"], "before after")
+ self.assertGreater(sum(composes), 0)
+
+
+class LogprobsStageCompositionTest(unittest.TestCase):
+ """The chat path runs the stop filter, then the thinking split, then the tool-call
+ parse. `logprobs.content` must describe `message.content` after all three."""
+
+ THINKING = ScriptedEngine(
+ chunks=("<think>", "why", "</think>", "the ", "answer"),
+ records=[(b"<think>", -0.1, [(1, -0.1)]),
+ (b"why", -0.2, [(2, -0.2)]),
+ (b"</think>", -0.3, [(3, -0.3)]),
+ (b"the ", -0.4, [(4, -0.4)]),
+ (b"answer", -0.5, [(5, -0.5)])])
+
+ def _chat(self, engine, **extra):
+ base = _spawn_test_server(self, engine)
+ body = {"model": "test-model", "max_tokens": 16, "logprobs": True,
+ "top_logprobs": 1, "messages": [{"role": "user", "content": "hi"}]}
+ body.update(extra)
+ with _post_chat(base, body) as response:
+ return json.load(response)["choices"][0]
+
+ def test_content_is_a_projection_of_the_stream_not_a_subset(self):
+ # Cardinality first, then absence: a build that returns an empty array satisfies
+ # every "X is not described" assertion trivially, so the count has to be asserted
+ # before anything about what is missing.
+ choice = self._chat(self.THINKING, enable_thinking=True)
+ entries = choice["logprobs"]["content"]
+ self.assertEqual(choice["message"]["content"], "the answer")
+ self.assertEqual(choice["message"]["reasoning_content"], "why")
+ self.assertEqual(len(entries), 2)
+ self.assertEqual([entry["token"] for entry in entries], ["the ", "answer"])
+ # Each entry's logprob is its OWN record's, not the one at the same index in the
+ # raw stream: a chain composed one stage short keeps the count and the text and
+ # still attaches the reasoning tokens' numbers.
+ self.assertEqual([entry["logprob"] for entry in entries], [-0.4, -0.5])
+ self.assertEqual("".join(entry["token"] for entry in entries),
+ choice["message"]["content"])
+ # ...and the reasoning and the markers are not described as content.
+ self.assertNotIn("why", [entry["token"] for entry in entries])
+
+ def test_tool_call_syntax_is_not_described_as_content(self):
+ engine = ScriptedEngine(
+ chunks=("before ", "<tool_call>search<arg_key>q</arg_key>"
+ "<arg_value>x</arg_value></tool_call>", " after"),
+ records=[(b"before ", -0.1, [(1, -0.1)]),
+ ("<tool_call>search<arg_key>q</arg_key>"
+ "<arg_value>x</arg_value></tool_call>".encode(), -0.2, [(2, -0.2)]),
+ (b" after", -0.3, [(3, -0.3)])])
+ choice = self._chat(engine, tools=[{
+ "type": "function",
+ "function": {"name": "search",
+ "parameters": {"type": "object",
+ "properties": {"q": {"type": "string"}}}}}])
+ entries = choice["logprobs"]["content"]
+ self.assertEqual(choice["message"]["content"], "before after")
+ self.assertEqual(len(choice["message"]["tool_calls"]), 1)
+ self.assertEqual(len(entries), 2)
+ self.assertEqual([entry["token"] for entry in entries], ["before ", " after"])
+ self.assertEqual([entry["logprob"] for entry in entries], [-0.1, -0.3])
+ self.assertEqual("".join(entry["token"] for entry in entries),
+ choice["message"]["content"])
+
+ def test_all_three_stages_compose_in_one_turn(self):
+ # Thinking and tools together: the stop filter, the thinking split and the
+ # tool-call parse each cut something, and the chain has to be composed in that
+ # order. Composed the other way round, or stopped one stage early, the entries
+ # stop joining back to `message.content`.
+ engine = ScriptedEngine(
+ chunks=("<think>", "why", "</think>", "before ",
+ "<tool_call>search<arg_key>q</arg_key>"
+ "<arg_value>x</arg_value></tool_call>", " after"),
+ records=[(b"<think>", -0.1, [(1, -0.1)]),
+ (b"why", -0.2, [(2, -0.2)]),
+ (b"</think>", -0.3, [(3, -0.3)]),
+ (b"before ", -0.4, [(4, -0.4)]),
+ ("<tool_call>search<arg_key>q</arg_key>"
+ "<arg_value>x</arg_value></tool_call>".encode(), -0.5, [(5, -0.5)]),
+ (b" after", -0.6, [(6, -0.6)])])
+ choice = self._chat(engine, enable_thinking=True, tools=[{
+ "type": "function",
+ "function": {"name": "search",
+ "parameters": {"type": "object",
+ "properties": {"q": {"type": "string"}}}}}])
+ entries = choice["logprobs"]["content"]
+ self.assertEqual(choice["message"]["reasoning_content"], "why")
+ self.assertEqual(choice["message"]["content"], "before after")
+ self.assertEqual(len(choice["message"]["tool_calls"]), 1)
+ self.assertEqual(len(entries), 2)
+ self.assertEqual([entry["token"] for entry in entries], ["before ", " after"])
+ self.assertEqual([entry["logprob"] for entry in entries], [-0.4, -0.6])
+ self.assertEqual("".join(entry["token"] for entry in entries),
+ choice["message"]["content"])
+
+ def test_a_plain_chat_turn_describes_every_token(self):
+ # The control: with no stage cutting anything, nothing is dropped.
+ choice = self._chat(ScriptedEngine(chunks=("the ", "answer")))
+ entries = choice["logprobs"]["content"]
+ self.assertEqual(len(entries), 2)
+ self.assertEqual("".join(entry["token"] for entry in entries),
+ choice["message"]["content"])
+
+
+class EchoCoverageTest(unittest.TestCase):
+ """`echo: true` rebuilds `text` from the logprob records, because `text` and
+ `text_offset` have to describe the same reconstruction. If the records stop short of
+ the generated stream that rebuild silently drops the uncovered tail and the client gets
+ a truncated completion with a 200 -- the worst shape a fault here can take, because
+ nothing in the response says anything is missing. It is refused by name instead."""
+
+ BODY = {"model": "test-model", "prompt": "PQ", "echo": True,
+ "logprobs": 1, "max_tokens": 3}
+
+ def _post(self, engine, body):
+ return _post_completions(_spawn_test_server(self, engine), body)
+
+ def test_records_short_of_the_stream_are_refused_not_truncated(self):
+ # Unrefused, this is a 200 carrying "PQAA" with the client's "BB" and "CC" gone.
+ status, error = _error_body(
+ self, lambda: self._post(_partial_coverage_engine(covered=1), dict(self.BODY)))
+ self.assertEqual(status, 500)
+ self.assertEqual(error["code"], "engine_logprob_records_incomplete")
+
+ def test_full_coverage_is_served_normally(self):
+ # The control, and it is not optional: without it the check above is satisfied
+ # just as well by refusing every echo request.
+ with self._post(_partial_coverage_engine(covered=3), dict(self.BODY)) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertEqual(choice["text"], "PQAABBCC")
+ self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])
+ self.assertEqual(choice["logprobs"]["text_offset"], [0, 1, 2, 4, 6])
+
+ def test_a_generation_ending_mid_codepoint_is_served(self):
+ # Control 1 of 2. A coverage check written as a string comparison reads an
+ # unflushed incremental decoder as a short generation and turns this valid
+ # response into a 500. Counting in raw-stream coordinates cannot.
+ engine = _partial_coverage_engine(
+ covered=3, chunks=("AA", "BB", "οΏ½"),
+ record_bytes=(b"AA", b"BB", b"\xc3"))
+ with self._post(engine, dict(self.BODY)) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertEqual(choice["text"], "PQAABBοΏ½")
+ self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])
+
+ def test_a_codepoint_split_across_the_seam_is_served(self):
+ # Control 2 of 2. With `echo` on, the reconstruction decodes the generated half
+ # with the decoder that already consumed the prompt bytes, so a codepoint
+ # straddling the seam reads as one character there and as replacement characters
+ # in the raw stream, on purpose.
+ engine = _partial_coverage_engine(
+ covered=1, chunks=("οΏ½οΏ½",), record_bytes=(b"\x82\xac",),
+ echo_bytes=(b"A", b"\xe2"))
+ with self._post(engine, dict(self.BODY)) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertEqual(choice["text"], "A€")
+ self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])
+
+ def test_echo_false_is_refused_on_the_same_coverage_gap(self):
+ # The arrays have to describe the whole text on every shape that carries them, not
+ # only the one that rebuilds `text` out of them. Without this, `echo: false`
+ # returns "AABBCC" with `tokens == ["AA"]` and a 200.
+ status, error = _error_body(
+ self, lambda: self._post(_partial_coverage_engine(covered=1),
+ dict(self.BODY, echo=False)))
+ self.assertEqual(status, 500)
+ self.assertEqual(error["code"], "engine_logprob_records_incomplete")
+ self.assertEqual(error["param"], "logprobs")
+
+ def test_echo_false_full_coverage_is_served(self):
+ # The control for the refusal above: same engine, same shape, every record present.
+ with self._post(_partial_coverage_engine(covered=3),
+ dict(self.BODY, echo=False)) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertEqual(choice["text"], "AABBCC")
+ self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])
+
+ def test_chat_is_refused_on_the_same_coverage_gap(self):
+ # The worst shape this surface can take, reached by the other door: a 200 with a
+ # full message.content and logprobs.content describing a prefix of it.
+ for covered, tokens in ((1, "short"), (0, "absent")):
+ with self.subTest(records=tokens):
+ base = _spawn_test_server(self, _partial_coverage_engine(covered=covered))
+ status, error = _error_body(self, lambda: _post_chat(base, {
+ "model": "test-model", "max_tokens": 3, "logprobs": True,
+ "top_logprobs": 1,
+ "messages": [{"role": "user", "content": "hi"}]}))
+ self.assertEqual(status, 500)
+ self.assertEqual(error["code"], "engine_logprob_records_incomplete")
+ self.assertEqual(error["param"], "logprobs")
+
+ def test_chat_full_coverage_is_served(self):
+ base = _spawn_test_server(self, _partial_coverage_engine(covered=3))
+ with _post_chat(base, {"model": "test-model", "max_tokens": 3, "logprobs": True,
+ "top_logprobs": 1,
+ "messages": [{"role": "user", "content": "hi"}]}) as response:
+ choice = json.load(response)["choices"][0]
+ entries = choice["logprobs"]["content"]
+ self.assertEqual("".join(entry["token"] for entry in entries),
+ choice["message"]["content"])
+ self.assertEqual(choice["message"]["content"], "AABBCC")
+
+ def test_echo_with_no_prompt_echo_records_is_refused_by_name(self):
+ # The engine answered half the opt-in: every generated record, no ECHO frame. The
+ # prompt echo is the half `echo` asked for.
+ base = _spawn_test_server(self, ScriptedEngine(chunks=("Hi",), echoes=()))
+ status, error = _error_body(self, lambda: _post_completions(base, {
+ "model": "test-model", "prompt": "Hi", "max_tokens": 2, "logprobs": 1,
+ "echo": True}))
+ self.assertEqual(status, 500)
+ self.assertEqual(error["code"], "engine_logprob_records_incomplete")
+ self.assertEqual(error["param"], "echo")
+
+
+class PinnedPrefixEchoTest(unittest.TestCase):
+ """A KV slot holding a pin snapshot echoes prompt positions from the snapshot's length
+ rather than from 0. Re-basing that run would attach every logprob to the wrong prompt
+ token, so it is refused by name."""
+
+ def test_a_pinned_prefix_is_refused_by_name(self):
+ base = _spawn_test_server(self, _pinned_prefix_engine())
+ status, error = _error_body(self, lambda: _post_completions(base, {
+ "model": "test-model", "prompt": "abc", "logprobs": 1, "echo": True,
+ "max_tokens": 2}))
+ self.assertEqual(status, 503)
+ self.assertEqual(error["code"], "engine_pinned_prefix_not_echoed")
+ self.assertEqual(error["param"], "echo")
+
+ def test_an_unpinned_prompt_control_is_served(self):
+ base = _spawn_test_server(self, ScriptedEngine(
+ chunks=("ok",),
+ echoes=[(b"a", NAN, []), (b"b", -0.1, [(1, -0.1)])]))
+ with _post_completions(base, {"model": "test-model", "prompt": "ab",
+ "logprobs": 1, "echo": True,
+ "max_tokens": 2}) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertEqual(choice["text"], "abok")
+
+
+class ClientVisibleEngineFaultCodeTest(unittest.TestCase):
+ """Each engine-fault path fails closed AND tells the client which fault. Every test
+ reads the code and param out of the response body, because an exception with a good
+ name that never reaches the wire is the defect rather than the fix. Each has a control
+ that must come back 200, so "names the fault" can never be satisfied by a server that
+ refuses everything.
+
+ The codes say what the ENGINE did, never what this gateway did about it, so they
+ survive a change of reaction."""
+
+ def test_duplicate_candidate_names_itself_with_the_callers_own_param(self):
+ # The same engine table is requested by two different parameters, and a client
+ # branching on the code is told which of its own fields went unanswered.
+ for post, body, param in (
+ (_post_completions,
+ {"model": "test-model", "prompt": "hi", "logprobs": 2, "max_tokens": 1},
+ "logprobs"),
+ (_post_chat,
+ {"model": "test-model", "messages": [{"role": "user", "content": "hi"}],
+ "logprobs": True, "top_logprobs": 2, "max_tokens": 1},
+ "top_logprobs")):
+ with self.subTest(param=param):
+ base = _spawn_test_server(self, _duplicate_candidate_engine())
+ status, error = _error_body(self, lambda: post(base, body))
+ self.assertEqual(status, 500)
+ self.assertEqual(error["code"], "engine_duplicate_logprob_candidate")
+ self.assertEqual(error["param"], param)
+
+ def test_distinct_candidates_control_is_served_on_both_surfaces(self):
+ base = _spawn_test_server(self, _duplicate_candidate_engine(duplicate=False))
+ with _post_completions(base, {"model": "test-model", "prompt": "hi",
+ "logprobs": 2, "max_tokens": 1}) as response:
+ body = json.load(response)
+ self.assertEqual(len(body["choices"][0]["logprobs"]["top_logprobs"][0]), 2)
+ base = _spawn_test_server(self, _duplicate_candidate_engine(duplicate=False))
+ with _post_chat(base, {"model": "test-model",
+ "messages": [{"role": "user", "content": "hi"}],
+ "logprobs": True, "top_logprobs": 2,
+ "max_tokens": 1}) as response:
+ body = json.load(response)
+ self.assertEqual(
+ len(body["choices"][0]["logprobs"]["content"][0]["top_logprobs"]), 2)
+
+ def _tail_engine(self, frame):
+ """A real Engine over a fake process, so the frame is parsed by the dispatcher
+ exactly as it is in production and the named error has to travel from the
+ dispatcher thread to the waiting request thread and out onto the wire."""
+ def respond(process, written):
+ request_id = written.split()[1]
+ process.stdout.feed(frame.replace(b"{id}", request_id))
+ process.stdout.feed(b"DONE " + request_id + b" STAT 1 1 0 1 1 0\n")
+
+ process = FakeProcess(respond)
+ with patch("openai_server.subprocess.Popen", return_value=process):
+ engine = Engine("glm", "model")
+ self.addCleanup(engine.close)
+ return engine
+
+ def test_malformed_tail_names_itself_to_the_client(self):
+ base = _spawn_test_server(
+ self, self._tail_engine(b"DATA {id} 1 -0.5 1 3\nx\n"))
+ status, error = _error_body(self, lambda: _post_completions(base, {
+ "model": "test-model", "prompt": "hi", "logprobs": 1, "max_tokens": 1}))
+ self.assertEqual(status, 500)
+ self.assertEqual(error["code"], "engine_logprob_tail_malformed")
+ self.assertEqual(error["param"], "logprobs")
+
+ def test_well_formed_tail_control_is_served(self):
+ # The control that matters most here: the relay must carry a good frame through
+ # untouched, or "names the fault" is satisfied by a build that fails every
+ # logprobs request.
+ base = _spawn_test_server(
+ self, self._tail_engine(b"DATA {id} 1 -0.5 1 3 -0.5\nx\n"))
+ with _post_completions(base, {"model": "test-model", "prompt": "hi",
+ "logprobs": 1, "max_tokens": 1}) as response:
+ body = json.load(response)
+ self.assertNotIn("error", body)
+ self.assertEqual(body["choices"][0]["text"], "x")
+
+ def test_an_unsupported_engine_names_its_refusal_in_the_body(self):
+ base = _spawn_test_server(self, ScriptedEngine(supports_logprobs_echo=False))
+ status, error = _error_body(self, lambda: _post_completions(base, {
+ "model": "test-model", "prompt": "hi", "logprobs": 1}))
+ self.assertEqual(status, 400)
+ self.assertEqual(error["code"], "unsupported_parameter")
+ self.assertEqual(error["param"], "logprobs")
+
+ def test_the_same_engine_serves_a_plain_request(self):
+ base = _spawn_test_server(self, ScriptedEngine(supports_logprobs_echo=False))
+ with _post_completions(base, {"model": "test-model", "prompt": "hi"}) as response:
+ body = json.load(response)
+ self.assertEqual(body["choices"][0]["text"], "ok")
+ self.assertIsNone(body["choices"][0]["logprobs"])
+
+
+class CapabilitySplitIndependenceTest(unittest.TestCase):
+ """`supports_logprobs_echo` and `supports_tok_ids` gate two unrelated SUBMIT extension
+ keys and must be settable one without the other. Both are `arch == "glm"` today, so
+ nothing short of a double with the two flags set to different values can show the
+ split is real rather than cosmetic."""
+
+ def test_logprobs_is_refused_when_only_token_id_intake_is_available(self):
+ base = _spawn_test_server(self, ScriptedEngine(supports_logprobs_echo=False,
+ supports_tok_ids=True))
+ status, error = _error_body(self, lambda: _post_completions(base, {
+ "model": "test-model", "prompt": "hi", "logprobs": 1}))
+ self.assertEqual(status, 400)
+ self.assertEqual(error["param"], "logprobs")
+
+ def test_logprobs_is_served_when_only_the_numeric_channel_is_available(self):
+ base = _spawn_test_server(self, ScriptedEngine(supports_logprobs_echo=True,
+ supports_tok_ids=False))
+ with _post_completions(base, {"model": "test-model", "prompt": "hi",
+ "logprobs": 1}) as response:
+ choice = json.load(response)["choices"][0]
+ self.assertEqual(choice["logprobs"]["tokens"], ["ok"])
+
+ def test_an_engine_sets_the_two_flags_independently_of_each_other(self):
+ # Set from the same arch check today, but each read site checks the one it means:
+ # one flag forced off must not take the other with it.
+ process = FakeProcess(lambda process, written: None)
+ with patch("openai_server.subprocess.Popen", return_value=process):
+ engine = Engine("glm", "model")
+ self.addCleanup(engine.close)
+ self.assertTrue(engine.supports_logprobs_echo)
+ self.assertTrue(engine.supports_tok_ids)
+ engine.supports_logprobs_echo = False
+ self.assertTrue(engine.supports_tok_ids)
+
+
+class SubmitHeaderExtensionTest(unittest.TestCase):
+ """The SUBMIT header, byte for byte. A request that opts into nothing must produce the
+ header it produced before this channel existed, and an opted-in one must carry the
+ extension keys behind the seventh numeric field."""
+
+ def _engine(self, expected, frames):
+ def respond(process, frame):
+ self.assertEqual(frame, expected)
+ process.stdout.feed(frames)
+
+ process = FakeProcess(respond)
+ with patch("openai_server.subprocess.Popen", return_value=process):
+ engine = Engine("glm", "model")
+ return engine, process
+
+ def test_a_plain_request_submit_header_is_byte_identical(self):
+ expected = b"SUBMIT 1 0 5 4 0.25 0.9\n" + b"hello\n"
+ engine, process = self._engine(
+ expected, b"DATA 1 2\nok\nDONE 1 STAT 1 2.5 0 1.0 5 0\n")
+ engine.generate("hello", 4, 0.25, 0.9, [].append)
+ engine.close()
+ self.assertEqual(process.writes, [expected])
+
+ def test_an_opted_in_request_carries_the_extension_behind_the_seventh_field(self):
+ # coli_submit_parse reaches its key=value arm only after seven numeric fields, and
+ # sending the extension on every request would make the plain-header test above
+ # fail.
+ expected = b"SUBMIT 1 0 5 4 0.25 0.9 0 logprobs=2\n" + b"hello\n"
+ engine, process = self._engine(
+ expected,
+ b"ACCEPT 1 5\nECHO 1 1 0 nan 0\nh\n"
+ b"DATA 1 2 -0.223144 1 3 -0.223144\nok\nDONE 1 STAT 1 2.5 0 1.0 5 0\n")
+ stats = engine.generate("hello", 4, 0.25, 0.9, [].append, logprobs=2,
+ gbytes_before_ext=True)
+ engine.close()
+ self.assertEqual(process.writes, [expected])
+ self.assertEqual(stats["logprobs"]["generated"],
+ [(b"ok", {"lp": -0.223144, "topk": [(3, -0.223144)]})])
+
+ def test_the_seventh_field_is_sent_only_when_the_caller_asks_for_it(self):
+ # It is a request, not a rule: engines that parse a header without that field
+ # reject or mis-read one carrying it, so only a caller whose engine uses
+ # coli_submit_parse asks for it. `pin` rides the same request.
+ expected = b"SUBMIT 1 0 5 0 0.25 0.9 logprobs=1 pin=1\n" + b"hello\n"
+ engine, process = self._engine(
+ expected, b"ACCEPT 1 5\nECHO 1 1 0 nan 0\nh\nDONE 1 STAT 0 2.5 0 1.0 5 0\n")
+ engine.generate("hello", 0, 0.25, 0.9, lambda _chunk: None, logprobs=1, pin=True,
+ on_echo=lambda _record: None)
+ engine.close()
+ self.assertEqual(process.writes, [expected])
+
+
+class EngineExtensionCallSiteTest(unittest.TestCase):
+ """Each of this server's call sites into Engine.generate() applies the same extension
+ profile, so none can be left behind. For each: an opted-in request emits the extended
+ header, and a plain request emits the base header."""
+
+ def _header(self, path, body, engine_frames=None):
+ """The SUBMIT header one HTTP request produces, taken off a fake engine process."""
+ headers = []
+
+ def respond(process, frame):
+ headers.append(frame.split(b"\n", 1)[0])
+ request_id = frame.split()[1]
+ process.stdout.feed(engine_frames.replace(b"{id}", request_id)
+ if engine_frames else
+ b"ACCEPT " + request_id + b" 1\n"
+ b"DATA " + request_id + b" 2\nok\n"
+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
+
+ process = FakeProcess(respond)
+ with patch("openai_server.subprocess.Popen", return_value=process):
+ engine = Engine("glm", "model")
+ self.addCleanup(engine.close)
+ base = _spawn_test_server(self, engine)
+ with _post_json(base, path, body) as response:
+ response.read()
+ return headers[0]
+
+ OPTED_IN_FRAMES = (b"ACCEPT {id} 1\n"
+ b"DATA {id} 2 -0.5 1 3 -0.5\nok\n"
+ b"DONE {id} STAT 1 2.5 0 1.0 5 0\n")
+
+ def test_non_streaming_completions_site(self):
+ plain = self._header("/v1/completions", {"model": "test-model", "prompt": "hi"})
+ self.assertNotIn(b"logprobs=", plain)
+ opted = self._header("/v1/completions",
+ {"model": "test-model", "prompt": "hi", "logprobs": 2},
+ self.OPTED_IN_FRAMES)
+ self.assertIn(b" 0 logprobs=2", opted)
+
+ def test_non_streaming_chat_site(self):
+ messages = [{"role": "user", "content": "hi"}]
+ plain = self._header("/v1/chat/completions",
+ {"model": "test-model", "messages": messages})
+ self.assertNotIn(b"logprobs=", plain)
+ opted = self._header("/v1/chat/completions",
+ {"model": "test-model", "messages": messages,
+ "logprobs": True, "top_logprobs": 1},
+ self.OPTED_IN_FRAMES)
+ self.assertIn(b" 0 logprobs=1", opted)
+
+ def test_streaming_plain_site_keeps_the_base_header(self):
+ # Streaming with logprobs is a named 400, so this site can only ever be reached
+ # without the extension -- and must never grow it.
+ header = self._header("/v1/completions",
+ {"model": "test-model", "prompt": "hi", "stream": True})
+ self.assertNotIn(b"logprobs=", header)
+ status, error = _error_body(self, lambda: self._header(
+ "/v1/completions",
+ {"model": "test-model", "prompt": "hi", "stream": True, "logprobs": 1}))
+ self.assertEqual((status, error["param"]), (400, "logprobs"))
+
+ def test_streaming_chat_with_tools_site_keeps_the_base_header(self):
+ messages = [{"role": "user", "content": "hi"}]
+ tools = [{"type": "function",
+ "function": {"name": "search",
+ "parameters": {"type": "object", "properties": {}}}}]
+ header = self._header("/v1/chat/completions",
+ {"model": "test-model", "messages": messages,
+ "tools": tools, "stream": True})
+ self.assertNotIn(b"logprobs=", header)
+ status, error = _error_body(self, lambda: self._header(
+ "/v1/chat/completions",
+ {"model": "test-model", "messages": messages, "tools": tools,
+ "stream": True, "logprobs": True}))
+ self.assertEqual((status, error["param"]), (400, "logprobs"))
+
+ def test_the_messages_endpoint_ignores_a_logprobs_field(self):
+ # `logprobs` is not in the Anthropic request shape, and this endpoint builds its
+ # own translated request rather than forwarding the client's body, so the field
+ # never reaches the options parser. A 200 with no logprobs anywhere, for either
+ # endpoint's spelling of the field.
+ for value in (True, 1):
+ with self.subTest(value=value):
+ base = _spawn_test_server(self, ScriptedEngine(chunks=("Hello",)))
+ with _post_json(base, "/v1/messages",
+ {"model": "test-model", "max_tokens": 4,
+ "messages": [{"role": "user", "content": "hi"}],
+ "logprobs": value}) as response:
+ body = json.load(response)
+ self.assertEqual(body["content"], [{"type": "text", "text": "Hello"}])
+ self.assertNotIn("logprobs", json.dumps(body))
+
+ def test_the_messages_endpoint_keeps_the_base_header(self):
+ # /v1/messages sends no extension key, and nothing about this change may give it
+ # one.
+ header = self._header("/v1/messages",
+ {"model": "test-model", "max_tokens": 4,
+ "messages": [{"role": "user", "content": "hi"}]})
+ self.assertEqual(header.split(b" ")[:2], [b"SUBMIT", b"1"])
+ self.assertNotIn(b"logprobs=", header)
+ self.assertNotIn(b"=", header)
+
+
+class DispatcherLogprobTailTest(unittest.TestCase):
+ """A numeric tail is parsed only for the request that asked for the channel. A
+ malformed one fails that request alone; a frame carrying fields a request never asked
+ for is tolerated, as it was before the channel existed."""
+
+ def _engine(self, respond):
+ process = FakeProcess(respond)
+ with patch("openai_server.subprocess.Popen", return_value=process):
+ engine = Engine("glm", "model")
+ self.addCleanup(engine.close)
+ return engine, process
+
+ def test_extra_fields_on_a_plain_requests_frame_are_tolerated(self):
+ # The request never asked for the channel, so the tail is not parsed -- not even
+ # to reject it. This is the base behaviour, restored.
+ def respond(process, frame):
+ request_id = frame.split()[1]
+ process.stdout.feed(b"DATA " + request_id + b" 2 garbage fields here\nok\n"
+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
+
+ engine, _process = self._engine(respond)
+ chunks = []
+ stats = engine.generate("hi", 4, 0.25, 0.9, chunks.append)
+ self.assertEqual(chunks, ["ok"])
+ self.assertNotIn("logprobs", stats)
+
+ def _concurrent_fault(self, bad_frame):
+ """Drive one plain request and one opted-in request over the same engine, with the
+ opted-in request's fault arriving while the plain one is still in flight.
+
+ The plain request is submitted first and left waiting for its DONE. The opted-in
+ request's SUBMIT then triggers the bad frame, that request's terminal frame, and
+ finally the plain request's. Returns `(plain result, opted-in exception)`."""
+ first_submitted = threading.Event()
+
+ def respond(process, frame):
+ request_id = frame.split()[1]
+ if b"logprobs=" in frame.split(b"\n", 1)[0]:
+ process.stdout.feed(bad_frame.replace(b"{id}", request_id))
+ process.stdout.feed(b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
+ process.stdout.feed(b"DONE 1 STAT 1 2.5 0 1.0 5 0\n")
+ else:
+ process.stdout.feed(b"DATA " + request_id + b" 2\nok\n")
+ first_submitted.set()
+
+ engine, _process = self._engine(respond)
+ plain = {}
+
+ def run_plain():
+ chunks = []
+ try:
+ plain["stats"] = engine.generate("hi", 4, 0.25, 0.9, chunks.append)
+ plain["chunks"] = chunks
+ except Exception as error: # recorded, asserted by the caller
+ plain["error"] = error
+
+ thread = threading.Thread(target=run_plain, daemon=True)
+ thread.start()
+ self.assertTrue(first_submitted.wait(2))
+ with self.assertRaises(APIError) as caught:
+ engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=1,
+ gbytes_before_ext=True)
+ thread.join(timeout=2)
+ self.assertFalse(thread.is_alive())
+ return plain, caught.exception
+
+ def test_a_malformed_tail_spares_a_request_that_is_in_flight(self):
+ # The containment rule's whole point: not that the dispatcher survived the fault,
+ # but that the request which never asked for the channel is untouched by it.
+ # Failing every pending request instead leaves the plain request holding an error
+ # it did not earn.
+ plain, error = self._concurrent_fault(b"DATA {id} 2 -0.5 1 3\nok\n")
+ self.assertEqual(error.status, 500)
+ self.assertEqual(error.code, "engine_logprob_tail_malformed")
+ self.assertNotIn("error", plain)
+ self.assertEqual(plain["chunks"], ["ok"])
+ self.assertEqual(plain["stats"]["completion_tokens"], 1)
+
+ def test_a_malformed_echo_logprob_on_a_request_that_never_opted_in(self):
+ # The `lp` field is read for EVERY ECHO frame, opted in or not, because the
+ # closed-set scorer's own `logprob` key comes from it -- so unlike the tail, this
+ # one is reachable by a request with no numeric channel at all. That is the row of
+ # the four that was still reaching the dispatcher's blanket handler, which fails
+ # every request in flight and stops the dispatcher for good. Here the bad frame
+ # goes to the NON-opted-in request and a second request is in flight beside it.
+ second_submitted = threading.Event()
+
+ def respond(process, frame):
+ request_id = frame.split()[1]
+ if b"logprobs=" in frame.split(b"\n", 1)[0]:
+ process.stdout.feed(b"DATA " + request_id + b" 2 -0.5 1 3 -0.5\nok\n")
+ second_submitted.set()
+ else:
+ process.stdout.feed(b"ECHO " + request_id + b" 1 0 zz 0\nh\n")
+ # The fault is delivered on the turn's terminal frame, so the engine has
+ # to end the turn for the request to be answered at all.
+ process.stdout.feed(
+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
+
+ engine, _process = self._engine(respond)
+ opted = {}
+
+ def run_opted():
+ chunks = []
+ try:
+ opted["stats"] = engine.generate(
+ "hi", 4, 0.25, 0.9, chunks.append, logprobs=1,
+ gbytes_before_ext=True, on_echo=lambda _r: None)
+ opted["chunks"] = chunks
+ except Exception as error: # recorded, asserted below
+ opted["error"] = error
+
+ thread = threading.Thread(target=run_opted, daemon=True)
+ thread.start()
+ self.assertTrue(second_submitted.wait(2))
+ with self.assertRaises(APIError) as caught:
+ engine.generate("hi", 4, 0.25, 0.9, [].append, on_echo=lambda _r: None)
+ self.assertEqual(caught.exception.status, 500)
+ self.assertEqual(caught.exception.code, "engine_logprob_tail_malformed")
+ self.assertIsNone(engine.dispatcher_error)
+ # The request beside it never saw the frame and is still being served.
+ self.assertNotIn("error", opted)
+ self.assertTrue(thread.is_alive())
+ thread.join(timeout=0.1)
+
+ def test_a_malformed_echo_position_spares_a_request_that_is_in_flight(self):
+ # The ECHO `pos` field rides the same opted-in frame as the numeric tail and gets
+ # the same containment. Raising inside the dispatcher instead kills it, and every
+ # concurrent request on the engine fails with that request's fault.
+ plain, error = self._concurrent_fault(b"ECHO {id} 1 x nan 0\nh\n")
+ self.assertEqual(error.status, 500)
+ # Its own code: the position and the numeric tail are different fields of the
+ # frame, and a client branching on the code is told which one was unreadable.
+ self.assertEqual(error.code, "engine_echo_position_malformed")
+ self.assertEqual(error.param, "echo")
+ self.assertNotIn("error", plain)
+ self.assertEqual(plain["chunks"], ["ok"])
+
+ def _poll(self, predicate, what):
+ deadline = time.monotonic() + 2
+ while time.monotonic() < deadline:
+ if predicate():
+ return
+ time.sleep(0.005)
+ self.fail(what)
+
+ def test_a_recorded_fault_keeps_the_turn_open_until_its_terminal_frame(self):
+ # The fault is found mid-turn and raised at the end of it. Delivering it when it
+ # is found would unwind the request thread, and with it the scheduler admission,
+ # while the engine is still generating and no CANCEL has been written -- the next
+ # request would then submit into a pipe nobody is reading. The observable form of
+ # "the turn is still the engine's" is that the pending entry is still there.
+ release = threading.Event()
+
+ def respond(process, frame):
+ request_id = frame.split()[1]
+ process.stdout.feed(b"DATA " + request_id + b" 2 -0.5 1 3\nok\n")
+
+ def terminal():
+ release.wait(2)
+ process.stdout.feed(
+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
+
+ threading.Thread(target=terminal, daemon=True).start()
+
+ engine, process = self._engine(respond)
+ result = {}
+
+ def run():
+ try:
+ engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=1,
+ gbytes_before_ext=True)
+ except APIError as error:
+ result["error"] = error
+
+ thread = threading.Thread(target=run, daemon=True)
+ thread.start()
+ self._poll(lambda: (engine.pending.get("1") is not None
+ and engine.pending["1"].failed is not None),
+ "the fault was never recorded")
+ # Recorded, held, and not yet delivered.
+ self.assertIn("1", engine.pending)
+ self.assertNotIn("error", result)
+ self.assertTrue(thread.is_alive())
+ release.set()
+ thread.join(timeout=2)
+ self.assertFalse(thread.is_alive())
+ self.assertEqual(result["error"].code, "engine_logprob_tail_malformed")
+ self.assertNotIn("1", engine.pending)
+ # No CANCEL was written: the turn was never abandoned.
+ self.assertEqual([w for w in process.writes if w.startswith(b"CANCEL")], [])
+
+ def test_frames_for_a_failed_or_finished_request_are_ignored(self):
+ # While the fault is recorded the entry stays, and that request's later frames are
+ # routed to it and discarded; once its terminal frame has been delivered the entry
+ # is gone, and anything further for that id belongs to no request. Neither may be
+ # treated as a fault of its own: the dispatcher keeps running and the next request
+ # on the same engine completes normally.
+ def respond(process, frame):
+ request_id = frame.split()[1]
+ if b"logprobs=" in frame.split(b"\n", 1)[0]:
+ process.stdout.feed(b"DATA " + request_id + b" 2 -0.5 1 3\nok\n")
+ process.stdout.feed(b"DATA " + request_id + b" 2\nxx\n")
+ process.stdout.feed(b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
+ # ...and two frames the engine sends after the turn it already ended,
+ # which belong to no pending entry at all.
+ process.stdout.feed(b"DATA " + request_id + b" 2\nyy\n")
+ process.stdout.feed(b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
+ else:
+ process.stdout.feed(b"DATA " + request_id + b" 2\nok\n"
+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
+
+ engine, _process = self._engine(respond)
+ with self.assertRaises(APIError) as caught:
+ engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=1,
+ gbytes_before_ext=True)
+ self.assertEqual(caught.exception.code, "engine_logprob_tail_malformed")
+ chunks = []
+ stats = engine.generate("hi", 4, 0.25, 0.9, chunks.append)
+ self.assertEqual(chunks, ["ok"])
+ self.assertEqual(stats["completion_tokens"], 1)
+ self.assertIsNone(engine.dispatcher_error)
+
+ def _faulted_turn(self, terminal, cancelled=None, settle=1.0):
+ """One opted-in request whose numeric tail is malformed, with `terminal` delivered
+ only once the server answers the fault -- so the turn stays open long enough for
+ the request thread's idle poll to run, which is where that answer is written.
+
+ A timer delivers `terminal` anyway after `settle` seconds, so a build that answers
+ nothing fails an assertion instead of hanging the suite. Returns `(raised exception
+ or None, the frames the server wrote)`."""
+ answered = threading.Event()
+
+ def respond(process, frame):
+ if not frame.startswith(b"SUBMIT"):
+ answered.set() # STOP or CANCEL: the turn can end now
+ process.stdout.feed(terminal.replace(b"{id}", frame.split()[1]))
+ return
+ request_id = frame.split()[1]
+ process.stdout.feed(b"DATA " + request_id + b" 2 -0.5 1 3\nok\n")
+
+ def fallback():
+ if not answered.wait(settle):
+ process.stdout.feed(terminal.replace(b"{id}", request_id))
+
+ threading.Thread(target=fallback, daemon=True).start()
+
+ engine, process = self._engine(respond)
+ raised = None
+ try:
+ engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=1,
+ gbytes_before_ext=True, cancelled=cancelled)
+ except Exception as error: # the class is what the test asserts
+ raised = error
+ return raised, process.writes
+
+ def test_a_cancellation_outranks_the_recorded_fault_on_a_terminal_done(self):
+ # The engine may end a cancelled turn with DONE rather than ERROR CANCELLED. The
+ # client left either way, so the turn is a cancellation, not a server failure --
+ # which is what the scheduler books when anything but ClientCancelled comes out,
+ # and what the metrics then count.
+ raised, writes = self._faulted_turn(b"DONE {id} STAT 1 2.5 0 1.0 5 0\n",
+ cancelled=lambda: True)
+ self.assertIsInstance(raised, ClientCancelled)
+ self.assertEqual([w for w in writes if w.startswith(b"CANCEL")], [b"CANCEL 1\n"])
+ self.assertEqual([w for w in writes if w.startswith(b"STOP")], [])
+
+ def test_a_cancellation_outranks_the_recorded_fault(self):
+ # The client left mid-turn on a request that also carries a fault of its own. That
+ # is answered the way it is answered on a healthy turn -- ClientCancelled, not a
+ # 500 written to a socket that is already closed.
+ raised, writes = self._faulted_turn(b"ERROR {id} CANCELLED\n",
+ cancelled=lambda: True)
+ self.assertIsInstance(raised, ClientCancelled)
+ # The disconnect is what the engine was told about: CANCEL, not STOP.
+ self.assertEqual([w for w in writes if w.startswith(b"CANCEL")], [b"CANCEL 1\n"])
+ self.assertEqual([w for w in writes if w.startswith(b"STOP")], [])
+
+ def test_any_other_terminal_error_keeps_the_fault_and_carries_the_engines_text(self):
+ # The fault happened first and names the framing defect, so it is what the client
+ # is told; the engine's own parting word is appended rather than dropped. Reporting
+ # the engine's error instead would tell a client with an over-long prompt to
+ # shorten it when the real defect was a frame this server could not read.
+ raised, _writes = self._faulted_turn(b"ERROR {id} CONTEXT_EXCEEDED 9 8\n")
+ self.assertIsInstance(raised, APIError)
+ self.assertEqual(raised.status, 500)
+ self.assertEqual(raised.code, "engine_logprob_tail_malformed")
+ self.assertIn("malformed per-token logprob tail", raised.message)
+ self.assertIn("The engine then reported:", raised.message)
+ self.assertIn("CONTEXT_EXCEEDED", raised.message)
+
+ def test_a_faulted_turn_is_stopped_once_and_then_drained(self):
+ # The turn is doomed but still the engine's: the admission is held to its terminal
+ # frame either way, and one STOP keeps it from spending the rest of the budget on a
+ # response nobody will receive. Exactly one, and no CANCEL.
+ raised, writes = self._faulted_turn(b"DONE {id} STAT 1 2.5 0 1.0 5 0\n")
+ self.assertIsInstance(raised, APIError)
+ self.assertEqual(raised.code, "engine_logprob_tail_malformed")
+ self.assertEqual([w for w in writes if w.startswith(b"STOP")], [b"STOP 1\n"])
+ self.assertEqual([w for w in writes if w.startswith(b"CANCEL")], [])
+
+ def test_a_healthy_slow_turn_is_not_stopped(self):
+ # The control for the fault guard, and it has to IDLE to be one: a fixture that
+ # feeds every frame back to back never reaches the `queue.Empty` branch, which is
+ # the only place the guard lives. This one leaves a gap wider than the 0.05 s poll
+ # between DATA and DONE -- an ordinary inter-token interval on a large model -- so
+ # a build that stops on every idle tick writes a STOP here and also skips the
+ # decode for the rest of the turn, returning a 200 with an empty completion.
+ def respond(process, frame):
+ request_id = frame.split()[1]
+ process.stdout.feed(b"DATA " + request_id + b" 2 -0.5 1 3 -0.5\nok\n")
+
+ def terminal():
+ time.sleep(0.15) # three idle polls, well under the ceiling
+ process.stdout.feed(
+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
+
+ threading.Thread(target=terminal, daemon=True).start()
+
+ engine, process = self._engine(respond)
+ chunks = []
+ stats = engine.generate("hi", 4, 0.25, 0.9, chunks.append, logprobs=1,
+ gbytes_before_ext=True)
+ self.assertEqual([w for w in process.writes if w.startswith(b"STOP")], [])
+ self.assertEqual([w for w in process.writes if w.startswith(b"CANCEL")], [])
+ # ...and the turn still delivered, which is what a stray STOP would cost.
+ self.assertEqual(chunks, ["ok"])
+ self.assertEqual(stats["completion_tokens"], 1)
+
+ def test_a_failed_fault_stop_write_answers_by_name_and_keeps_the_connection(self):
+ # The STOP written for a recorded fault can itself fail at the pipe. An engine this
+ # server cannot write to outranks a fault in the stream it cannot read: the request
+ # is answered with a named error rather than having its connection dropped, which
+ # is what an unhandled write error does. Written against this tree's own STOP write
+ # so the pin survives the composition with the checked writer.
+ class DeadOnStop:
+ """The engine's stdin: accepts the SUBMIT, refuses the STOP."""
+ def __init__(self, process):
+ self.process = process
+
+ def write(self, data):
++ # The checked writer (#1721) hands every write a `memoryview`
++ # slice, so normalise before pattern-matching the frame bytes --
++ # the same normalisation `FakeProcess.write` above does.
++ data = bytes(data)
+ if data.startswith(b"STOP"):
+ raise BrokenPipeError(32, "Broken pipe")
+ return self.process.real_write(data)
+
+ def flush(self):
+ pass
+
+ def respond(process, frame):
+ request_id = frame.split()[1]
+ process.stdout.feed(b"DATA " + request_id + b" 2 -0.5 1 3\nok\n")
+
+ def terminal():
+ time.sleep(0.15)
+ process.stdout.feed(
+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
+
+ threading.Thread(target=terminal, daemon=True).start()
+
+ engine, process = self._engine(respond)
+ process.real_write = process.write
+ process.stdin = DeadOnStop(process)
+ with self.assertRaises(Exception) as caught:
+ engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=1,
+ gbytes_before_ext=True)
+ # The turn is over and nothing will ever answer it: the entry must not be left for
+ # the dispatcher to deliver to a caller that has already unwound, nor to hold a
+ # request id the next turn could be given.
+ self.assertNotIn("1", engine.pending)
+ self.assertEqual(engine.pending, {})
+ # Named, and reaching the caller: an unraised write error would instead unwind
+ # through the handler and close the socket with no status line on it.
+ raised = caught.exception
+ self.assertNotIsInstance(raised, BrokenPipeError)
+ named = raised.code if isinstance(raised, APIError) else str(raised)
+ self.assertTrue(named, "the failed STOP write must be reported by name")
+ self.assertIn("STOP", str(raised).upper() + (raised.message.upper()
+ if isinstance(raised, APIError) else ""))
+
+ def test_the_malformed_tail_battery(self):
+ # A required field absent or non-numeric, a k outside the engine's interface, or a
+ # negative token id. Every one of them is a missing or unreadable field, never a
+ # frame that merely carries more than the grammar names.
+ for tail in (b"-0.5", b"-0.5 1 3", b"x 1 3 -0.5", b"-0.5 99 3 -0.5",
+ b"-0.5 1 -3 -0.5", b"-0.5 x 3 -0.5"):
+ with self.subTest(tail=tail):
+ def respond(process, frame, tail=tail):
+ request_id = frame.split()[1]
+ process.stdout.feed(b"DATA " + request_id + b" 2 " + tail + b"\nok\n"
+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
+
+ engine, _process = self._engine(respond)
+ with self.assertRaises(APIError) as caught:
+ engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=1,
+ gbytes_before_ext=True)
+ self.assertEqual(caught.exception.code, "engine_logprob_tail_malformed")
+
+ def test_extra_trailing_fields_on_an_opted_in_frame_are_served(self):
+ # The parser consumes the fields the grammar defines and ignores what follows. The
+ # base tolerated a longer frame for every caller, and this parser now runs for
+ # internal consumers of the channel that read only part of the record.
+ def respond(process, frame):
+ request_id = frame.split()[1]
+ process.stdout.feed(
+ b"DATA " + request_id + b" 2 -0.5 1 3 -0.5 9 -9.0\nok\n"
+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
+
+ engine, _process = self._engine(respond)
+ chunks = []
+ stats = engine.generate("hi", 4, 0.25, 0.9, chunks.append, logprobs=1,
+ gbytes_before_ext=True)
+ self.assertEqual(chunks, ["ok"])
+ self.assertEqual(stats["logprobs"]["generated"],
+ [(b"ok", {"lp": -0.5, "topk": [(3, -0.5)]})])
+
+ def test_the_closed_set_scoring_path_keeps_the_bases_frame_tolerance(self):
+ # The other caller of the numeric channel reads `pos`/`logprob`/`text` and nothing
+ # else, and its engine's ECHO frames may carry more than this parser names. Driven
+ # through a real Engine and the real dispatcher, in that caller's own call shape,
+ # because a FakeEngine cannot see a framing change at all.
+ for label, echo_frame in (
+ ("exact", b"ECHO {id} 1 0 nan 0\nh\n"),
+ ("one extra trailing field", b"ECHO {id} 1 0 nan 0 7\nh\n"),
+ ("a populated table plus an extra", b"ECHO {id} 1 0 -0.5 1 3 -0.5 7\nh\n")):
+ with self.subTest(frame=label):
+ def respond(process, frame, echo_frame=echo_frame):
+ request_id = frame.split()[1]
+ process.stdout.feed(echo_frame.replace(b"{id}", request_id))
+ process.stdout.feed(b"DONE " + request_id + b" STAT 0 2.5 0 1.0 5 0\n")
+
+ engine, _process = self._engine(respond)
+ echoes = []
+ engine.generate("hi", 0, 0.0, 1.0, lambda _chunk: None,
+ logprobs=1, pin=True, on_echo=echoes.append)
+ self.assertEqual(len(echoes), 1)
+ self.assertEqual(echoes[0]["pos"], 0)
+ self.assertEqual(echoes[0]["text"], "h")
+
+ def test_a_well_formed_tail_reaches_the_caller(self):
+ def respond(process, frame):
+ request_id = frame.split()[1]
+ process.stdout.feed(
+ b"ACCEPT " + request_id + b" 1\n"
+ b"ECHO " + request_id + b" 1 0 nan 0\nh\n"
+ b"DATA " + request_id + b" 2 -0.5 2 3 -0.5 4 -1.25\nok\n"
+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")
+
+ engine, _process = self._engine(respond)
+ echoes = []
+ stats = engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=2,
+ gbytes_before_ext=True, on_echo=echoes.append)
+ self.assertEqual(stats["logprobs"]["generated"],
+ [(b"ok", {"lp": -0.5, "topk": [(3, -0.5), (4, -1.25)]})])
+ # The echo record keeps the scorer's three keys and their meanings, and gains the
+ # payload and the numeric tail the logprobs surface needs.
+ self.assertEqual(echoes[0]["pos"], 0)
+ self.assertIsNone(echoes[0]["logprob"])
+ self.assertEqual(echoes[0]["text"], "h")
+ self.assertEqual(echoes[0]["bytes"], b"h")
+ self.assertTrue(math.isnan(echoes[0]["lp"]))
+ self.assertEqual(echoes[0]["topk"], [])
+
+
if __name__ == "__main__":
unittest.main()
diff --cc docs/api.md
index 3b5dbc2f,50034b49..2e12792f
--- a/docs/api.md
+++ b/docs/api.md
@@@ -78,127 -78,18 +78,134 @@@ sequences. The extensio
The server serves one generation at a time: the model stays in one persistent
process, so concurrent HTTP requests queue instead of loading duplicate model
copies. Tool calling depends on the active engine; see the support matrix below.
-Images, log probabilities, and token penalties return an explicit error rather
-than being silently ignored. `seed` is accepted and ignored (see below).
-Audio is accepted only by Inkling checkpoints with
-audio support. The default bind address is localhost; set `COLI_API_KEY` before
-exposing the server beyond the machine.
+Images and token penalties return an explicit error rather than being silently
+ignored. The OpenAI-compatible endpoints request log probabilities only from a
+glm engine (see below); on every other engine such a request is refused with a
- named error, never silently ignored. Audio is accepted only by Inkling
- checkpoints with audio support. The default bind address is localhost; set
- `COLI_API_KEY` before exposing the server beyond the machine.
++named error, never silently ignored. `seed` is accepted and ignored (see
++below). Audio is accepted only by Inkling checkpoints with audio support. The
++default bind address is localhost; set `COLI_API_KEY` before exposing the
++server beyond the machine.
+
+ ### `seed`
+
+ `seed` is accepted (not rejected) for OpenAI-API request-shape compatibility;
+ the value is not validated. This server sends no per-request seed on the
+ wire, so the value has no effect at any temperature.
+### Log probabilities and prompt echo
+
+`/v1/completions` accepts the legacy integer `logprobs` (**0–32**; 0 means no log
+probabilities at all, see below; the upper bound is the engine's top-32 read-out
+interface, and anything above 32 is a named 400) and boolean `echo`;
+`/v1/chat/completions` accepts boolean `logprobs` plus integer `top_logprobs` (0–32) and
+returns `choices[].logprobs.content[]` (`{token, logprob, bytes, top_logprobs}` per
+generated token) β€” chat has no `echo` concept and rejects one with a 400. Each endpoint
+takes the OpenAI request shape for it and refuses the other's by name. A non-boolean
+`echo` is a named 400 (`invalid_value`) on both endpoints, independent of whether
+`logprobs` is requested at all. On chat, `top_logprobs` is type- and range-checked even
+when `logprobs` is false or absent, so a malformed `top_logprobs` is a named 400 whether
+or not the gate it would feed is open; a valid `top_logprobs` with `logprobs` off remains
+a documented no-op.
+
+The zero semantics are explicit, not a truthiness accident: on `/v1/completions`,
+`logprobs: 0`, `false` and `null` all mean **no log probabilities** (the request succeeds
+with `choices[].logprobs: null`, exactly as if the field were omitted), while boolean
+`true` is a named 400 β€” the legacy field is an integer count, and a boolean carries no
+count. On `/v1/chat/completions` the field is a boolean gate (`null` behaves like `false`;
+any integer is a named 400). `echo: null` normalises to absent on both endpoints, so a
+client that serialises its whole request model with nulls is never refused for a field it
+did not mean to set. `logprobs` together with `stream` is a named 400: per-delta log
+probabilities are not built.
+
+`/v1/completions` with `echo: true` returns the full legacy `logprobs` object (`tokens`,
+`token_logprobs`, `top_logprobs`, `text_offset`) covering the echoed prompt plus any
+generated tokens, and `text` itself is the reconstructed prompt followed by the completion
+(the standard OpenAI legacy behavior for `echo: true`) rather than the completion alone;
+`echo: true` without an active `logprobs` request is a named 400 (`echo` requires
+`logprobs`): the prompt echo is built out of the engine's per-token records, so without
+them there is nothing to echo, and a request that asks for one is told so rather than
+served without it. Echoing a prompt with no logprobs at all is offered as a separate
+proposal. `text_offset` is a character offset into
+that same returned `text` string, always counted from 0 β€” including when `echo` is false,
+where `text` holds only the completion and the offsets describe only that text, not a
+position within the (unreturned) prompt.
+
+`"".join(tokens)` is the returned `text`: where a `stop` sequence matches partway through
+a token, that token's `tokens` entry is truncated to the characters actually emitted (a
+`stop` of `"lo"` against a token decoding to `"Hello"` reports `"Hel"`), rather than being
+dropped or reporting characters the client did not receive. On chat, a truncated entry's
+`bytes` β€” and its `top_logprobs` entries' `bytes` β€” are truncated with it, to the UTF-8 of
+the characters that entry reports. An entry that was not truncated keeps its frame's own
+payload, which need not be the encoding of its `token`: a frame carrying only part of a
+multi-byte codepoint reports `token: ""` with that frame's bytes.
+
+The requested top-k table is **unsorted** on the wire β€” do not assume the first entry is
+the argmax. Per-token values are printed by the engine to six decimal digits of precision.
+Non-finite values (a degenerate all-`-inf` logit row, say) serialize as JSON `null`, never
+a clamped number.
+
+These endpoints request the numeric per-token channel only from a glm engine; on every
+other engine the request returns a named 400 rather than being silently ignored. That is a
+statement about what these endpoints request, and about nothing else.
+
+Known limitations, current build:
+
+- **Cost.** Requesting `logprobs` at all β€” completions or chat, `echo` or not β€” asks the
+ engine for a full read-out pass over the whole prompt, because the request carries one
+ opt-in and not a separate "echo" one. Such a request normally forfeits prefix-cache
+ reuse. There is no cap on prompt length for it, so a very long prompt pays a
+ correspondingly large one-shot cost.
+- **Waiting.** A logprobs request waits for the engine exactly as any other request waits.
+ An engine that never acknowledges the submission β€” one older than this extension, or one
+ whose parser disagrees about the header's shape β€” leaves the request waiting, as it
+ would through a cold prefill. There is no separate bounded wait for the opt-in. A request
+ whose per-token frames could not be read waits the same way: the turn belongs to the
+ engine until it sends a terminal frame, so the refusal below is delivered then rather
+ than the moment the unreadable frame arrives. The engine is sent one stop when that
+ happens, so the turn does not spend the rest of its budget on a response nobody receives.
+- **Cancellation.** A cancel may not take effect until the read-out this request asked for
+ has finished, so the window in which a disconnect goes unnoticed is wider on a long
+ prompt than it is without `logprobs`.
+- **Alternative-token labels.** `top_logprobs` entries for candidate token ids other than
+ the position's own actual token are not decoded text (no server-side tokenizer exists, by
+ design) β€” they are labeled `<token_id:N>`. Only the position's own token, identified by
+ an exact logprob match rather than by id, gets its real decoded text.
+- **The sampled token is not guaranteed to appear in its own `top_logprobs` table.** The
+ engine's numeric channel reports the top-k candidates by its own read-out; if the chosen
+ token falls outside that table, no entry represents it, and the response's
+ `token_logprobs`/`logprob` field is still the chosen token's own value read from the
+ frame directly, not looked up in the table.
+- **The arrays describe the text the client received, exactly, and nothing beyond it.**
+ Every generated token is located in the returned text by its span in the raw engine
+ stream, so the stage that consumed a character decides how much of that token's entry
+ reaches the array: a token a matched `stop` cut partway through reports the characters
+ that were emitted, and generated text that does not reach `message.content` β€” reasoning,
+ template markers, tool-call syntax, a swallowed role marker β€” is not described there.
+ Reasoning, template and tool-call tokens are therefore not described by
+ `logprobs.content`; a separate field for the full stream is proposed separately. On
+ every response that carries logprobs, `"".join(tokens)` is the returned `text` and
+ `"".join` over `logprobs.content` is `message.content`. When the engine's records cannot
+ support that β€” they stop short of the generated text, or none arrive at all β€” the
+ request is refused by name rather than answered with arrays describing a prefix.
+- **Engine-side faults are named, not generic.** When the engine's per-token records cannot
+ answer the request, the 5xx body carries a `code` saying what the engine did:
+ `engine_logprob_tail_malformed` (a numeric tail this server cannot read β€” it fails that
+ one request and no other), `engine_echo_position_malformed` (an unreadable prompt-echo
+ position, which is a different field of the same frame),
+ `engine_duplicate_logprob_candidate` (a top-k table repeating a candidate at one
+ position), `engine_logprob_records_incomplete` (records that stop
+ short of the generated text, none at all, or β€” under `echo` β€” no prompt-echo records,
+ any of which would leave the arrays describing a prefix), and
+ `engine_pinned_prefix_not_echoed` (a KV slot holding a pin snapshot, so the echoed
+ positions would not start at 0). No logprobs-bearing response is ever a truncated or
+ half-described 200.
+- **Server-side buffering.** A request that asks to see the echoed prompt (`echo: true` on
+ `/v1/completions`) has its full echo table held in memory for the request's lifetime; no
+ streaming is allowed together with `logprobs`, so that hold is bounded by one
+ non-streaming response. A logprobs request that does not ask for the echo β€” all of chat,
+ and every `echo: false` completion β€” retains nothing: the engine still sends an ECHO frame
+ for every prompt position, and they are dropped as they arrive rather than held.
+
### Tool-calling support
| Engine | OpenAI `tools` | Anthropic `tool_use` | Native format |