Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(server): logprobs, echo and top_logprobs on the two completion endpoints
monotophic42f8d80da1e4
diff --git a/c/openai_server.py b/c/openai_server.pyindex c9a93a9c..729ded8c 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -72,11 +72,63 @@ class ClientCancelled(Exception):pass+class EchoPrefixPinned(Exception):+ """The engine echoed a contiguous run of prompt positions that does not start at 0,+ which is the shape a pin snapshot produces: the snapshot already covers `prefix`+ tokens, so the prefill reads out `prefix..nt-1` and there is nothing to read out+ before that.++ This is not corruption and must not be answered as if it were, but it is also not+ something this surface can serve: `echo: true` promises a logprob and a text offset for+ every prompt position. Re-basing the run to start at 0 would attach every logprob to+ the wrong prompt token -- a 200 carrying silently wrong data -- and filling the gap is+ not available either, since the engine holds no logits for a pinned prefix. So the+ caller is told by name what the engine could not do. Carries `prefix` so the message+ can name it."""++ def __init__(self, prefix):+ super().__init__(f"the engine echoed prompt positions from {prefix}, not 0")+ self.prefix = prefix++def error_object(error):return {"error": {"message": error.message, "type": error.error_type,"param": error.param, "code": error.code}}+def _malformed_logprob_tail(detail):+ """The engine's numeric tail did not parse, named for the client.++ Every shape of it is the same event from the client's side: the engine sent a per-token+ logprob frame this gateway cannot read, so the numbers the request asked for do not+ exist. `engine_logprob_tail_malformed` says what the engine did rather than naming the+ reaction to it, so the code stays true if the reaction ever changes."""+ return APIError(500, "The colibri engine sent a malformed per-token logprob "+ "tail: %s" % detail, "logprobs",+ "engine_logprob_tail_malformed", "server_error")+++def _malformed_echo_position(detail):+ """The `pos` field of an ECHO frame did not parse, named for the client.++ Its own code rather than the numeric tail's: the two are different fields of the frame+ and a client branching on the code should not be told the tail was unreadable when the+ position was."""+ return APIError(500, "The colibri engine sent an unreadable prompt-echo position: %s"+ % detail, "echo", "engine_echo_position_malformed", "server_error")+++def _with_engine_reason(fault, message):+ """`fault` again, with the engine's own terminal error text appended.++ The recorded fault happened first and names the framing defect, so it is what the client+ is told; the engine's parting word is kept in the message rather than dropped, because it+ is often the only account of what the turn did after the frame this server could not+ read."""+ return APIError(fault.status, f"{fault.message} The engine then reported: {message}",+ fault.param, fault.code, fault.error_type)++def _engine_error(fields, message):"""Turn an engine ERROR frame into the right exception type.@@ -573,7 +625,7 @@ def _marker_cuts(text, marker):return cuts-def _tool_call_content_spans(reply, tail):+def _tool_call_content_spans(reply, tail, track_spans=True):"""parse_tool_calls' content derivation as `(content, box_map, content_map)`, both mapsin `reply`'s own coordinates.@@ -585,39 +637,54 @@ def _tool_call_content_spans(reply, tail):Both maps are None for inkling, whose marker stripping is not modelled here: an inklingengine does not support the numeric logprobs channel, so no logprobs object can reach- this text to be aligned."""+ this text to be aligned. They are also None when `track_spans` is off, which is how a+ request that never asked for logprobs skips composing them.++ The per-step survivor lists `_apply_cuts` returns are not gated, because they are the+ deletion itself -- the text is built by slicing them -- rather than a map kept for a+ later reader. Only the composition and the exported maps are optional."""+ def chain(previous, step):+ return previous and track_spans and _compose_span_maps(previous, step)+cuts = [(match.start(), match.end()) for match in _BOX_RE.finditer(reply)]text, box_map = _apply_cuts(reply, cuts)if tail is not None: # drop the recovered tail from the visible contenttext, step = _apply_cuts(text, [(text.rindex(BOX_START), len(text))])- box_map = _compose_span_maps(box_map, step)+ box_map = chain(box_map, step)content_map = box_mapif ARCH == "inkling":text = strip_inkling_markers(text) # thinking is reasoning, not answerbox_map = content_map = Noneif THINK_CLOSE in text:text, step = _apply_cuts(text, [(0, text.index(THINK_CLOSE) + len(THINK_CLOSE))])- content_map = content_map and _compose_span_maps(content_map, step)+ content_map = chain(content_map, step)for marker in (THINK_OPEN, THINK_CLOSE):text, step = _apply_cuts(text, _marker_cuts(text, marker))- content_map = content_map and _compose_span_maps(content_map, step)+ content_map = chain(content_map, step)stripped = text.strip()head = len(text) - len(text.lstrip())text, step = _apply_cuts(text, [(0, head), (head + len(stripped), len(text))])- content_map = content_map and _compose_span_maps(content_map, step)+ content_map = chain(content_map, step)+ if not track_spans:+ return text, None, Nonereturn text, box_map, content_mapdef parse_tool_calls(reply, tools=None):"""Return (content, tool_calls). Strict GLM parse; optional de-mangler (COLI_TOOL_SALVAGE=1)rescues malformed int4 output by mapping a lone payload onto the tool's primary parameter."""- content, calls, _box_map, _content_map = parse_tool_calls_spans(reply, tools)+ content, calls, _box_map, _content_map = _parse_tool_calls(reply, tools, False)return content, callsdef parse_tool_calls_spans(reply, tools=None):- """parse_tool_calls plus the tool-call stage's maps: `(content, tool_calls, box_map,- content_map)`. One implementation, so the maps cannot drift from the text."""+ """parse_tool_calls plus the tool-call stage's maps."""+ return _parse_tool_calls(reply, tools, True)+++def _parse_tool_calls(reply, tools, track_spans):+ """The parse itself: `(content, tool_calls, box_map, content_map)`. One implementation+ behind both spellings, so the maps cannot drift from the text they describe."""param_order = _tool_param_order(tools)param_types = _tool_param_types(tools)calls, salvaged = [], []@@ -660,7 +727,7 @@ def parse_tool_calls_spans(reply, tools=None):sys.stderr.write("[api] tools declared and tool-call markers present, but no call ""parsed -- output may be quantization-mangled; try COLI_TOOL_SALVAGE=1\n")sys.stderr.flush()- text, box_map, content_map = _tool_call_content_spans(reply, tail)+ text, box_map, content_map = _tool_call_content_spans(reply, tail, track_spans)if calls:dm, rec = len(salvaged), (1 if tail is not None else 0)sys.stderr.write("[api] tool-calls: %d total, %d strict, %d unclosed-recovered, "@@ -831,11 +898,17 @@ def parse_dsv41_tool_calls(reply):def parse_arch_tool_calls(reply, tools, tool_reply=None):"""Architecture-appropriate tool-call parser. Returns (content, tool_calls)."""- content, calls, _box_map, _content_map = parse_arch_tool_calls_spans(reply, tools, tool_reply)+ content, calls, _box_map, _content_map = _parse_arch_tool_calls(+ reply, tools, tool_reply, False)return content, callsdef parse_arch_tool_calls_spans(reply, tools, tool_reply=None):+ """parse_arch_tool_calls plus the tool-call stage's maps."""+ return _parse_arch_tool_calls(reply, tools, tool_reply, True)+++def _parse_arch_tool_calls(reply, tools, tool_reply, track_spans):"""parse_arch_tool_calls plus the tool-call stage's maps: `(content, tool_calls,box_map, content_map)`.@@ -854,7 +927,7 @@ def parse_arch_tool_calls_spans(reply, tools, tool_reply=None):return parse_k3_tool_calls(reply, tools) + (None, None) # pre-#1147 enginesif ARCH == "qwen38":return parse_qwen38_tool_calls(reply, tools) + (None, None)- return parse_tool_calls_spans(reply, tools)+ return _parse_tool_calls(reply, tools, track_spans)def _tool_stream_markers():@@ -2795,12 +2868,12 @@ class ThinkingStreamSplit:coordinates. Every character is routed to exactly one bucket or is marker syntaxrouted to neither, and which bucket decides which array a token record belongs to.- Unlike the stop filter's map these are built unconditionally: they cost two list- appends and a merge per emitted run, which is not worth a flag and a second code path- through the splitter."""+ Like the stop filter's map they are built only when `track_spans` is set, so a request+ that never asked for per-token logprobs pays for none of this."""MARKERS = (THINK_OPEN, THINK_CLOSE)- def __init__(self, on_thinking, on_text, on_thinking_end=None, initial_thinking=True):+ def __init__(self, on_thinking, on_text, on_thinking_end=None, initial_thinking=True,+ track_spans=False):self.on_thinking = on_thinkingself.on_text = on_textself.on_thinking_end = on_thinking_end@@ -2809,6 +2882,7 @@ class ThinkingStreamSplit:# the splitter must start in text mode or it would file the whole answer as reasoning.self.thinking = initial_thinkingself.buf = ""+ self.track_spans = track_spansself.thinking_spans = []self.text_spans = []self.consumed = 0 # input characters resolved, marker or not@@ -2816,6 +2890,9 @@ class ThinkingStreamSplit:def _emit(self, text):if text:(self.on_thinking if self.thinking else self.on_text)(text)+ if not self.track_spans:+ return+ if text:spans = self.thinking_spans if self.thinking else self.text_spans_append_span(spans, self.consumed, self.consumed + len(text),spans[-1][2] + (spans[-1][1] - spans[-1][0]) if spans else 0)@@ -2832,7 +2909,8 @@ class ThinkingStreamSplit:offset, marker = min(hits, key=lambda hit: hit[0])self._emit(self.buf[:offset])self.buf = self.buf[offset + len(marker):]- self.consumed += len(marker) # the marker itself reaches neither bucket+ if self.track_spans:+ self.consumed += len(marker) # the marker itself reaches neither bucketif marker == THINK_CLOSE and self.thinking:self.thinking = Falseif self.on_thinking_end:@@ -2858,18 +2936,25 @@ class ThinkingStreamSplit:def split_thinking_reply(text, enable_thinking=True, add_generation_prompt=True):"""Return the marker-free (thinking, answer) portions of one GLM reply."""- thinking, answer, _spans = split_thinking_reply_spans(text, enable_thinking,- add_generation_prompt)+ thinking, answer, _spans = _split_thinking(text, enable_thinking, False,+ add_generation_prompt)return thinking, answerdef split_thinking_reply_spans(text, enable_thinking=True, add_generation_prompt=True):"""split_thinking_reply plus the split's maps: `(thinking, answer, (thinking_map,answer_map))`, both in `text`'s own coordinates."""+ return _split_thinking(text, enable_thinking, True, add_generation_prompt)+++def _split_thinking(text, enable_thinking, track_spans, add_generation_prompt=True):+ """The split itself. One implementation behind both spellings, so the maps can never+ describe a different routing than the text they came with."""thinking, answer = [], []split = ThinkingStreamSplit(thinking.append, answer.append,initial_thinking=starts_in_reasoning(enable_thinking,- add_generation_prompt))+ add_generation_prompt),+ track_spans=track_spans)split.feed(text)split.finish()return "".join(thinking), "".join(answer), (split.thinking_spans, split.text_spans)@@ -3119,8 +3204,9 @@ class StopFilter:`spans` is this stage's map: the intervals of the raw stream -- the concatenation ofeverything ever fed -- that reached `emit`, each with where it landed in the emitted- text. It is built only when `track_spans` is set, so a request that never asked for- per-token logprobs pays nothing for it."""+ text. It is built only when `track_spans` is set, and the thinking split and the+ tool-call parse are gated the same way, so a request that never asked for per-token+ logprobs has no stage compose or retain a map for it."""def __init__(self, sequences, emit, ignore_leading=False, track_spans=False):self.sequences = tuple(sequences)self.emit = emit@@ -3269,8 +3355,6 @@ def generation_options(body, limit):if choice != "none" and not (body.get("tools") or body.get("functions")):raise APIError(400, "`tool_choice` requires `tools`.", "tool_choice", "invalid_value")stop_sequences = parse_stop_sequences(body)- if body.get("logprobs"):- raise APIError(400, "Log probabilities are not supported yet.", "logprobs", "unsupported_parameter")if body.get("frequency_penalty", 0) or body.get("presence_penalty", 0):raise APIError(400, "Token penalties are not supported yet.", None, "unsupported_parameter")if body.get("seed") is not None:@@ -3346,6 +3430,494 @@ def generation_options(body, limit):return maximum, float(temperature), float(top_p), grammar, stop_sequences+# The per-token top-k emission cap the engine's numeric logprobs channel (the SUBMIT+# `logprobs=` key) supports, mirrored in c/decode_batch.h as COLI_SUBMIT_TOPK_MAX. The+# public request range is bound to exactly that interface: 1..32, a named 400 above it.+LOGPROBS_TOP_K_CAP = 32+++def logprobs_options(body, chat, engine_supports):+ """Validate the request's logprobs/echo/top_logprobs fields and translate them into+ `(engine_k, echo, display_k)`.++ - `engine_k` is the SUBMIT `logprobs=` value to send; 0 leaves the per-token numeric+ channel off entirely, so no ECHO or extended DATA frames are emitted.+ - `echo` is whether the response includes the prompt-echo positions. Chat has no echo+ concept, so it is always False there and `echo: true` on chat is a named 400.+ - `display_k` is how many `top_logprobs` alternatives the client asked to see, 0 to+ LOGPROBS_TOP_K_CAP. It can be lower than `engine_k` when a chat request asks for+ logprobs with `top_logprobs: 0`.++ Completions' `logprobs` is the legacy integer top-k count; chat's is a boolean gate+ plus a separate `top_logprobs` count. Each endpoint takes its own shape and refuses+ the other's by name.++ `null`, `false` and `0` all normalise to absent and are never errors: on completions+ the request then succeeds with `choices[].logprobs: null`, and on chat `true` for+ `logprobs` is the only value that opens the gate. `top_logprobs` is type- and+ range-checked even when `logprobs` is off, so a malformed value is never silently+ ignored because a sibling field made it moot; a valid one with `logprobs` off is a+ documented no-op. `top_logprobs` is not read at all on completions, where it is not+ part of the request shape.++ `echo: true` with no active `logprobs` request is a named 400 rather than a silent+ no-op: the prompt echo is built out of the engine's per-token records, so without them+ there is nothing to echo, and a 200 that quietly drops a requested field is the shape+ this surface exists to stop."""+ def _require_logprobs_for_echo(wanted):+ if wanted:+ raise APIError(400, "`echo` requires `logprobs`.", "echo",+ "unsupported_parameter")++ echo = body.get("echo", False)+ if echo is None:+ echo = False # null == absent, same as `logprobs`+ if not isinstance(echo, bool):+ raise APIError(400, "`echo` must be a boolean.", "echo", "invalid_value")+ if chat:+ if echo:+ raise APIError(400, "`echo` is not supported for chat completions.",+ "echo", "unsupported_parameter")+ logprobs = body.get("logprobs", False)+ if logprobs is None:+ logprobs = False # null == absent == no logprobs+ if not isinstance(logprobs, bool):+ raise APIError(400, "`logprobs` must be a boolean.", "logprobs", "invalid_value")+ display_k, param = body.get("top_logprobs", 0), "top_logprobs"+ if display_k is None:+ display_k = 0 # null == absent, same as `logprobs`+ else:+ logprobs = body.get("logprobs")+ if logprobs is None or logprobs is False:+ _require_logprobs_for_echo(echo)+ return 0, False, 0+ display_k, param = logprobs, "logprobs"+ if (isinstance(display_k, bool) or not isinstance(display_k, int) or+ not 0 <= display_k <= LOGPROBS_TOP_K_CAP):+ raise APIError(400, f"`{param}` must be an integer between 0 and "+ f"{LOGPROBS_TOP_K_CAP}.", param, "invalid_value")+ if chat and not logprobs:+ return 0, False, 0 # top_logprobs validated above; logprobs off is still a no-op+ if not chat and display_k == 0:+ _require_logprobs_for_echo(echo)+ return 0, False, 0 # 0 = no logprobs, documented+ if not engine_supports:+ # A sender-side capability gate: these endpoints request the numeric per-token+ # channel only from an engine this server has pinned for it, and refusing before+ # generate() builds the extension header keeps a rejected-SUBMIT payload drain+ # unreachable. The message names the gate that refused, not the engine's arch.+ raise APIError(400, "Log probabilities are not requested from this engine by "+ "these endpoints.", "logprobs", "unsupported_parameter")+ return max(1, display_k), echo, display_k+++def _json_float(value):+ """A logprob the engine's numeric channel emits as nan/inf/-inf must serialise as JSON+ `null`, never the invalid-JSON literals json.dumps would otherwise write and never+ clamped to a made-up finite number."""+ return value if math.isfinite(value) else None+++def _order_echo_records(prompt_records):+ """Place each prompt-echo record at its own wire `pos` index rather than trusting the+ order the frames arrived in, and return `(payload bytes, record)` pairs -- the same+ shape generated-token records already arrive in.++ A duplicate, negative, out-of-range or (by pigeonhole) missing position among the N+ records that must fill slots 0..N-1 raises RuntimeError, the class every other+ malformed-engine-output path here raises. One shape is told apart from that: a+ contiguous run starting above 0, which is what a pin snapshot produces rather than a+ malformed engine. That raises EchoPrefixPinned, which the caller answers by name."""+ positions = [record["pos"] for record in prompt_records]+ if (positions and all(isinstance(p, int) and not isinstance(p, bool)+ for p in positions)+ and sorted(positions) == list(range(min(positions),+ min(positions) + len(positions)))+ and min(positions) > 0):+ raise EchoPrefixPinned(min(positions))+ ordered = [None] * len(prompt_records)+ for record in prompt_records:+ pos = record["pos"]+ if (not isinstance(pos, int) or isinstance(pos, bool)+ or not 0 <= pos < len(ordered) or ordered[pos] is not None):+ raise RuntimeError(+ f"invalid engine ECHO position {pos!r} (expected each of "+ f"0..{len(ordered) - 1} exactly once, got {len(prompt_records)} records)")+ ordered[pos] = (record["bytes"], record)+ return ordered+++def _generated_record_texts(generated_records):+ """Each generated record's own decoded text, from one stateful incremental UTF-8+ decoder over the generated sequence alone, with the trailing flush landing on the last+ record.++ This is the decode the engine's `on_text` callback performed while the stop filter was+ watching -- same bytes, same fresh decoder, same final flush -- so the running sum of+ these lengths is the raw-stream character coordinate the filter's span map is expressed+ in. Nothing else may be used to derive record spans."""+ decoder = codecs.getincrementaldecoder("utf-8")("replace")+ texts = [decoder.decode(data) for data, _record in generated_records]+ tail = decoder.decode(b"", final=True)+ if tail and texts:+ texts[-1] += tail+ return texts+++def _echo_decoded_texts(prompt_records, generated_records):+ """`(prompt texts, generated texts)` from a single stateful incremental UTF-8 decoder+ spanning both halves, prompt-echo positions first.++ A codepoint can arrive split across the prompt/generated seam: its leading byte on the+ last echo frame, its continuation bytes on the first data frame. One decoder spanning+ both resolves that into one character; two independent decodes turn each half into+ replacement characters, and `text` and `text_offset` then disagree about how long that+ character is. This is therefore not the same decode as _generated_record_texts: the+ span map is counted in that function's coordinates, while these texts are what the+ echo reconstruction shows."""+ decoder = codecs.getincrementaldecoder("utf-8")("replace")+ prompt_texts = [decoder.decode(data) for data, _record in _order_echo_records(prompt_records)]+ generated_texts = [decoder.decode(data) for data, _record in generated_records]+ tail = decoder.decode(b"", final=True)+ if tail:+ if generated_texts:+ generated_texts[-1] += tail+ elif prompt_texts:+ prompt_texts[-1] += tail+ return prompt_texts, generated_texts+++def _logprob_positions(prompt_records, generated_records, texts=None):+ """Merge prompt-echo and generated-token records into one ordered list of+ `{"text", "raw_lp", "topk", "bytes"}` dicts: echoed positions first, reassembled by+ wire `pos`, then generated tokens in emission order.++ `texts` supplies each position's text when the caller already decoded them -- the+ response path has, because it needs the generated half truncated to what the stop+ filter emitted. `bytes` always stays that frame's own raw payload regardless of what+ text, if any, it decoded to."""+ raw = _order_echo_records(prompt_records)+ raw += generated_records+ if texts is None:+ prompt_texts, generated_texts = _echo_decoded_texts(prompt_records, generated_records)+ texts = prompt_texts + generated_texts+ return [{"text": text, "raw_lp": record["lp"], "topk": record["topk"], "bytes": data}+ for (data, record), text in zip(raw, texts)]+++def _own_token_label(entry, topk, idx):+ """The label for one top-k candidate: the position's own text when the candidate is+ the chosen token, otherwise `<token_id:N>`.++ The wire's top-k table carries raw candidate token ids and no decoded text, and there+ is no server-side tokenizer. The one candidate that can be labelled without decoding is+ the position's own chosen token: its table entry and its `lp` come from the same+ computation, so an exact float match identifies it without comparing token ids.++ The engine prints logprobs to six decimal digits, so two candidates can legitimately+ share the chosen token's printed value. With no id to break the tie, only the first+ table entry in wire order whose value matches is labelled as the chosen token; a later+ match is labelled by its raw id like any other unidentified candidate. The table is+ unsorted on the wire, so this never assumes the first entry is the argmax."""+ tid, tlp = topk[idx]+ if tlp != entry["raw_lp"]:+ return f"<token_id:{tid}>"+ if any(topk[j][1] == entry["raw_lp"] for j in range(idx)):+ return f"<token_id:{tid}>"+ return entry["text"]+++def _refuse_duplicate_candidate(label, seen, requested, param):+ """Refuse a top-k table that repeats a label within one position.++ The two response shapes would answer it differently: the legacy completions+ `top_logprobs` is a JSON object, so the second entry overwrites the first and the+ client silently receives fewer candidates than the engine supplied, while chat's+ `content[].top_logprobs` is a list and keeps both. Refused on both surfaces, because a+ client cannot tell the two apart from the outside.++ `engine_duplicate_logprob_candidate` describes the table the engine sent rather than+ this function's reaction to it. `param` is the caller's own parameter -- `logprobs` on+ legacy completions, `top_logprobs` on chat -- so a client branching on the code is told+ which of its fields is unanswerable."""+ if label in seen:+ raise APIError(+ 500, f"The colibri engine sent a top-k table carrying the duplicate "+ f"candidate {label!r} at a single position ({requested} "+ f"requested).", param, "engine_duplicate_logprob_candidate",+ "server_error")+ seen.add(label)+++def _completions_logprobs_object(prompt_records, generated_records, display_k, texts=None):+ """Build the legacy `/v1/completions` `logprobs` object: `tokens[]`,+ `token_logprobs[]`, `top_logprobs[]` (one dict per position, keyed by token text) and+ `text_offset[]` (character offsets into the reconstructed text, counted from 0).++ The first prompt position's `token_logprobs` entry comes out null because the engine's+ own echo position 0 carries a non-finite sentinel -- there is nothing to condition the+ first token on. `texts` is each position's text when the caller already has it, and+ `text_offset` is counted from those same strings, so the offsets always index the text+ this object's own `tokens` join to."""+ positions = _logprob_positions(prompt_records, generated_records, texts)+ tokens = [p["text"] for p in positions]+ token_logprobs = [_json_float(p["raw_lp"]) for p in positions]+ top_logprobs = []+ for p in positions:+ table = {}+ displayed = p["topk"][:display_k]+ seen = set()+ for idx, (tid, tlp) in enumerate(displayed):+ label = _own_token_label(p, displayed, idx)+ _refuse_duplicate_candidate(label, seen, len(displayed), "logprobs")+ table[label] = _json_float(tlp)+ top_logprobs.append(table)+ text_offset = []+ offset = 0+ for text in tokens:+ text_offset.append(offset)+ offset += len(text)+ return {"tokens": tokens, "token_logprobs": token_logprobs,+ "top_logprobs": top_logprobs, "text_offset": text_offset}+++def _chat_logprobs_content(generated_records, display_k, texts=None):+ """Build chat completions' `choices[].logprobs.content[]`: one+ `{token, logprob, bytes, top_logprobs}` entry per generated token, each alternative a+ `{token, logprob, bytes}`. Chat has no echo concept, so records are used in emission+ order.++ Every record's text is decoded first, in one pass, including the final flush, before+ any entry is built: building entries interleaved with decoding would let the last+ entry's `token` gain the flushed text while its own candidate label stayed stale.+ `texts` supplies each entry's token text when the caller already has it -- the chat+ path passes the text each record contributed to `message.content`, which is a slice of+ that record's own decoding wherever a stop, a thinking split or a tool-call parse cut+ through the middle of a token."""+ if texts is None:+ texts = _generated_record_texts(generated_records)+ content = []+ for (data, record), text in zip(generated_records, texts):+ entry = {"text": text, "raw_lp": record["lp"], "topk": record["topk"]}+ alternatives = []+ displayed = record["topk"][:display_k]+ seen = set()+ for idx, (tid, tlp) in enumerate(displayed):+ label = _own_token_label(entry, displayed, idx)+ _refuse_duplicate_candidate(label, seen, len(displayed), "top_logprobs")+ alternatives.append({"token": label, "logprob": _json_float(tlp),+ "bytes": list(data) if label == text else None})+ content.append({"token": text, "logprob": _json_float(record["lp"]),+ "bytes": list(data), "top_logprobs": alternatives})+ return content+++def _records_cover_stream(generated_records, text, span_map):+ """True when the generated records account for the whole raw stream the filter emitted+ from -- the question every logprobs-bearing response must answer before its arrays may+ claim to describe the text.++ Both sides are character counts in raw-stream coordinates. The records' extent is the+ running sum _generated_record_texts() produces, which is the coordinate system the span+ map is expressed in; the filter's reach is the far end of that map. No decoded string+ is compared against another anywhere here, which is what keeps a legitimately+ transformed text -- an unflushed decoder, or the echo reconstruction's own seam decode+ -- from reading as a gap and turning a working request into a 500.++ `text` is read only when the map is empty, where it is the one thing that tells+ "nothing was emitted" apart from "nothing was recorded". `span_map` takes no default: a+ degenerate "assume everything was emitted" arm here would be unreachable and therefore+ untested, which is the worst thing a safety check can contain.++ No records at all is a gap like any other whenever text was emitted: the arrays would+ describe nothing while the response carries a completion. It is only vacuously covered+ when nothing was emitted either."""+ extent = sum(len(piece) for piece in _generated_record_texts(generated_records))+ if not span_map:+ # Not vacuously true: an empty map means either that nothing was emitted -- and+ # `text` is then empty too -- or that a filter was built without `track_spans`+ # while text was emitted. `text` tells the two apart.+ return not text+ # Every span, not just the last: a map can carry a hole, and a run that ends inside the+ # records while an earlier one runs past them is still a gap.+ return all(high <= extent for _low, high, _out in span_map)+++def _align_generated_records(generated_records, text, span_map, display_texts=None):+ """Locate every generated-token record in `text` by raw-stream span and return+ `(data, record, emitted_text)` for each record that contributed at least one character+ to it, in `text` order, which is also record order.++ `span_map` is the composed map from raw-stream coordinates to `text`'s: the stop+ filter's `spans` alone when `text` is the stop-filtered stream, or that chained with+ the thinking split's and the tool-call parse's when `text` is `message.content`. Each+ record occupies `[sum(len(decoded_j) for j < k), + len(decoded_k))`, and where that+ lands is a lookup rather than a prefix match of decoded bytes against a string the+ stage has already rewritten.++ A record whose span is fully inside the map is kept whole; fully outside, dropped;+ partially inside, kept with the characters that were emitted. A kept record's `data` is+ narrowed to the bytes of the characters it reports whenever it was truncated, so+ `bytes` can never describe more than `token`.++ `display_texts` is the echo reconstruction's own decode, where the generated half is+ decoded by the decoder that already consumed the prompt bytes. A record that survives+ whole reports that reading directly; a record that is both cut and straddling has the+ real character spliced back onto the raw-stream slice, which is exact because the two+ decodes differ only over that leading run. A cut falling inside the straddling+ codepoint itself has no right answer, and keeps the raw slice.++ `span_map` takes no default; None means "every character of `text` was emitted, from+ the start of the stream", which a caller with no filter in front of it must ask for+ rather than fall into."""+ if span_map is None:+ span_map = [(0, len(text), 0)] if text else []+ texts = _generated_record_texts(generated_records)+ aligned, offset = [], 0+ for index, (data, record) in enumerate(generated_records):+ piece = texts[index]+ start, offset = offset, offset + len(piece)+ if not piece:+ # A frame that only completes a pending codepoint decodes to "" and owns no+ # span, so its fate is that of the character its bytes went into. Testing the+ # position instead keeps the entry even when that character was deleted.+ if _project_span(span_map, start, start + 1):+ aligned.append((data, record, ""))+ continue+ image = _project_span(span_map, start, offset)+ if image is None:+ continue+ kept = image[1] - image[0]+ emitted = text[image[0]:image[1]]+ alternate = display_texts[index] if display_texts is not None else piece+ if alternate != piece:+ if kept == len(piece):+ # Survived whole: the seam decode is this record's text. Continuation+ # bytes that complete nothing give `alternate == ""`, which is right.+ emitted = alternate+ else:+ # Cut and straddling: the raw-stream decode opens with `head` replacement+ # characters where the seam decode has one real one, agreeing from there+ # on. Splice it back only where that is exact.+ head = len(piece) - len(alternate) + 1+ if kept >= head and _project_span(span_map, start, start + kept) == image:+ emitted = alternate[:1] + emitted[head:]+ if emitted != piece:+ # `bytes` must never describe more than `token`, so a truncated entry carries+ # the bytes it reports; an untruncated one keeps the frame's own payload.+ data = emitted.encode("utf-8")+ aligned.append((data, record, emitted))+ return aligned+++def _completion_choice_fields(stats, *, raw_text, text, chat, engine_k, echo, display_k,+ prompt_echoes, stop_spans, content_spans, cache_slot):+ """The stop/trim/echo/logprobs tail every non-streaming completion shares, in one+ place. Returns `(text, logprobs_obj, finish_reason)`.++ Callers differ only in what they do before this point -- the inkling split, the+ thinking split, the tool-call parse and the tool sideband. After it they differ in+ nothing, so the stop/trim/echo interaction exists here once instead of once per+ completion shape.++ `raw_text` is the pre-split, pre-tool-parse text the stop filter emitted; `text` is+ what the caller intends to return, after whichever splits it applies. They are the same+ string when no split fired. `chat` selects the chat `content`/`refusal` shape over the+ legacy completions object; `engine_k`, `echo` and `display_k` come from+ logprobs_options(). `prompt_echoes` is the echo records the caller collected through+ its own `on_echo` sink. `cache_slot` is carried only so a pinned prefix can be named in+ the one error that reports it.++ `stop_spans` is the stop filter's own map, raw stream to `raw_text`; `content_spans` is+ `(target text, composed map)` chaining it onward to the string the arrays must describe+ -- `message.content` on the chat path -- or None when no further stage can vouch for+ one. Every parameter but `stats` is keyword-only, and the span parameters take no+ default: `raw_text` and `text` are same-typed neighbours whose transposition is silent,+ and a defaulted span would leave whichever call site forgot to pass one on unaligned+ behaviour while the other was fixed.++ This holds no state between calls. Every incremental decoder involved is created per+ call and the span maps arrive as arguments, so a caller that assembles several+ completions in a row cannot carry one's trailing partial codepoint or coordinates into+ the next.++ `finish_reason` is the engine's own length/stop reason; a caller that can finish for a+ reason of its own overrides it."""+ finish_reason = "length" if stats["length_limited"] else "stop"+ logprobs_obj = None+ if engine_k:+ channel = stats.get("logprobs") or {"generated": []}+ prompt_records = prompt_echoes if echo else []+ target, target_spans = ((content_spans[0], content_spans[1]) if content_spans+ else (raw_text, stop_spans))+ # The arrays describe the WHOLE text on every shape that carries them, not only+ # the one that rebuilds `text` out of them: short records, or none at all, would+ # otherwise ship a 200 whose arrays describe a prefix and say nothing about it.+ if not _records_cover_stream(channel["generated"], target, target_spans):+ raise APIError(+ 500, "The colibri engine sent per-token logprob records that do not "+ "cover the generated text, so the logprobs arrays would describe "+ "only part of it.", "echo" if echo else "logprobs",+ "engine_logprob_records_incomplete", "server_error")+ if echo and not prompt_records:+ # An engine that sends the generated records and no ECHO frames has answered+ # half the opt-in, and the prompt echo is the half `echo` asked for.+ raise APIError(+ 500, "The colibri engine sent no prompt-echo records, so `echo` "+ "cannot be answered for this prompt.", "echo",+ "engine_logprob_records_incomplete", "server_error")+ try:+ prompt_texts, seam_texts = _echo_decoded_texts(prompt_records,+ channel["generated"])+ except EchoPrefixPinned as pinned:+ # The engine's own state, not a bad request and not a broken engine: this slot+ # holds a pin snapshot covering the first `prefix` tokens, which the prefill+ # never reads out, so `echo` cannot be answered in full for this prompt.+ raise APIError(+ 503, f"The colibri engine cannot echo this prompt: KV slot "+ f"{cache_slot} holds a pinned prefix of {pinned.prefix} "+ f"token(s), which the prefill does not read out, so the "+ f"echoed positions would start at {pinned.prefix} rather "+ f"than 0.", "echo", "engine_pinned_prefix_not_echoed",+ "server_error") from None+ aligned = _align_generated_records(channel["generated"], target, target_spans,+ display_texts=seam_texts if echo else None)+ generated = [(data, record) for data, record, _emitted in aligned]+ emitted_texts = [emitted for _data, _record, emitted in aligned]+ if chat:+ logprobs_obj = {"content": _chat_logprobs_content(generated, display_k,+ texts=emitted_texts),+ "refusal": None}+ else:+ logprobs_obj = _completions_logprobs_object(+ prompt_records, generated, display_k, texts=prompt_texts + emitted_texts)+ if not chat and echo:+ # The legacy shape concatenates the prompt and the completion in `text` when+ # `echo` is true and `text_offset` indexes that concatenation, so `text` is+ # rebuilt from the reconstruction the offsets are counted from rather than+ # from two independently decoded halves.+ text = "".join(logprobs_obj["tokens"])+ return text, logprobs_obj, finish_reason+++def _engine_extension_args(engine_k):+ """The `Engine.generate()` keyword arguments that carry this request's SUBMIT+ extension, built once from the parsed options and spread at each call site.++ An empty mapping when nothing was requested, which is what keeps a plain request's+ header byte-identical to one built with no extension at all. When the channel is+ requested it also asks for the seventh numeric field, which coli_submit_parse expects+ before the first key=value token.++ That is this helper's decision for THESE endpoints, not a property of the request+ builder. `gbytes_before_ext` is a per-call-site parameter, and the other caller that+ sends an extension key -- /v1/brio -- does not pass it and keeps the header it has+ always sent; changing that endpoint's wire is not this change's to make."""+ if not engine_k:+ return {}+ return {"logprobs": engine_k, "gbytes_before_ext": True}++def read_engine_turn(stream, sentinel, on_bytes):pending = b""while True:@@ -3553,6 +4125,28 @@ def _win_kill_on_close_job(pid):return None # never let process bookkeeping break starting the engine+class _Pending:+ """One in-flight engine request: the queue its frames are delivered on, whether it asked+ for the per-token numeric channel, and any fault the dispatcher has recorded against it.++ The dispatcher parses a frame's numeric tail only for a request that asked for it, so a+ frame carrying fields a request never requested is tolerated exactly as it was before+ the channel existed, and a fault in one request's frames can only fail that request.++ `failed` is how that stays true without abandoning the engine's turn. The entry is kept+ rather than dropped: the request's later frames are routed here and discarded, and the+ error is delivered on its terminal frame, so the thread holding the scheduler admission+ keeps holding it until the engine says the turn is over. Dropping the entry instead+ would free the admission while the engine is still generating, and the next request+ would submit into a pipe nobody is reading."""+ __slots__ = ("events", "logprob_channel", "failed")++ def __init__(self, logprob_channel=False):+ self.events = queue.Queue()+ self.logprob_channel = logprob_channel+ self.failed = None++class Engine:# cap=None = "not explicitly set": a glm-arch model's engine resolves the# 0 sentinel (8 historically, 1 on Metal+darwin+fast SSD -- colibri.c@@ -3572,6 +4166,13 @@ class Engine:arch = family.idself.family = familyself.model_dir = str(model)+ # The extended SUBMIT namespace, gated once at launch rather than per request: an+ # accepted-but-ignored opt-in is a silently wrong answer rather than an error. Two+ # flags, not one -- the numeric logprobs/echo channel (`logprobs=`) and+ # pre-tokenized token-id intake (`ids=`) are unrelated capabilities with separate+ # coli_submit_ext keys, set from one arch check only because one engine has both.+ self.supports_logprobs_echo = (arch == "glm")+ self.supports_tok_ids = (arch == "glm")child_env = dict(env or os.environ, SNAP=str(model), SERVE="1", SERVE_BATCH="1",NGEN=str(max_tokens), KV_SLOTS=str(kv_slots))tune_child_env(child_env, arch)@@ -3626,8 +4227,8 @@ class Engine:with self.pending_lock:requests = list(self.pending.values())self.pending.clear()- for events in requests:- events.put(("error", error))+ for entry in requests:+ entry.events.put(("error", error))def _read_exact(self, size):chunks = []@@ -3640,6 +4241,58 @@ class Engine:remaining -= len(chunk)return b"".join(chunks)+ @staticmethod+ def _parse_logprob_tail(fields, i):+ """Parse the numeric tail an opted-in DATA/ECHO frame carries:+ "<lp> <k> [<tid> <tlp>]*k".++ float() reads the engine's numeric wire tokens directly, "nan"/"inf"/"-inf" and any+ %g precision alike -- an echo's position 0 has nothing to condition on and carries+ "nan 0". The table is unsorted on the wire, so callers must not assume the first+ pair is the argmax. Every malformed shape -- a short or over-long field list,+ non-numeric fields, an out-of-range k, or a negative token id -- is an APIError+ carrying `engine_logprob_tail_malformed`, never a silent partial record.++ Fields past the ones the grammar defines are IGNORED, not refused. Tolerating a+ longer frame is what the base did for every caller, and this parser now runs for+ internal consumers of the channel that read only part of the record."""+ if len(fields) < i + 2:+ raise _malformed_logprob_tail("missing lp/k")+ try:+ lp = float(fields[i])+ k = int(fields[i + 1])+ except ValueError as error:+ raise _malformed_logprob_tail(error) from error+ if not 0 <= k <= LOGPROBS_TOP_K_CAP:+ raise _malformed_logprob_tail(f"k={k} out of range")+ if len(fields) < i + 2 + 2 * k:+ raise _malformed_logprob_tail("fewer candidate fields than k declares")+ topk = []+ j = i + 2+ for _ in range(k):+ try:+ tid = int(fields[j])+ tlp = float(fields[j + 1])+ except ValueError as error:+ raise _malformed_logprob_tail(error) from error+ if tid < 0:+ raise _malformed_logprob_tail(f"negative token id {tid}")+ topk.append((tid, tlp))+ j += 2+ return {"lp": lp, "topk": topk}++ def _record_fault(self, entry, error):+ """Record a per-request fault without ending the request here.++ The entry stays in the pending map, so the engine's later frames for it still land+ somewhere (and are discarded), and `generate()` raises this error when the turn's+ terminal frame arrives. The first fault wins: a stream that goes bad usually goes+ bad more than once, and the first reading is the one that describes what happened.+ The dispatcher itself keeps running, and every other in-flight request is+ untouched."""+ if entry.failed is None:+ entry.failed = error+def _dispatch_stdout(self):try:while True:@@ -3666,9 +4319,20 @@ class Engine:if self._read_exact(1) != b"\n":raise RuntimeError("invalid engine DATA terminator")with self.pending_lock:- events = self.pending.get(request_id)- if events is not None:- events.put(("data", data))+ entry = self.pending.get(request_id)+ if entry is None or entry.failed is not None:+ continue+ record = None+ if entry.logprob_channel and len(fields) > 3:+ # Parsed only for a request that asked, with the payload already+ # drained, so a malformed tail fails that one request rather than+ # stopping the dispatcher.+ try:+ record = self._parse_logprob_tail(fields, 3)+ except APIError as error:+ self._record_fault(entry, error)+ continue+ entry.events.put(("data", data if record is None else (data, record)))elif kind == "TOOL" and len(fields) == 3:# Opaque, request-scoped structured output. K3 emits an# initial zero-byte frame before generation so DATA marker@@ -3681,9 +4345,9 @@ class Engine:if self._read_exact(1) != b"\n":raise RuntimeError("invalid engine TOOL terminator")with self.pending_lock:- events = self.pending.get(request_id)- if events is not None:- events.put(("tool", data))+ entry = self.pending.get(request_id)+ if entry is not None and entry.failed is None:+ entry.events.put(("tool", data))elif kind == "ECHO" and len(fields) >= 6:# U7a prefill read-out: "ECHO <id> <n> <pos> <lp> <k># [tid tlp]*k" plus a DATA-framed payload (n bytes + LF).@@ -3698,33 +4362,71 @@ class Engine:if self._read_exact(1) != b"\n":raise RuntimeError("invalid engine DATA terminator")request_id = fields[1]- lp = fields[4]with self.pending_lock:- events = self.pending.get(request_id)- if events is not None:- events.put(("echo", {- "pos": int(fields[3]),- # " nan 0" = niente su cui condizionare: la prima- # posizione assoluta non ha un predittore.- "logprob": None if lp in ("nan", "-nan") else float(lp),- "text": piece.decode("utf-8", "replace"),- }))+ entry = self.pending.get(request_id)+ if entry is None or entry.failed is not None:+ continue+ try:+ pos = int(fields[3])+ except ValueError:+ # Same frame as the numeric tail, so the same containment: this+ # request's fault, never a stop for every request in flight.+ self._record_fault(entry, _malformed_echo_position(repr(fields[3])))+ continue+ lp = fields[4]+ try:+ # `lp` is the numeric tail's own first field, so an unreadable one+ # is a malformed tail and carries that name. It is read for every+ # ECHO frame, opted in or not, because the closed-set scorer's+ # `logprob` key comes from it -- which is why it needs the same+ # containment as the position beside it, rather than reaching the+ # dispatcher's blanket handler and stopping it for every request.+ logprob = None if lp in ("nan", "-nan") else float(lp)+ except ValueError:+ self._record_fault(entry, _malformed_logprob_tail(+ "unreadable prompt-echo log probability %s" % (lp,)))+ continue+ record = None+ if entry.logprob_channel:+ try:+ record = self._parse_logprob_tail(fields, 4)+ except APIError as error:+ self._record_fault(entry, error)+ continue+ entry.events.put(("echo", {+ "pos": pos,+ # " nan 0" = niente su cui condizionare: la prima+ # posizione assoluta non ha un predittore.+ "logprob": logprob,+ "text": piece.decode("utf-8", "replace"),+ # The payload bytes and the numeric tail: the tail is the only+ # place the candidate ids exist, and text offsets are rebuilt from+ # raw bytes by one decoder spanning the whole sequence. A consumer+ # reading only `pos`/`logprob`/`text` is unaffected.+ "bytes": piece,+ "lp": record["lp"] if record else None,+ "topk": record["topk"] if record else [],+ }))elif kind == "ACCEPT" and len(fields) >= 3:# #597: the engine validated the submission (fits context) before prefill.# Keep it pending — DATA/DONE still follow — and let generate() commit the# HTTP stream only now, so an earlier CONTEXT_EXCEEDED stays a clean 400.request_id = fields[1]with self.pending_lock:- events = self.pending.get(request_id)- if events is not None:- events.put(("accept", {"prompt_tokens": int(fields[2])}))+ entry = self.pending.get(request_id)+ if entry is not None and entry.failed is None:+ entry.events.put(("accept", {"prompt_tokens": int(fields[2])}))elif kind == "DONE" and len(fields) >= 7:request_id = fields[1]stats = self._stats(fields[2:])with self.pending_lock:- events = self.pending.pop(request_id, None)- if events is not None:- events.put(("done", stats))+ entry = self.pending.pop(request_id, None)+ if entry is not None:+ # The turn is over, so a fault recorded mid-turn is delivered now+ # rather than when it was found, which would have freed the+ # admission while the engine was still generating.+ entry.events.put(("error", entry.failed) if entry.failed+ else ("done", stats))elif kind == "HWINFO" and len(fields) >= 7:parts = " ".join(fields[6:]).split("|")self.hwinfo = {"cores": int(fields[1]), "ram_total_gb": float(fields[2]),@@ -3759,9 +4461,18 @@ class Engine:request_id = fields[1]message = " ".join(fields[2:]) or "engine request failed"with self.pending_lock:- events = self.pending.pop(request_id, None)- if events is not None:- events.put(("error", _engine_error(fields[2:], message)))+ entry = self.pending.pop(request_id, None)+ if entry is not None:+ reported = _engine_error(fields[2:], message)+ if entry.failed is not None and not (+ isinstance(reported, RuntimeError)+ and str(reported) == "CANCELLED"):+ # The recorded fault happened first and names the framing+ # defect, so it wins -- except against a cancellation, which+ # is the client leaving and is answered here exactly as it is+ # on a healthy turn.+ reported = _with_engine_reason(entry.failed, message)+ entry.events.put(("error", reported))else:raise RuntimeError(f"invalid engine response: {' '.join(fields)}")except Exception as error:@@ -3771,7 +4482,8 @@ class Engine:def generate(self, prompt, max_tokens, temperature, top_p, on_text, cache_slot=0,cancelled=None, grammar=None, stopped=None, on_accept=None, audio=None,- on_tool=None, image=None, logprobs=0, pin=False, on_echo=None):+ on_tool=None, image=None, logprobs=0, pin=False, on_echo=None,+ gbytes_before_ext=False):if isinstance(cache_slot, bool) or not isinstance(cache_slot, int) or not 0 <= cache_slot < self.kv_slots:raise APIError(400, "Invalid cache slot.", "cache_slot")payload = prompt.encode("utf-8")@@ -3801,7 +4513,8 @@ class Engine:# request's sideband authoritative even when no call follows.on_tool(text)- events = queue.Queue()+ pending = _Pending(bool(logprobs))+ events = pending.eventswith self.pending_lock:if self.closed:raise RuntimeError("colibri engine is shutting down")@@ -3811,7 +4524,7 @@ class Engine:raise RuntimeError("colibri engine is not running")request_id = str(self.next_request_id)self.next_request_id += 1- self.pending[request_id] = events+ self.pending[request_id] = pendingxpayload = gpayload or apayload# DeepSeek V4 prefix hint (optional 8th header field): the byte length of# the rendered prompt up to the first user/assistant turn marker — the@@ -3836,9 +4549,14 @@ class Engine:ext += f" logprobs={int(logprobs)}"if pin:ext += " pin=1"+ # `gbytes_before_ext` asks for the 7th numeric field ahead of the extension keys+ # even with no grammar or audio payload to describe. It defaults to False, so+ # every caller that does not ask keeps the wire it had.+ gbytes_field = (f" {len(xpayload)}"+ if (xpayload or (ext and gbytes_before_ext)) else "")header = (f"SUBMIT {request_id} {cache_slot} {len(payload)} {max_tokens} "f"{temperature:.8g} {top_p:.8g}"- + (prefix_field if prefix_field else (f" {len(xpayload)}" if xpayload else ""))+ + (prefix_field if prefix_field else gbytes_field)+ ext+ "\n").encode()try:@@ -3865,6 +4583,9 @@ class Engine:cancel_sent = Falsestop_sent = Falseaccepted = False+ # DATA's numeric tail, in emission order. Stays empty when logprobs is 0, because+ # the dispatcher never parses a tail for a request that did not ask for one.+ generated_logprobs = []def _accept(info):# #597: commit exactly once, on the first of ACCEPT / DATA / DONE. A new engine sends@@ -3899,6 +4620,28 @@ class Engine:with self.write_lock:self.process.stdin.write(f"CANCEL {request_id}\n".encode())self.process.stdin.flush()+ elif not cancel_sent and not stop_sent and pending.failed is not None:+ # A fault was recorded for this request, so nothing it generates from+ # here can reach the client. The turn is still the engine's -- the+ # admission is held to its terminal frame either way -- but it should+ # not run the rest of the budget for a response nobody will get.+ stop_sent = True+ try:+ with self.write_lock:+ self.process.stdin.write(f"STOP {request_id}\n".encode())+ self.process.stdin.flush()+ except OSError as error:+ # An engine this server cannot write to outranks a fault in the+ # stream it cannot read: the request is answered by name instead+ # of having the write error unwind through the handler, which+ # closes the client's connection with no status line on it. The+ # entry goes too -- a STOP that never reached the engine will+ # never be answered with a terminal frame, so waiting for one+ # would hold the admission for the process's lifetime.+ with self.pending_lock:+ self.pending.pop(request_id, None)+ raise RuntimeError(+ f"failed to write STOP to the engine ({error})") from errorcontinueif kind == "accept":if accepted:@@ -3906,8 +4649,13 @@ class Engine:_accept(value)elif kind == "data":_accept({"prompt_tokens": None})+ # The dispatcher wraps the payload in a tuple only when a logprob record+ # rides along; bare bytes is the non-opted-in shape.+ data, record = value if isinstance(value, tuple) else (value, None)+ if record is not None:+ generated_logprobs.append((data, record))if not cancel_sent and not stop_sent:- decode(value)+ decode(data)if stopped and stopped():stop_sent = Truewith self.write_lock:@@ -3956,8 +4704,19 @@ class Engine:tool_tail = tool_decoder.decode(b"", final=True)if tool_tail and on_tool is not None:on_tool(tool_tail)+ if logprobs:+ value["logprobs"] = {"generated": generated_logprobs}return value- elif cancel_sent and isinstance(value, RuntimeError) and str(value) == "CANCELLED":+ elif cancel_sent and (value is pending.failed+ or (isinstance(value, RuntimeError)+ and str(value) == "CANCELLED")):+ # The client left, on the two terminal frames that report nothing else: an+ # ERROR CANCELLED, which is the engine acknowledging the cancel, and a DONE,+ # which hands back the fault recorded while nobody was reading. Raising the+ # fault instead books a departed client as a server failure in the+ # scheduler's outcome, and tries to write a 500 to a socket that is already+ # closed. An ERROR carrying any other text is the engine's own account of+ # why the turn ended; that is news, and it is still reported as a failure.raise ClientCancelled()else:raise value@@ -4951,6 +5710,8 @@ class APIHandler(BaseHTTPRequestHandler):# grammar payload extension would desync its stdin framing.raise APIError(400, f"`response_format` grammars are not supported by the {ARCH} ""engine yet.", "response_format", "unsupported_parameter")+ engine_k, echo, display_k = logprobs_options(+ body, chat, getattr(self.server.engine, "supports_logprobs_echo", False))stop_sequences, ignore_leading_stop = stop_policy(body, chat)# tools and tool_choice come from chat_completion() already processed/filteredif chat and tool_choice == "none":@@ -4971,6 +5732,12 @@ class APIHandler(BaseHTTPRequestHandler):stream = body.get("stream", False)if not isinstance(stream, bool):raise APIError(400, "`stream` must be a boolean.", "stream")+ if engine_k and stream:+ # A named 400, not a silent drop of the numeric channel: streamed per-delta+ # logprobs are not built, and nothing below this point threads the channel+ # into a streaming call.+ raise APIError(400, "`logprobs` is not supported together with `stream`.",+ "logprobs", "unsupported_parameter")stream_options = body.get("stream_options") if stream else Noneif stream and stream_options is not None and not isinstance(stream_options, dict):raise APIError(400, "`stream_options` must be an object.", "stream_options")@@ -4986,9 +5753,13 @@ class APIHandler(BaseHTTPRequestHandler):queue_headers = {"x-colibri-queue-wait-ms": str(round(queue_wait * 1000))}if not stream:output = []- stop_filter = StopFilter(stop_sequences, output.append, ignore_leading_stop)+ stop_filter = StopFilter(stop_sequences, output.append, ignore_leading_stop,+ track_spans=bool(engine_k))sideband = ToolSideband(ARCH == "kimi" and chat and bool(tools),stop_sequences, ignore_leading_stop)+ # The engine sends an ECHO frame for every prompt position of every+ # opted-in request, so passing a sink is the retention opt-in.+ prompt_echoes = []def generation_stopped():return stop_filter.stopped() or sideband.stopped()@@ -4998,36 +5769,73 @@ class APIHandler(BaseHTTPRequestHandler):self.client_disconnected, grammar=grammar, stopped=generation_stopped,**({"on_tool": sideband.feed} if sideband.enabled else {}),**({"audio": audio} if audio else {}),- **({"image": image} if image is not None else {}))+ **({"image": image} if image is not None else {}),+ **({"on_echo": prompt_echoes.append} if engine_k and echo else {}),+ **_engine_extension_args(engine_k))stop_filter.finish()sideband.finish()text = "".join(output)+ # What the stop filter emitted, before any split or tool-call parse: the+ # base the rest of the pipeline's maps are chained onto.+ raw_text = textreasoning = ""+ answer_spans = tool_spans = Noneif ARCH == "inkling":text, reasoning = split_inkling(text)elif chat:# #597 item 4: GLM emits reasoning then </think> then the answer. Route the# reasoning to reasoning_content instead of dumping it (or the raw </think>)# into the visible answer / tool-call parser.- reasoning, text = split_thinking_reply(text, enable_thinking,- add_generation_prompt)- length_finish = "length" if stats["length_limited"] else "stop"+ if engine_k:+ reasoning, text, (_thinking_spans, answer_spans) = (+ split_thinking_reply_spans(text, enable_thinking,+ add_generation_prompt))+ else:+ reasoning, text = split_thinking_reply(text, enable_thinking,+ add_generation_prompt)+ # The tool-call parse runs before the logprobs tail because the arrays+ # describe the string it produces. It reads nothing the tail writes.+ content, calls, content_spans = None, [], None+ if chat and tools:+ if engine_k:+ content, calls, _box_spans, tool_spans = parse_arch_tool_calls_spans(+ text, tools, sideband.reply())+ else:+ content, calls = parse_arch_tool_calls(text, tools, sideband.reply())+ # Compose the stages' maps into one raw-stream to returned-string map;+ # a stage with no map leaves the chain at the last string it vouches for.+ if chat and answer_spans is not None and engine_k:+ chain = _compose_span_maps(stop_filter.spans, answer_spans)+ target = text+ if tools:+ if tool_spans is None:+ chain = None+ else:+ chain, target = _compose_span_maps(chain, tool_spans), content+ if chain is not None:+ content_spans = (target, chain)+ text, logprobs_obj, length_finish = _completion_choice_fields(+ stats, raw_text=raw_text, text=text, chat=chat, engine_k=engine_k,+ echo=echo, display_k=display_k, prompt_echoes=prompt_echoes,+ stop_spans=stop_filter.spans, content_spans=content_spans,+ cache_slot=cache_slot)if chat and tools:- content, calls = parse_arch_tool_calls(text, tools, sideband.reply())message = {"role": "assistant", "content": content or None, "refusal": None}if reasoning:message["reasoning_content"] = reasoningif calls:message["tool_calls"] = callsfinish = "tool_calls" if calls else length_finish- choice = {"index": 0, "message": message, "logprobs": None, "finish_reason": finish}+ choice = {"index": 0, "message": message, "logprobs": logprobs_obj,+ "finish_reason": finish}else:_msg = {"role": "assistant", "content": text, "refusal": None}if reasoning:_msg["reasoning_content"] = reasoningchoice = ({"index": 0, "message": _msg,- "logprobs": None, "finish_reason": length_finish} if chat else- {"index": 0, "text": text, "logprobs": None, "finish_reason": length_finish})+ "logprobs": logprobs_obj, "finish_reason": length_finish} if chat else+ {"index": 0, "text": text, "logprobs": logprobs_obj,+ "finish_reason": length_finish})self.send_json(200, {"id": completion_id, "object": object_name, "created": created,"model": self.server.model_id, "choices": [choice], "usage": self.usage(stats)},request_id, queue_headers)@@ -5184,7 +5992,8 @@ class APIHandler(BaseHTTPRequestHandler):self.client_disconnected, grammar=grammar, stopped=generation_stopped,**({"on_tool": sideband.feed} if sideband.enabled else {}),on_accept=start_stream, **({"audio": audio} if audio else {}),- **({"image": image} if image is not None else {}))+ **({"image": image} if image is not None else {}),+ **_engine_extension_args(engine_k))stop_filter.finish()sideband.finish()if think:@@ -5218,7 +6027,8 @@ class APIHandler(BaseHTTPRequestHandler):prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,self.client_disconnected, grammar=grammar, stopped=stop_filter.stopped,on_accept=start_stream, **({"audio": audio} if audio else {}),- **({"image": image} if image is not None else {}))+ **({"image": image} if image is not None else {}),+ **_engine_extension_args(engine_k))stop_filter.finish()if content_split:content_split.close()diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex 1c5e41ff..4f06ed8e 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -15,9 +15,11 @@ from urllib.error import HTTPErrorfrom urllib.request import Request, urlopenfrom pathlib import Path+import openai_serverfrom openai_server import (APIError, APIHandler, APIServer, ClientCancelled,CONTINUATION_FAMILIES, _marker_cuts,DEFAULT_CHAT_STOP_SEQUENCES, END, GenerationScheduler,+ LOGPROBS_TOP_K_CAP, logprobs_options,READY, Engine, InklingStreamSplit, StopFilter, ThinkingStreamSplit,_engine_error, _image_bytes_from_url, cap_for_arch, conversation_cache_slot, model_arch,generation_options, parse_tool_calls, parse_dsv4_tool_calls,@@ -25,8 +27,7 @@ from openai_server import (APIError, APIHandler, APIServer, ClientCancelled,read_engine_turn, render_chat, render_chat_for_arch,render_chat_glm53, render_chat_inkling, render_chat_kimi,render_chat_olmoe,- render_chat_qwen38, render_chat_v4, render_chat_dsv41,- _dsv4_tool_calls, serve,+ render_chat_qwen38, render_chat_v4, render_chat_dsv41, serve,resolve_generation_prompt, split_thinking_reply,split_thinking_reply_spans, starts_in_reasoning,parse_tool_calls_spans, parse_arch_tool_calls_spans,@@ -35,22 +36,281 @@ from openai_server import (APIError, APIHandler, APIServer, ClientCancelled,stop_policy, tune_child_env)+def echo_record(pos, data, lp, topk):+ """One engine ECHO record shaped exactly as Engine._dispatch_stdout builds it:+ `pos`/`logprob`/`text`, the three keys the closed-set scorer reads with its own+ None-for-nan convention, plus `bytes`/`lp`/`topk`, the payload and numeric tail the+ OpenAI logprobs surface reads. Every fixture builds its echo records through this one+ helper, so none can drift from the shape the dispatcher emits."""+ return {"pos": pos,+ "logprob": None if math.isnan(lp) else lp,+ "text": data.decode("utf-8", "replace"),+ "bytes": data,+ "lp": lp,+ "topk": topk}+++def _spawn_test_server(case, engine, kv_slots=1, max_tokens=16):+ """A throwaway APIServer on an ephemeral port, torn down with the test case."""+ server = APIServer(("127.0.0.1", 0), engine, "test-model", "secret", max_tokens,+ kv_slots=kv_slots)+ thread = threading.Thread(target=server.serve_forever, args=(0.01,), daemon=True)+ thread.start()+ # LIFO: the join is registered first so it runs last. Joining a still-serving thread+ # before shutdown() burns the whole timeout on every test that does it.+ case.addCleanup(thread.join, timeout=2)+ case.addCleanup(server.server_close)+ case.addCleanup(server.shutdown)+ case.addCleanup(server.scheduler.close)+ return f"http://127.0.0.1:{server.server_port}"+++def _post_json(base, path, body):+ return urlopen(Request(base + path, data=json.dumps(body).encode(),+ headers={"Authorization": "Bearer secret",+ "Content-Type": "application/json"}), timeout=5)+++def _post_completions(base, body):+ return _post_json(base, "/v1/completions", body)+++def _post_chat(base, body):+ return _post_json(base, "/v1/chat/completions", body)+++def _error_body(case, call):+ """The status and the parsed `error` object a refused request puts on the wire. Read+ from the RESPONSE BODY, never from an exception object: a named exception that never+ reaches the client is the defect, not the fix."""+ with case.assertRaises(HTTPError) as caught:+ call()+ raw = caught.exception.read()+ caught.exception.close()+ return caught.exception.code, json.loads(raw)["error"]+++class ScriptedEngine:+ """An engine double driven entirely by data: the text chunks it feeds to `on_text`, the+ per-token numeric records it reports, and the prompt-echo frames it sends.++ Every alignment fixture below is one of these with different data, so the boundary a+ test describes lives in the test rather than in a subclass of its own."""++ def __init__(self, chunks=("ok",), records="derive", echoes=(), echo_base=0,+ prompt_tokens=2, length_limited=False, supports_logprobs_echo=True,+ supports_tok_ids=True):+ self.chunks = tuple(chunks)+ # "derive" = one record per chunk, carrying that chunk's own bytes. None = the+ # engine sends no numeric channel at all. A list = exactly those (bytes, lp, topk)+ # triples, which is how a fixture makes the raw stream and the record bytes+ # legitimately disagree.+ self.records = records+ self.echoes = tuple(echoes)+ self.echo_base = echo_base # the first wire `pos`, above 0 for a pin+ self.prompt_tokens = prompt_tokens+ self.length_limited = length_limited+ self.supports_logprobs_echo = supports_logprobs_echo+ self.supports_tok_ids = supports_tok_ids+ self.calls = []+ self.stop_requests = 0+ self.last_logprobs = 0+ self.last_echo = False+ self.last_gbytes = False++ def generate(self, prompt, maximum, temperature, top_p, on_text, cache_slot=0,+ cancelled=None, grammar=None, stopped=None, on_accept=None, audio=None,+ on_tool=None, image=None, logprobs=0, pin=False, on_echo=None,+ gbytes_before_ext=False):+ self.calls.append((prompt, maximum, temperature, top_p, cache_slot, grammar))+ self.last_logprobs = logprobs+ self.last_echo = on_echo is not None+ self.last_gbytes = gbytes_before_ext+ if on_accept is not None:+ on_accept({"prompt_tokens": self.prompt_tokens})+ if logprobs and on_echo is not None:+ for offset, (data, lp, topk) in enumerate(self.echoes):+ on_echo(echo_record(self.echo_base + offset, data, lp, topk))+ for chunk in self.chunks:+ on_text(chunk)+ if stopped and stopped():+ self.stop_requests += 1+ break+ stats = {"prompt_tokens": self.prompt_tokens,+ "completion_tokens": len(self.chunks),+ "length_limited": self.length_limited}+ if logprobs and self.records is not None:+ triples = ([(chunk.encode("utf-8"), -0.1, [(1, -0.1)]) for chunk in self.chunks]+ if self.records == "derive" else self.records)+ stats["logprobs"] = {"generated": [(data, {"lp": lp, "topk": topk})+ for data, lp, topk in triples]}+ return stats+++NAN = float("nan")++# Three generated frames whose first is the GLM role marker the default chat stop policy+# swallows. `raw_text` is "Hello world", and the+# marker's record describes characters the client never received while the other two+# describe characters it did.+def _ignored_leading_marker_engine():+ return ScriptedEngine(+ chunks=("<|user|>", "Hello", " world"),+ records=[(b"<|user|>", -0.1, [(0, -0.1)]),+ (b"Hello", -0.2, [(1, -0.2)]),+ (b" world", -0.30000000000000004, [(2, -0.30000000000000004)])],+ echoes=[(b"h", NAN, []), (b"i", -0.1, [(1, -0.1)])])+++# One frame decoding to "Hello" against a `stop` of "lo". The filter emits "Hel" and matches, so the frame's record describes five characters+# of which the client received three. The prompt echo is "pr" + "ompt".+def _mid_token_stop_echo_engine():+ return ScriptedEngine(+ chunks=("Hello",),+ records=[(b"Hello", -0.5, [(5, -0.5)])],+ echoes=[(b"pr", NAN, []), (b"ompt", -0.1, [(1, -0.1)])])+++# A matched stop sequence withholds its own text and everything after it: neither the+# "STOP" record nor the " more" record after it may survive into the response.+def _stop_token_engine():+ return ScriptedEngine(+ chunks=("ok ", "STOP", " more"),+ records=[(b"ok ", -0.1, [(1, -0.1)]),+ (b"STOP", -0.2, [(2, -0.2)]),+ (b" more", -0.3, [(3, -0.3)])],+ echoes=[(b"h", NAN, []), (b"i", -0.1, [(1, -0.1)])])+++# The Euro sign split 1+2 across the prompt/generated seam -- lead byte on the last echo+# frame -- with a later frame cut by a matched `stop`. A decoder with no leading-byte+# context reads b"\x82\xac" as two replacement characters, and that is the raw stream the+# stop filter sees.+def _seam_split_stop_engine():+ return ScriptedEngine(+ chunks=("��", "xSTOPy"),+ records=[(b"\x82\xac", -0.2, [(2, -0.2)]),+ (b"xSTOPy", -0.3, [(3, -0.3)])],+ echoes=[(b"A", NAN, []), (b"\xe2", -0.1, [(1, -0.1)])])+++# The seam and the stop on the SAME record: one frame b"\x82\xacHello" whose first two+# bytes complete a Euro sign whose lead byte rode the last prompt frame, cut after "Hel".+def _seam_split_same_record_stop_engine():+ return ScriptedEngine(+ chunks=("��Hello",),+ records=[(b"\x82\xacHello", -0.2, [(2, -0.2)])],+ echoes=[(b"A", NAN, []), (b"\xe2", -0.1, [(1, -0.1)])])+++# The Euro sign split 1+1+1 across three frames: the first generated frame completes+# nothing, so its own decoding is "" while the raw stream reads it as a replacement+# character.+def _seam_split_across_two_frames_engine():+ return ScriptedEngine(+ chunks=("�", "�Hello"),+ records=[(b"\x82", -0.2, [(2, -0.2)]),+ (b"\xacHello", -0.3, [(3, -0.3)])],+ echoes=[(b"A", NAN, []), (b"\xe2", -0.1, [(1, -0.1)])])+++# A frame that decodes to nothing, sitting exactly on the boundary of the emitted region,+# whose bytes go into the `stop` sequence itself: "ok " occupies raw [0, 3), and the next+# two frames are the halves of "é", which is the stop, so the map ends at 3.+def _suppressed_stop_byte_engine():+ return ScriptedEngine(+ chunks=("ok ", "é"),+ records=[(b"ok ", -0.1, [(1, -0.1)]),+ (b"\xc3", -0.3, [(3, -0.3)]),+ (b"\xa9", -0.4, [(4, -0.4)])])+++# Three generated chunks with a numeric tail for only `covered` of them. The real engine+# gives an opted-in request a tail on every token, but a dropped frame or a different build+# can do this, and the echo path rebuilds `text` from the records.+def _partial_coverage_engine(covered, chunks=("AA", "BB", "CC"), record_bytes=None,+ echo_bytes=(b"P", b"Q")):+ payloads = record_bytes or tuple(c.encode("utf-8") for c in chunks)+ return ScriptedEngine(+ chunks=chunks,+ records=[(payload, -0.2, [(9, -0.2)]) for payload in payloads[:covered]],+ echoes=[(piece, NAN if pos == 0 else -0.1, [] if pos == 0 else [(1, -0.1)])+ for pos, piece in enumerate(echo_bytes)])+++# One generated chunk whose top-k table repeats a candidate id. `duplicate=False` is the+# control: the same engine, the same shape, distinct ids.+def _duplicate_candidate_engine(duplicate=True):+ return ScriptedEngine(+ chunks=("cat",),+ records=[(b"cat", -0.5,+ [(7, -0.1), (7, -0.2)] if duplicate else [(7, -0.1), (8, -0.2)])])+++# A pin snapshot covers the first prompt token, so the prefill reads out positions 1..n-1+# and there is nothing to read out before that.+def _pinned_prefix_engine():+ return ScriptedEngine(chunks=("ok",), echo_base=1, prompt_tokens=3,+ echoes=[(b"b", -0.1, [(1, -0.1)]), (b"c", -0.2, [(2, -0.2)])])++class FakeEngine:+ # Both capability gates, on: tests run under the module's default ARCH="glm", where+ # Engine.__init__ sets both from the same arch check. The gate tests set them+ # explicitly rather than relying on this.+ supports_logprobs_echo = True+ supports_tok_ids = True+def __init__(self):self.calls = []self.stop_requests = 0+ self.last_logprobs = 0+ self.last_echo = False+ self.last_gbytes = Falsedef generate(self, prompt, maximum, temperature, top_p, on_text, cache_slot=0,- cancelled=None, grammar=None, stopped=None, on_accept=None):+ cancelled=None, grammar=None, stopped=None, on_accept=None, logprobs=0,+ pin=False, on_echo=None, gbytes_before_ext=False):self.calls.append((prompt, maximum, temperature, top_p, cache_slot, grammar))+ self.last_logprobs = logprobs+ self.last_echo = on_echo is not None+ self.last_gbytes = gbytes_before_extif on_accept is not None: # simulate the engine's ACCEPT frame (#597)on_accept({"prompt_tokens": 7})+ if logprobs and on_echo is not None:+ for record in self.echo_records(logprobs):+ on_echo(record)for chunk in ("Hé", "llo"):on_text(chunk)if stopped and stopped():self.stop_requests += 1break- return {"prompt_tokens": 7, "completion_tokens": 2, "length_limited": False}+ stats = {"prompt_tokens": 7, "completion_tokens": 2, "length_limited": False}+ if logprobs:+ stats["logprobs"] = self.logprobs_channel(logprobs)+ return stats++ def echo_records(self, engine_k):+ """A canned prefill read-out, delivered through `on_echo` exactly as+ Engine.generate() delivers the real thing. Position 0 carries the engine's own+ "nothing to condition on" sentinel. The tail values are non-dyadic so a fixture+ rendered at one precision is distinguishable from one rendered at another."""+ k = min(engine_k, 2)+ return [echo_record(0, b"H", float("nan"), []),+ echo_record(1, b"\xc3\xa9", -0.3, [(72, -0.3), (100, -1.7)][:k])]++ def logprobs_channel(self, engine_k):+ """Canned generated-token records, shaped exactly like Engine.generate()'s return+ value. Each token's own lp is bit-identical to one of its own top-k entries, which+ is the engine's logprob_tail invariant. The records cover, byte for byte, the text+ generate() feeds to on_text: the real engine emits a tail for every token of an+ opted-in request, so a double that emits text it has no records for would model+ nothing the engine does."""+ k = min(engine_k, 2)+ return {"generated": [+ (b"H", {"lp": -0.2, "topk": [(72, -0.2), (200, -2.4)][:k]}),+ (b"\xc3\xa9", {"lp": -0.4, "topk": [(101, -0.4), (300, -3.6)][:k]}),+ (b"llo", {"lp": -0.6, "topk": [(400, -0.6), (500, -5.2)][:k]})]}class BlockingEngine(FakeEngine):@@ -60,11 +320,13 @@ class BlockingEngine(FakeEngine):self.release = threading.Event()def generate(self, prompt, maximum, temperature, top_p, on_text, cache_slot=0,- cancelled=None, grammar=None, stopped=None, on_accept=None):+ cancelled=None, grammar=None, stopped=None, on_accept=None, logprobs=0,+ pin=False, on_echo=None, gbytes_before_ext=False):self.entered.set()self.release.wait(2)return super().generate(prompt, maximum, temperature, top_p, on_text, cache_slot,- cancelled, grammar, stopped, on_accept)+ cancelled, grammar, stopped, on_accept, logprobs, pin,+ on_echo, gbytes_before_ext)class TemplateTest(unittest.TestCase):@@ -3828,5 +4090,1758 @@ class ContextExceededMessageTest(unittest.TestCase):self.assertIn("the context", text)+class LogprobsOptionsTest(unittest.TestCase):+ """logprobs_options(): pure validation and translation, no HTTP and no engine."""++ def test_completions_valid_integer_logprobs(self):+ self.assertEqual(logprobs_options({"logprobs": 3}, False, True), (3, False, 3))+ self.assertEqual(logprobs_options({"logprobs": 3, "echo": True}, False, True),+ (3, True, 3))+ self.assertEqual(logprobs_options({"logprobs": 1}, False, True), (1, False, 1))++ def test_completions_zero_false_null_mean_no_logprobs(self):+ # Explicit, never a truthiness accident and never a channel floored on at k=1.+ for off in ({"logprobs": 0}, {"logprobs": False}, {"logprobs": None}, {}):+ with self.subTest(off=off):+ self.assertEqual(logprobs_options(off, False, True), (0, False, 0))+ # ...and `echo: true` on top of an off `logprobs` is a named 400, not a no-op.+ with self.assertRaises(APIError) as caught:+ logprobs_options({"logprobs": 0, "echo": True}, False, True)+ self.assertEqual((caught.exception.param, caught.exception.code),+ ("echo", "unsupported_parameter"))++ def test_completions_true_is_a_named_400(self):+ # The legacy completions field is an integer count; `true` carries no count.+ with self.assertRaises(APIError) as caught:+ logprobs_options({"logprobs": True}, False, True)+ self.assertEqual((caught.exception.status, caught.exception.param,+ caught.exception.code), (400, "logprobs", "invalid_value"))++ def test_completions_break_it_battery_non_integer_negative_huge(self):+ for bad in (1.5, "5", -1, 33):+ with self.subTest(bad=bad):+ with self.assertRaises(APIError) as caught:+ logprobs_options({"logprobs": bad}, False, True)+ self.assertEqual((caught.exception.status, caught.exception.param,+ caught.exception.code),+ (400, "logprobs", "invalid_value"))++ def test_chat_logprobs_requires_a_boolean(self):+ with self.assertRaises(APIError) as caught:+ logprobs_options({"logprobs": 1}, True, True)+ self.assertEqual((caught.exception.param, caught.exception.code),+ ("logprobs", "invalid_value"))++ def test_chat_false_and_null_mean_no_logprobs(self):+ for off in ({"logprobs": False}, {"logprobs": None}, {}):+ with self.subTest(off=off):+ self.assertEqual(logprobs_options(off, True, True), (0, False, 0))++ def test_chat_echo_is_always_rejected(self):+ # Chat has no echo concept at all: a named 400, not a silent ignore, whether or+ # not logprobs was also requested.+ for body in ({"echo": True}, {"echo": True, "logprobs": True}):+ with self.subTest(body=body):+ with self.assertRaises(APIError) as caught:+ logprobs_options(body, True, True)+ self.assertEqual(caught.exception.param, "echo")++ def test_chat_top_logprobs_default_and_cap(self):+ self.assertEqual(logprobs_options({"logprobs": True}, True, True), (1, False, 0))+ self.assertEqual(+ logprobs_options({"logprobs": True, "top_logprobs": 5}, True, True),+ (5, False, 5))+ with self.assertRaises(APIError) as caught:+ logprobs_options({"logprobs": True, "top_logprobs": 33}, True, True)+ self.assertEqual(caught.exception.param, "top_logprobs")++ def test_cap_boundary_is_thirty_two_on_both_endpoints(self):+ # Literal 32/33 rather than the constant plus or minus one, so the boundary stays+ # meaningful if the constant itself ever drifts.+ self.assertEqual(logprobs_options({"logprobs": 32}, False, True), (32, False, 32))+ self.assertEqual(+ logprobs_options({"logprobs": True, "top_logprobs": 32}, True, True),+ (32, False, 32))+ for body, chat, param in (({"logprobs": 33}, False, "logprobs"),+ ({"logprobs": True, "top_logprobs": 33}, True,+ "top_logprobs")):+ with self.subTest(param=param):+ with self.assertRaises(APIError) as caught:+ logprobs_options(body, chat, True)+ self.assertEqual((caught.exception.param, caught.exception.code),+ (param, "invalid_value"))+ self.assertEqual(LOGPROBS_TOP_K_CAP, 32)++ def test_capability_gate_rejects_an_engine_that_is_not_asked_for_the_channel(self):+ # Never a silent downgrade to "no logprobs".+ for body, chat in (({"logprobs": 1}, False), ({"logprobs": True}, True)):+ with self.subTest(chat=chat):+ with self.assertRaises(APIError) as caught:+ logprobs_options(body, chat, False)+ self.assertEqual((caught.exception.status, caught.exception.code),+ (400, "unsupported_parameter"))+ # Absent or zero logprobs never reaches the capability check at all.+ self.assertEqual(logprobs_options({}, False, False), (0, False, 0))+ self.assertEqual(logprobs_options({"logprobs": 0}, False, False), (0, False, 0))++ def test_range_error_precedes_capability_error_and_names_the_cap(self):+ with self.assertRaises(APIError) as caught:+ logprobs_options({"logprobs": 999}, False, False)+ self.assertEqual(caught.exception.code, "invalid_value")+ self.assertIn("32", caught.exception.message)++ def test_chat_top_logprobs_validated_even_when_logprobs_is_off(self):+ # A malformed `top_logprobs` is a named 400 whether or not the gate that would use+ # it is open; a valid one with `logprobs` off stays a documented no-op.+ for bad in ("x", -1, 999):+ with self.subTest(bad=bad):+ with self.assertRaises(APIError) as caught:+ logprobs_options({"top_logprobs": bad}, True, True)+ self.assertEqual((caught.exception.status, caught.exception.param,+ caught.exception.code),+ (400, "top_logprobs", "invalid_value"))+ self.assertEqual(logprobs_options({"top_logprobs": 5}, True, True), (0, False, 0))++ def test_chat_top_logprobs_null_normalizes_to_absent(self):+ # On both sides of the `logprobs` gate, and it never itself reaches the+ # type/range check.+ self.assertEqual(+ logprobs_options({"logprobs": False, "top_logprobs": None}, True, True),+ (0, False, 0))+ self.assertEqual(+ logprobs_options({"logprobs": True, "top_logprobs": None}, True, True),+ (1, False, 0))++ def test_echo_null_normalizes_to_absent_both_endpoints(self):+ # `echo: null` normalises to absent before the type check, exactly like `logprobs`+ # and `top_logprobs`. An SDK that serialises its whole request model with nulls+ # must not get a 400 for a field it never meant to set.+ self.assertEqual(logprobs_options({"echo": None}, False, True), (0, False, 0))+ self.assertEqual(logprobs_options({"echo": None}, True, True), (0, False, 0))++ def test_echo_non_bool_rejected_the_same_way_on_both_endpoints(self):+ for bad in (1, "true"):+ for chat in (False, True):+ with self.subTest(bad=bad, chat=chat):+ with self.assertRaises(APIError) as caught:+ logprobs_options({"echo": bad}, chat, True)+ self.assertEqual((caught.exception.param, caught.exception.code),+ ("echo", "invalid_value"))++ def test_echo_refusals_stay_distinct(self):+ # The type refusal and chat's capability refusal must not collapse into each+ # other: a non-bool `echo` always gets the type message, and only an actual `True`+ # on chat draws the unsupported one.+ with self.assertRaises(APIError) as caught:+ logprobs_options({"echo": "yes"}, True, True)+ self.assertEqual(caught.exception.code, "invalid_value")+ self.assertIn("must be a boolean", caught.exception.message)+ with self.assertRaises(APIError) as caught:+ logprobs_options({"echo": True}, True, True)+ self.assertEqual(caught.exception.code, "unsupported_parameter")+ self.assertIn("not supported for chat completions", caught.exception.message)++ def test_completions_ignores_top_logprobs_entirely(self):+ # A chat-only field in the OpenAI request shape; completions never reads it.+ self.assertEqual(+ logprobs_options({"logprobs": 2, "top_logprobs": 999}, False, True),+ (2, False, 2))+++class LogprobsHTTPTest(unittest.TestCase):+ """End-to-end behaviour of `logprobs`/`echo`/`top_logprobs` against a real APIServer,+ with FakeEngine standing in for the engine subprocess."""++ @classmethod+ def setUpClass(cls):+ cls.engine = FakeEngine()+ cls.server = APIServer(("127.0.0.1", 0), cls.engine, "test-model", "secret", 16,+ kv_slots=2)+ cls.thread = threading.Thread(target=cls.server.serve_forever, args=(0.01,),+ daemon=True)+ cls.thread.start()+ cls.base = f"http://127.0.0.1:{cls.server.server_port}"++ @classmethod+ def tearDownClass(cls):+ cls.server.scheduler.close()+ cls.server.shutdown()+ cls.server.server_close()+ cls.thread.join(timeout=2)++ def chat(self, body):+ return _post_chat(self.base, dict({"model": "test-model",+ "messages": [{"role": "user", "content": "Hi"}]},+ **body))++ def completions(self, body):+ return _post_completions(self.base, dict({"model": "test-model", "prompt": "Hi"},+ **body))++ # ---- shape -------------------------------------------------------------++ def test_chat_logprobs_content_shape(self):+ with self.chat({"logprobs": True, "top_logprobs": 2}) as response:+ body = json.load(response)+ choice = body["choices"][0]+ content = choice["logprobs"]["content"]+ self.assertEqual([entry["token"] for entry in content], ["H", "é", "llo"])+ self.assertIsNone(choice["logprobs"]["refusal"])+ for entry in content:+ self.assertEqual(set(entry), {"token", "logprob", "bytes", "top_logprobs"})+ for alternative in entry["top_logprobs"]:+ self.assertEqual(set(alternative), {"token", "logprob", "bytes"})+ self.assertNotIn("echo", choice)++ def test_chat_content_joins_back_to_the_message(self):+ with self.chat({"logprobs": True, "top_logprobs": 1}) as response:+ choice = json.load(response)["choices"][0]+ self.assertEqual("".join(entry["token"] for entry in choice["logprobs"]["content"]),+ choice["message"]["content"])++ def test_chat_top_logprobs_omitted_yields_empty_alternatives(self):+ with self.chat({"logprobs": True}) as response:+ content = json.load(response)["choices"][0]["logprobs"]["content"]+ self.assertTrue(content)+ for entry in content:+ self.assertEqual(entry["top_logprobs"], [])++ def test_the_chosen_tokens_logprob_comes_from_its_own_record(self):+ # The label is found by exact float match against the position's own logprob, and+ # the table is unsorted on the wire, so this can never be "whatever came first".+ with self.chat({"logprobs": True, "top_logprobs": 2}) as response:+ content = json.load(response)["choices"][0]["logprobs"]["content"]+ # Against the fixture's declared per-record values, not against the alternative+ # the entry itself labelled -- that comparison holds however the value was sourced.+ self.assertEqual([entry["logprob"] for entry in content], [-0.2, -0.4, -0.6])+ for entry in content:+ own = [a for a in entry["top_logprobs"] if a["token"] == entry["token"]]+ self.assertEqual(len(own), 1)+ self.assertEqual(own[0]["logprob"], entry["logprob"])+ self.assertEqual(own[0]["bytes"], entry["bytes"])++ def test_the_chosen_token_is_found_by_value_not_by_table_rank(self):+ # The table is unsorted on the wire, so the position's own token can sit anywhere+ # in it. Labelling by rank would call the first entry the chosen token.+ base = _spawn_test_server(self, ScriptedEngine(+ chunks=("cat",), records=[(b"cat", -0.8, [(999, -0.05), (42, -0.8)])]))+ with _post_chat(base, {"model": "test-model", "logprobs": True, "top_logprobs": 2,+ "messages": [{"role": "user", "content": "hi"}]}) as response:+ entry = json.load(response)["choices"][0]["logprobs"]["content"][0]+ self.assertEqual([alternative["token"] for alternative in entry["top_logprobs"]],+ ["<token_id:999>", "cat"])+ self.assertEqual(entry["logprob"], -0.8)++ def test_a_tie_on_the_printed_value_is_broken_deterministically(self):+ # The engine prints six decimal digits, so two candidates can share the chosen+ # token's printed value. Only the first match in wire order is labelled as the+ # chosen token; a later one is labelled by its id like any other candidate, which+ # is also what keeps the two labels distinct.+ base = _spawn_test_server(self, ScriptedEngine(+ chunks=("cat",),+ records=[(b"cat", -0.223144, [(1, -0.223144), (2, -0.223144)])]))+ with _post_chat(base, {"model": "test-model", "logprobs": True, "top_logprobs": 2,+ "messages": [{"role": "user", "content": "hi"}]}) as response:+ entry = json.load(response)["choices"][0]["logprobs"]["content"][0]+ self.assertEqual([alternative["token"] for alternative in entry["top_logprobs"]],+ ["cat", "<token_id:2>"])++ def test_legacy_token_logprobs_come_from_the_records_own_value(self):+ # The completions twin of the chat test above: the table is unsorted on the wire,+ # so `token_logprobs` must be the record's own `lp`, never the first table entry.+ base = _spawn_test_server(self, ScriptedEngine(+ chunks=("cat",), records=[(b"cat", -0.8, [(999, -0.05), (42, -0.8)])]))+ with _post_completions(base, {"model": "test-model", "prompt": "hi",+ "logprobs": 2, "max_tokens": 1}) as response:+ logprobs = json.load(response)["choices"][0]["logprobs"]+ self.assertEqual(logprobs["tokens"], ["cat"])+ self.assertEqual(logprobs["token_logprobs"], [-0.8])+ self.assertEqual(logprobs["top_logprobs"], [{"<token_id:999>": -0.05, "cat": -0.8}])++ def test_completions_logprobs_object_shape_and_text_offsets(self):+ with self.completions({"logprobs": 2}) as response:+ choice = json.load(response)["choices"][0]+ logprobs = choice["logprobs"]+ self.assertEqual(set(logprobs),+ {"tokens", "token_logprobs", "top_logprobs", "text_offset"})+ self.assertEqual(logprobs["tokens"], ["H", "é", "llo"])+ # Characters into `text` from 0, not bytes: "é" is one character and two bytes.+ self.assertEqual(logprobs["text_offset"], [0, 1, 2])+ self.assertEqual("".join(logprobs["tokens"]), choice["text"])++ def test_echo_reconstructs_the_prompt_and_counts_offsets_from_zero(self):+ with self.completions({"logprobs": 2, "echo": True}) as response:+ choice = json.load(response)["choices"][0]+ logprobs = choice["logprobs"]+ self.assertEqual(logprobs["tokens"], ["H", "é", "H", "é", "llo"])+ self.assertEqual(logprobs["text_offset"], [0, 1, 2, 3, 4])+ self.assertEqual("".join(logprobs["tokens"]), choice["text"])+ # The engine's own echo position 0 carries a non-finite sentinel: there is nothing+ # to condition the first prompt token on.+ self.assertIsNone(logprobs["token_logprobs"][0])++ def test_nan_logprob_serializes_as_json_null_over_the_wire(self):+ # Not the invalid-JSON NaN literal, and not clamped to a made-up finite number.+ with self.completions({"logprobs": 1, "echo": True}) as response:+ raw = response.read().decode()+ self.assertNotIn("NaN", raw)+ self.assertIsNone(json.loads(raw)["choices"][0]["logprobs"]["token_logprobs"][0])++ # ---- normalisation and refusals ----------------------------------------++ def test_zero_false_null_normalize_to_absent(self):+ # Each endpoint's own shape: `0` is a completions value, `false`/`null` are+ # accepted on both, and none of them is ever an error.+ for field, value in (("logprobs", 0), ("logprobs", False), ("logprobs", None),+ ("echo", False), ("echo", None)):+ with self.subTest(endpoint="completions", field=field, value=value):+ with self.completions({field: value}) as response:+ choice = json.load(response)["choices"][0]+ self.assertIsNone(choice["logprobs"])+ self.assertEqual(choice["text"], "Héllo")+ for field, value in (("logprobs", False), ("logprobs", None), ("echo", None),+ ("top_logprobs", None), ("top_logprobs", 0)):+ with self.subTest(endpoint="chat", field=field, value=value):+ with self.chat({field: value}) as response:+ choice = json.load(response)["choices"][0]+ self.assertIsNone(choice["logprobs"])++ def test_echo_without_logprobs_is_refused_by_name(self):+ # The prompt echo is built out of the engine's per-token records, so without them+ # there is nothing to echo. A 200 that quietly drops a requested field is the shape+ # this surface exists to stop.+ for body in ({"echo": True}, {"echo": True, "logprobs": 0},+ {"echo": True, "logprobs": None}):+ with self.subTest(body=body):+ status, error = _error_body(self, lambda: self.completions(body))+ self.assertEqual(status, 400)+ self.assertEqual(error["param"], "echo")+ self.assertEqual(error["code"], "unsupported_parameter")++ def test_echo_false_and_null_stay_absent_without_logprobs(self):+ # The control: only `true` needs a channel, and the two absent spellings must keep+ # serving the request they always served.+ for value in (False, None):+ with self.subTest(value=value):+ with self.completions({"echo": value}) as response:+ choice = json.load(response)["choices"][0]+ self.assertIsNone(choice["logprobs"])+ self.assertEqual(choice["text"], "Héllo")++ def test_each_endpoint_refuses_the_others_request_shape_by_name(self):+ for post, body, param in ((self.completions, {"logprobs": True}, "logprobs"),+ (self.chat, {"logprobs": 2}, "logprobs"),+ (self.chat, {"echo": True}, "echo")):+ with self.subTest(body=body):+ status, error = _error_body(self, lambda: post(body))+ self.assertEqual(status, 400)+ self.assertEqual(error["param"], param)++ def test_break_it_logprobs_out_of_range(self):+ status, error = _error_body(self, lambda: self.completions({"logprobs": 99}))+ self.assertEqual(status, 400)+ self.assertEqual(error["code"], "invalid_value")++ def test_break_it_streaming_plus_logprobs_is_named_400(self):+ for post, body in ((self.completions, {"logprobs": 1, "stream": True}),+ (self.chat, {"logprobs": True, "stream": True})):+ with self.subTest(body=body):+ status, error = _error_body(self, lambda: post(body))+ self.assertEqual(status, 400)+ self.assertEqual(error["param"], "logprobs")+ self.assertEqual(error["code"], "unsupported_parameter")++ def test_chat_echo_false_normalises_to_absent_over_http(self):+ # The completions side of this is covered; chat is the half that was only true by+ # construction. `false` is not `true`, so it never reaches chat's echo refusal:+ # the request is served exactly as one that never named the field.+ with self.chat({"echo": False}) as response:+ choice = json.load(response)["choices"][0]+ self.assertIsNone(choice["logprobs"])+ self.assertEqual(choice["message"]["content"], "Héllo")+ with self.chat({"echo": False, "logprobs": True, "top_logprobs": 1}) as response:+ choice = json.load(response)["choices"][0]+ self.assertEqual([entry["token"] for entry in choice["logprobs"]["content"]],+ ["H", "é", "llo"])++ def test_a_continued_turn_is_prompt_and_the_arrays_describe_only_what_was_generated(self):+ """A trailing assistant turn is prefill, not output: the prompt ends inside it. The+ text the client sent must therefore reach the engine as prompt and must not appear+ in the arrays, which describe the generated text alone.++ `enable_thinking` is on because that is the axis where the two features meet. A+ continued turn opens no reasoning block, so the split starts in text mode -- and the+ span map the arrays are projected through comes from that same split. Primed the+ other way the split files the whole answer as reasoning, which leaves `content`+ empty and the arrays describing a string that is no longer there."""+ opening = "The capital of France is Par"+ with self.chat({"logprobs": True, "top_logprobs": 2, "enable_thinking": True,+ "messages": [{"role": "user", "content": "Where is it?"},+ {"role": "assistant", "content": opening}]}) as response:+ choice = json.load(response)["choices"][0]+ self.assertTrue(self.engine.calls[-1][0].endswith(opening),+ self.engine.calls[-1][0][-60:])+ self.assertEqual(choice["message"]["content"], "Héllo")+ self.assertNotIn("reasoning_content", choice["message"])+ tokens = [entry["token"] for entry in choice["logprobs"]["content"]]+ self.assertEqual(tokens, ["H", "é", "llo"])+ self.assertEqual("".join(tokens), choice["message"]["content"])++ def test_a_continued_turn_does_not_give_chat_an_echo(self):+ # `echo` is the legacy completions shape and chat refuses it by name. A continued+ # turn puts client text in the prompt, which is the one thing that could look like+ # a reason to answer one here; it is not, and the refusal is unchanged.+ status, error = _error_body(self, lambda: _post_chat(self.base, {+ "model": "test-model", "logprobs": True, "echo": True,+ "messages": [{"role": "user", "content": "Where is it?"},+ {"role": "assistant", "content": "The capital of France is Par"}]}))+ self.assertEqual(status, 400)+ self.assertEqual(error["param"], "echo")++ def test_the_capability_refusal_names_the_gate_not_the_engines_arch(self):+ # The refusal reads the flag it failed, not the module's ARCH: in production the+ # two agree, so naming the arch would be a guess that happens to be right, and+ # under the test module's own ARCH ("glm") it would read "not supported by the glm+ # engine" for an engine whose flag is off -- which is the wrong sentence.+ base = _spawn_test_server(self, ScriptedEngine(supports_logprobs_echo=False))+ status, error = _error_body(self, lambda: _post_completions(base, {+ "model": "test-model", "prompt": "hi", "logprobs": 1}))+ self.assertEqual(status, 400)+ self.assertEqual(+ error["message"],+ "Log probabilities are not requested from this engine by these endpoints.")+ self.assertNotIn(openai_server.ARCH, error["message"])+ self.assertNotIn("glm", error["message"])++ def test_break_it_logprobs_rejected_for_non_glm_engine(self):+ # A named 400, never a silent no-op, on an engine these endpoints do not request+ # the channel from.+ base = _spawn_test_server(self, ScriptedEngine(supports_logprobs_echo=False))+ for path, body in (("/v1/completions", {"model": "test-model", "prompt": "hi",+ "logprobs": 1}),+ ("/v1/chat/completions",+ {"model": "test-model", "logprobs": True,+ "messages": [{"role": "user", "content": "hi"}]})):+ with self.subTest(path=path):+ status, error = _error_body(self, lambda: _post_json(base, path, body))+ self.assertEqual(status, 400)+ self.assertEqual(error["param"], "logprobs")+ self.assertEqual(error["code"], "unsupported_parameter")++ def test_a_plain_request_never_asks_the_engine_for_the_channel(self):+ with self.completions({}) as response:+ choice = json.load(response)["choices"][0]+ self.assertIsNone(choice["logprobs"])+ self.assertEqual(self.engine.last_logprobs, 0)+ self.assertFalse(self.engine.last_echo)+ self.assertFalse(self.engine.last_gbytes)++ def test_the_echo_sink_is_passed_only_when_echo_was_asked_for(self):+ # The engine sends an ECHO frame for every prompt position of every opted-in+ # request, so passing a sink is the retention opt-in.+ with self.completions({"logprobs": 1}) as response:+ response.read()+ self.assertEqual(self.engine.last_logprobs, 1)+ self.assertFalse(self.engine.last_echo)+ with self.completions({"logprobs": 1, "echo": True}) as response:+ response.read()+ self.assertTrue(self.engine.last_echo)+++class LogprobsRawStreamAlignmentTest(unittest.TestCase):+ """The arrays describe the characters the client received, exactly. Every expectation+ here is written from the engine's own frames and the stop policy, and each test names+ inline what a build without the span layer returns for it."""++ def _chat(self, base, **extra):+ body = {"model": "test-model", "max_tokens": 8, "logprobs": True,+ "top_logprobs": 1, "messages": [{"role": "user", "content": "hi"}]}+ body.update(extra)+ with _post_chat(base, body) as response:+ return json.load(response)["choices"][0]++ def test_swallowed_leading_marker_keeps_the_other_two_records(self):+ # WITHOUT THE SPAN LAYER: logprobs.content == [] with a 200 and a full message.content. Two of+ # three, not three: the role marker was genuinely not emitted, so an entry for it+ # would describe a character the client never received.+ choice = self._chat(_spawn_test_server(self, _ignored_leading_marker_engine()))+ self.assertEqual(choice["message"]["content"], "Hello world")+ entries = choice["logprobs"]["content"]+ self.assertEqual([entry["token"] for entry in entries], ["Hello", " world"])+ self.assertEqual([entry["logprob"] for entry in entries],+ [-0.2, -0.30000000000000004])+ self.assertEqual("".join(entry["token"] for entry in entries),+ choice["message"]["content"])++ def test_swallowed_leading_marker_offsets_index_the_returned_text(self):+ # The legacy shape's arm of the same defect: "Hello" at 0 and " world" at 5, both+ # indexing "Hello world".+ base = _spawn_test_server(self, _ignored_leading_marker_engine())+ with _post_completions(base, {"model": "test-model", "prompt": "hi",+ "max_tokens": 8, "logprobs": 1,+ "stop": ["<|user|>"],+ "x_colibri_ignore_leading_stop": True}) as response:+ choice = json.load(response)["choices"][0]+ self.assertEqual(choice["text"], "Hello world")+ self.assertEqual(choice["logprobs"]["tokens"], ["Hello", " world"])+ self.assertEqual(choice["logprobs"]["text_offset"], [0, 5])+ self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])++ def test_a_record_the_stop_cut_in_half_keeps_the_emitted_characters(self):+ # WITHOUT THE SPAN LAYER: text "Hel" with tokens []. The boundary entry carries the emitted+ # span, so `tokens` is ["Hel"] -- not ["Hello"], and not [].+ base = _spawn_test_server(self, _mid_token_stop_echo_engine())+ with _post_completions(base, {"model": "test-model", "prompt": "prompt",+ "max_tokens": 4, "logprobs": 1,+ "stop": ["lo"]}) as response:+ choice = json.load(response)["choices"][0]+ self.assertEqual(choice["text"], "Hel")+ self.assertEqual(choice["logprobs"]["tokens"], ["Hel"])+ self.assertEqual(choice["logprobs"]["text_offset"], [0])+ self.assertEqual(choice["logprobs"]["token_logprobs"], [-0.5])+ self.assertEqual(choice["finish_reason"], "stop")+ self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])++ def test_echo_keeps_the_prompt_and_the_emitted_prefix(self):+ # WITHOUT THE SPAN LAYER: text collapses to "prompt" -- the "Hel" the client earned is deleted+ # by the echo rebuild.+ base = _spawn_test_server(self, _mid_token_stop_echo_engine())+ with _post_completions(base, {"model": "test-model", "prompt": "prompt",+ "max_tokens": 4, "logprobs": 1, "echo": True,+ "stop": ["lo"]}) as response:+ choice = json.load(response)["choices"][0]+ self.assertEqual(choice["text"], "promptHel")+ self.assertEqual(choice["logprobs"]["tokens"], ["pr", "ompt", "Hel"])+ self.assertEqual(choice["logprobs"]["text_offset"], [0, 2, 6])+ self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])++ def test_a_truncated_entrys_bytes_do_not_describe_the_withheld_text(self):+ # `bytes` is `token`'s sibling and is held to the same rule: the record's payload+ # decodes to "Hello", so leaving it whole would hand the client the "lo" the stop+ # withheld, one field over.+ choice = self._chat(_spawn_test_server(self, _mid_token_stop_echo_engine()),+ max_tokens=4, stop=["lo"])+ self.assertEqual(choice["message"]["content"], "Hel")+ entry = choice["logprobs"]["content"][0]+ self.assertEqual(entry["token"], "Hel")+ self.assertEqual(entry["bytes"], [72, 101, 108])+ self.assertEqual(bytes(entry["bytes"]).decode("utf-8"), entry["token"])+ for alternative in entry["top_logprobs"]:+ if alternative["bytes"] is not None:+ self.assertEqual(bytes(alternative["bytes"]).decode("utf-8"),+ alternative["token"])++ def test_a_matched_stop_drops_its_own_record_and_every_later_one(self):+ base = _spawn_test_server(self, _stop_token_engine())+ with _post_completions(base, {"model": "test-model", "prompt": "hi",+ "max_tokens": 8, "stop": ["STOP"],+ "logprobs": 1}) as response:+ choice = json.load(response)["choices"][0]+ self.assertEqual(choice["text"], "ok ")+ self.assertEqual(choice["logprobs"]["tokens"], ["ok "])+ self.assertEqual(len(choice["logprobs"]["token_logprobs"]), 1)+ self.assertEqual(len(choice["logprobs"]["top_logprobs"]), 1)++ def test_a_frame_inside_suppressed_text_leaves_no_phantom_entry(self):+ # A frame that decodes to "" owns no span, so "was it emitted?" has to be asked of+ # the character its bytes went into, not of its position, which sits on the+ # boundary of the emitted region and tests true at both ends. A phantom entry here+ # carries a byte of the suppressed stop sequence out in the one field that is+ # bytes.+ base = _spawn_test_server(self, _suppressed_stop_byte_engine())+ with _post_completions(base, {"model": "test-model", "prompt": "hi",+ "max_tokens": 8, "logprobs": 1,+ "stop": ["é"]}) as response:+ choice = json.load(response)["choices"][0]+ self.assertEqual(choice["text"], "ok ")+ self.assertEqual(choice["logprobs"]["tokens"], ["ok "])+ self.assertEqual(choice["logprobs"]["text_offset"], [0])+ self.assertEqual(choice["logprobs"]["token_logprobs"], [-0.1])++ def test_a_frame_that_completes_nothing_at_the_seam_reports_nothing(self):+ # Both rows of the two-frame seam case. Frame one's bytes finish no character, so+ # its token is "": reporting the raw stream's replacement character instead puts a+ # U+FFFD in front of the real one and shifts every later offset.+ for stop, text, tokens, offsets in (+ (None, "A€Hello", ["A", "", "", "€Hello"], [0, 1, 1, 1]),+ (["lo"], "A€Hel", ["A", "", "", "€Hel"], [0, 1, 1, 1])):+ with self.subTest(stop=stop):+ base = _spawn_test_server(self, _seam_split_across_two_frames_engine())+ body = {"model": "test-model", "prompt": "A€", "max_tokens": 8,+ "logprobs": 1, "echo": True}+ if stop is not None:+ body["stop"] = stop+ with _post_completions(base, body) as response:+ choice = json.load(response)["choices"][0]+ self.assertEqual(choice["text"], text)+ self.assertNotIn("�", choice["text"])+ self.assertEqual(choice["logprobs"]["tokens"], tokens)+ self.assertEqual(choice["logprobs"]["text_offset"], offsets)+ self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])++ def test_seam_codepoint_and_stop_on_the_same_record(self):+ # The raw-stream decode opens with two replacement characters where the seam+ # decode has one Euro sign; the stop cuts past that head, so the real character+ # splices back on and the withheld "lo" stays withheld.+ base = _spawn_test_server(self, _seam_split_same_record_stop_engine())+ with _post_completions(base, {"model": "test-model", "prompt": "A€",+ "max_tokens": 4, "logprobs": 1, "echo": True,+ "stop": ["lo"]}) as response:+ choice = json.load(response)["choices"][0]+ self.assertEqual(choice["text"], "A€Hel")+ self.assertNotIn("�", choice["text"])+ self.assertEqual(choice["logprobs"]["tokens"], ["A", "", "€Hel"])+ self.assertEqual(choice["logprobs"]["text_offset"], [0, 1, 1])+ self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])++ def test_seam_codepoint_survives_a_later_stop_truncation(self):+ # The Euro sign is split across the prompt/generated seam and must decode as one+ # character while a later frame is cut by the stop. Four positions for four+ # frames: the echo position carrying the lead byte resolves nothing on its own and+ # reports "", and the character lands on the frame that completed it.+ base = _spawn_test_server(self, _seam_split_stop_engine())+ with _post_completions(base, {"model": "test-model", "prompt": "A€",+ "max_tokens": 4, "logprobs": 1, "echo": True,+ "stop": ["STOP"]}) as response:+ choice = json.load(response)["choices"][0]+ self.assertEqual(choice["text"], "A€x")+ self.assertEqual(choice["text"].count("€"), 1)+ self.assertNotIn("�", choice["text"])+ self.assertEqual(choice["logprobs"]["tokens"], ["A", "", "€", "x"])+ self.assertEqual(choice["logprobs"]["text_offset"], [0, 1, 1, 2])+ self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])++ def test_an_engine_that_sends_no_records_is_refused_not_half_answered(self):+ # The opt-in was accepted and the channel came back empty. Answering with the+ # completion and an empty array is the shape the arrays exist to prevent.+ base = _spawn_test_server(self, ScriptedEngine(+ chunks=("AA", "BB"), records=None,+ echoes=[(b"P", NAN, []), (b"Q", -0.1, [(1, -0.1)])]))+ status, error = _error_body(self, lambda: _post_completions(base, {+ "model": "test-model", "prompt": "PQ", "max_tokens": 4, "logprobs": 1,+ "echo": True}))+ self.assertEqual(status, 500)+ self.assertEqual(error["code"], "engine_logprob_records_incomplete")++ def test_a_length_limited_turn_reports_finish_reason_length(self):+ base = _spawn_test_server(self, ScriptedEngine(chunks=("ok",),+ length_limited=True))+ with _post_completions(base, {"model": "test-model", "prompt": "hi",+ "max_tokens": 1, "logprobs": 1}) as response:+ choice = json.load(response)["choices"][0]+ self.assertEqual(choice["finish_reason"], "length")+ self.assertEqual(choice["logprobs"]["tokens"], ["ok"])++ def test_every_response_joins_its_tokens_back_to_the_text_it_returned(self):+ # The invariant as a test rather than a comment, over every fixture in this file+ # that can produce a boundary.+ for name, build in (("leading marker", _ignored_leading_marker_engine),+ ("mid-token stop", _mid_token_stop_echo_engine),+ ("stop token", _stop_token_engine),+ ("seam and stop", _seam_split_stop_engine)):+ for echo in (False, True):+ with self.subTest(engine=name, echo=echo):+ base = _spawn_test_server(self, build())+ with _post_completions(base, {+ "model": "test-model", "prompt": "A€", "max_tokens": 8,+ "logprobs": 1, "echo": echo,+ "stop": ["lo", "STOP", "<|user|>"]}) as response:+ choice = json.load(response)["choices"][0]+ tokens = choice["logprobs"]["tokens"]+ offsets = choice["logprobs"]["text_offset"]+ self.assertEqual("".join(tokens), choice["text"])+ self.assertEqual(offsets, sorted(offsets))+ for offset in offsets:+ self.assertLessEqual(offset, len(choice["text"]))+++class PlainPathSpanWorkTest(unittest.TestCase):+ """What a request that never asked for per-token logprobs actually pays for the span+ layer, stage by stage. Counted at a seam -- the primitives each stage would call --+ rather than against a timing or a golden number, so the assertion is about work done.++ The thinking split builds nothing. The tool-call stage composes and exports nothing,+ which is the part that was worth gating; it still walks its own cut lists, because+ those ARE the deletion the content is sliced from rather than a map kept for a later+ reader. The exact residue is asserted below rather than described, so it cannot drift+ upward unnoticed."""++ def _counted(self, name):+ """Replace one span primitive with a counting pass-through for the duration of a+ request, and return the list its calls are recorded in."""+ calls = []+ original = getattr(openai_server, name)++ def counting(*args, **kwargs):+ calls.append(1)+ return original(*args, **kwargs)++ patcher = patch.object(openai_server, name, counting)+ patcher.start()+ self.addCleanup(patcher.stop)+ return calls++ def _chat(self, engine, **extra):+ base = _spawn_test_server(self, engine)+ body = {"model": "test-model", "max_tokens": 16,+ "messages": [{"role": "user", "content": "hi"}]}+ body.update(extra)+ with _post_chat(base, body) as response:+ return json.load(response)["choices"][0]++ THINKING = ScriptedEngine(+ chunks=("<think>", "why", "</think>", "the ", "answer"),+ records=[(b"<think>", -0.1, [(1, -0.1)]), (b"why", -0.2, [(2, -0.2)]),+ (b"</think>", -0.3, [(3, -0.3)]), (b"the ", -0.4, [(4, -0.4)]),+ (b"answer", -0.5, [(5, -0.5)])])++ TOOLS = [{"type": "function",+ "function": {"name": "search",+ "parameters": {"type": "object",+ "properties": {"q": {"type": "string"}}}}}]++ def test_the_thinking_split_records_nothing_on_a_plain_request(self):+ # No tools, so the thinking split is the only stage that could append a span.+ appends = self._counted("_append_span")+ choice = self._chat(self.THINKING, enable_thinking=True)+ self.assertEqual(choice["message"]["content"], "the answer")+ self.assertEqual(choice["message"]["reasoning_content"], "why")+ self.assertEqual(sum(appends), 0)++ def test_the_thinking_split_records_when_logprobs_are_asked_for(self):+ # The control: the same request with the opt-in builds the map it needs, so the+ # test above cannot be satisfied by a stage that never records at all.+ appends = self._counted("_append_span")+ choice = self._chat(self.THINKING, enable_thinking=True, logprobs=True,+ top_logprobs=1)+ self.assertEqual(choice["message"]["content"], "the answer")+ self.assertGreater(sum(appends), 0)++ def test_the_tool_call_parse_composes_and_exports_nothing_on_a_plain_request(self):+ # The composition and the exported maps are what the gate removes. The per-step+ # cut lists remain: `_apply_cuts` returns the survivors the content is built by+ # slicing, so removing them would mean a second implementation of the deletion.+ # Every count is asserted, including the ones that are not zero, so "no span map"+ # can never quietly become "some span map".+ engine = ScriptedEngine(+ chunks=("before ", "<tool_call>search<arg_key>q</arg_key>"+ "<arg_value>x</arg_value></tool_call>", " after"))+ counts = {name: self._counted(name) for name in+ ("_compose_span_maps", "_append_span", "_cut_span_map", "_apply_cuts",+ "_marker_cuts", "_project_span")}+ choice = self._chat(engine, tools=self.TOOLS)+ self.assertEqual(choice["message"]["content"], "before after")+ self.assertEqual(len(choice["message"]["tool_calls"]), 1)+ self.assertEqual({name: sum(calls) for name, calls in counts.items()},+ {"_compose_span_maps": 0, # nothing composed+ "_project_span": 0, # nothing read back+ "_append_span": 5, # inside the cut lists below+ "_cut_span_map": 4,+ "_apply_cuts": 4,+ "_marker_cuts": 2})+ # ...and no map leaves the stage for anyone to read.+ _content, _calls, box_map, content_map = openai_server._parse_tool_calls(+ "before <tool_call>search</tool_call> after", self.TOOLS, False)+ self.assertIsNone(box_map)+ self.assertIsNone(content_map)++ def test_the_tool_call_parse_composes_when_logprobs_are_asked_for(self):+ engine = ScriptedEngine(+ chunks=("before ", "<tool_call>search<arg_key>q</arg_key>"+ "<arg_value>x</arg_value></tool_call>", " after"),+ records=[(b"before ", -0.1, [(1, -0.1)]),+ ("<tool_call>search<arg_key>q</arg_key>"+ "<arg_value>x</arg_value></tool_call>".encode(), -0.2, [(2, -0.2)]),+ (b" after", -0.3, [(3, -0.3)])])+ composes = self._counted("_compose_span_maps")+ choice = self._chat(engine, tools=self.TOOLS, logprobs=True, top_logprobs=1)+ self.assertEqual(choice["message"]["content"], "before after")+ self.assertGreater(sum(composes), 0)+++class LogprobsStageCompositionTest(unittest.TestCase):+ """The chat path runs the stop filter, then the thinking split, then the tool-call+ parse. `logprobs.content` must describe `message.content` after all three."""++ THINKING = ScriptedEngine(+ chunks=("<think>", "why", "</think>", "the ", "answer"),+ records=[(b"<think>", -0.1, [(1, -0.1)]),+ (b"why", -0.2, [(2, -0.2)]),+ (b"</think>", -0.3, [(3, -0.3)]),+ (b"the ", -0.4, [(4, -0.4)]),+ (b"answer", -0.5, [(5, -0.5)])])++ def _chat(self, engine, **extra):+ base = _spawn_test_server(self, engine)+ body = {"model": "test-model", "max_tokens": 16, "logprobs": True,+ "top_logprobs": 1, "messages": [{"role": "user", "content": "hi"}]}+ body.update(extra)+ with _post_chat(base, body) as response:+ return json.load(response)["choices"][0]++ def test_content_is_a_projection_of_the_stream_not_a_subset(self):+ # Cardinality first, then absence: a build that returns an empty array satisfies+ # every "X is not described" assertion trivially, so the count has to be asserted+ # before anything about what is missing.+ choice = self._chat(self.THINKING, enable_thinking=True)+ entries = choice["logprobs"]["content"]+ self.assertEqual(choice["message"]["content"], "the answer")+ self.assertEqual(choice["message"]["reasoning_content"], "why")+ self.assertEqual(len(entries), 2)+ self.assertEqual([entry["token"] for entry in entries], ["the ", "answer"])+ # Each entry's logprob is its OWN record's, not the one at the same index in the+ # raw stream: a chain composed one stage short keeps the count and the text and+ # still attaches the reasoning tokens' numbers.+ self.assertEqual([entry["logprob"] for entry in entries], [-0.4, -0.5])+ self.assertEqual("".join(entry["token"] for entry in entries),+ choice["message"]["content"])+ # ...and the reasoning and the markers are not described as content.+ self.assertNotIn("why", [entry["token"] for entry in entries])++ def test_tool_call_syntax_is_not_described_as_content(self):+ engine = ScriptedEngine(+ chunks=("before ", "<tool_call>search<arg_key>q</arg_key>"+ "<arg_value>x</arg_value></tool_call>", " after"),+ records=[(b"before ", -0.1, [(1, -0.1)]),+ ("<tool_call>search<arg_key>q</arg_key>"+ "<arg_value>x</arg_value></tool_call>".encode(), -0.2, [(2, -0.2)]),+ (b" after", -0.3, [(3, -0.3)])])+ choice = self._chat(engine, tools=[{+ "type": "function",+ "function": {"name": "search",+ "parameters": {"type": "object",+ "properties": {"q": {"type": "string"}}}}}])+ entries = choice["logprobs"]["content"]+ self.assertEqual(choice["message"]["content"], "before after")+ self.assertEqual(len(choice["message"]["tool_calls"]), 1)+ self.assertEqual(len(entries), 2)+ self.assertEqual([entry["token"] for entry in entries], ["before ", " after"])+ self.assertEqual([entry["logprob"] for entry in entries], [-0.1, -0.3])+ self.assertEqual("".join(entry["token"] for entry in entries),+ choice["message"]["content"])++ def test_all_three_stages_compose_in_one_turn(self):+ # Thinking and tools together: the stop filter, the thinking split and the+ # tool-call parse each cut something, and the chain has to be composed in that+ # order. Composed the other way round, or stopped one stage early, the entries+ # stop joining back to `message.content`.+ engine = ScriptedEngine(+ chunks=("<think>", "why", "</think>", "before ",+ "<tool_call>search<arg_key>q</arg_key>"+ "<arg_value>x</arg_value></tool_call>", " after"),+ records=[(b"<think>", -0.1, [(1, -0.1)]),+ (b"why", -0.2, [(2, -0.2)]),+ (b"</think>", -0.3, [(3, -0.3)]),+ (b"before ", -0.4, [(4, -0.4)]),+ ("<tool_call>search<arg_key>q</arg_key>"+ "<arg_value>x</arg_value></tool_call>".encode(), -0.5, [(5, -0.5)]),+ (b" after", -0.6, [(6, -0.6)])])+ choice = self._chat(engine, enable_thinking=True, tools=[{+ "type": "function",+ "function": {"name": "search",+ "parameters": {"type": "object",+ "properties": {"q": {"type": "string"}}}}}])+ entries = choice["logprobs"]["content"]+ self.assertEqual(choice["message"]["reasoning_content"], "why")+ self.assertEqual(choice["message"]["content"], "before after")+ self.assertEqual(len(choice["message"]["tool_calls"]), 1)+ self.assertEqual(len(entries), 2)+ self.assertEqual([entry["token"] for entry in entries], ["before ", " after"])+ self.assertEqual([entry["logprob"] for entry in entries], [-0.4, -0.6])+ self.assertEqual("".join(entry["token"] for entry in entries),+ choice["message"]["content"])++ def test_a_plain_chat_turn_describes_every_token(self):+ # The control: with no stage cutting anything, nothing is dropped.+ choice = self._chat(ScriptedEngine(chunks=("the ", "answer")))+ entries = choice["logprobs"]["content"]+ self.assertEqual(len(entries), 2)+ self.assertEqual("".join(entry["token"] for entry in entries),+ choice["message"]["content"])+++class EchoCoverageTest(unittest.TestCase):+ """`echo: true` rebuilds `text` from the logprob records, because `text` and+ `text_offset` have to describe the same reconstruction. If the records stop short of+ the generated stream that rebuild silently drops the uncovered tail and the client gets+ a truncated completion with a 200 -- the worst shape a fault here can take, because+ nothing in the response says anything is missing. It is refused by name instead."""++ BODY = {"model": "test-model", "prompt": "PQ", "echo": True,+ "logprobs": 1, "max_tokens": 3}++ def _post(self, engine, body):+ return _post_completions(_spawn_test_server(self, engine), body)++ def test_records_short_of_the_stream_are_refused_not_truncated(self):+ # Unrefused, this is a 200 carrying "PQAA" with the client's "BB" and "CC" gone.+ status, error = _error_body(+ self, lambda: self._post(_partial_coverage_engine(covered=1), dict(self.BODY)))+ self.assertEqual(status, 500)+ self.assertEqual(error["code"], "engine_logprob_records_incomplete")++ def test_full_coverage_is_served_normally(self):+ # The control, and it is not optional: without it the check above is satisfied+ # just as well by refusing every echo request.+ with self._post(_partial_coverage_engine(covered=3), dict(self.BODY)) as response:+ choice = json.load(response)["choices"][0]+ self.assertEqual(choice["text"], "PQAABBCC")+ self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])+ self.assertEqual(choice["logprobs"]["text_offset"], [0, 1, 2, 4, 6])++ def test_a_generation_ending_mid_codepoint_is_served(self):+ # Control 1 of 2. A coverage check written as a string comparison reads an+ # unflushed incremental decoder as a short generation and turns this valid+ # response into a 500. Counting in raw-stream coordinates cannot.+ engine = _partial_coverage_engine(+ covered=3, chunks=("AA", "BB", "�"),+ record_bytes=(b"AA", b"BB", b"\xc3"))+ with self._post(engine, dict(self.BODY)) as response:+ choice = json.load(response)["choices"][0]+ self.assertEqual(choice["text"], "PQAABB�")+ self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])++ def test_a_codepoint_split_across_the_seam_is_served(self):+ # Control 2 of 2. With `echo` on, the reconstruction decodes the generated half+ # with the decoder that already consumed the prompt bytes, so a codepoint+ # straddling the seam reads as one character there and as replacement characters+ # in the raw stream, on purpose.+ engine = _partial_coverage_engine(+ covered=1, chunks=("��",), record_bytes=(b"\x82\xac",),+ echo_bytes=(b"A", b"\xe2"))+ with self._post(engine, dict(self.BODY)) as response:+ choice = json.load(response)["choices"][0]+ self.assertEqual(choice["text"], "A€")+ self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])++ def test_echo_false_is_refused_on_the_same_coverage_gap(self):+ # The arrays have to describe the whole text on every shape that carries them, not+ # only the one that rebuilds `text` out of them. Without this, `echo: false`+ # returns "AABBCC" with `tokens == ["AA"]` and a 200.+ status, error = _error_body(+ self, lambda: self._post(_partial_coverage_engine(covered=1),+ dict(self.BODY, echo=False)))+ self.assertEqual(status, 500)+ self.assertEqual(error["code"], "engine_logprob_records_incomplete")+ self.assertEqual(error["param"], "logprobs")++ def test_echo_false_full_coverage_is_served(self):+ # The control for the refusal above: same engine, same shape, every record present.+ with self._post(_partial_coverage_engine(covered=3),+ dict(self.BODY, echo=False)) as response:+ choice = json.load(response)["choices"][0]+ self.assertEqual(choice["text"], "AABBCC")+ self.assertEqual("".join(choice["logprobs"]["tokens"]), choice["text"])++ def test_chat_is_refused_on_the_same_coverage_gap(self):+ # The worst shape this surface can take, reached by the other door: a 200 with a+ # full message.content and logprobs.content describing a prefix of it.+ for covered, tokens in ((1, "short"), (0, "absent")):+ with self.subTest(records=tokens):+ base = _spawn_test_server(self, _partial_coverage_engine(covered=covered))+ status, error = _error_body(self, lambda: _post_chat(base, {+ "model": "test-model", "max_tokens": 3, "logprobs": True,+ "top_logprobs": 1,+ "messages": [{"role": "user", "content": "hi"}]}))+ self.assertEqual(status, 500)+ self.assertEqual(error["code"], "engine_logprob_records_incomplete")+ self.assertEqual(error["param"], "logprobs")++ def test_chat_full_coverage_is_served(self):+ base = _spawn_test_server(self, _partial_coverage_engine(covered=3))+ with _post_chat(base, {"model": "test-model", "max_tokens": 3, "logprobs": True,+ "top_logprobs": 1,+ "messages": [{"role": "user", "content": "hi"}]}) as response:+ choice = json.load(response)["choices"][0]+ entries = choice["logprobs"]["content"]+ self.assertEqual("".join(entry["token"] for entry in entries),+ choice["message"]["content"])+ self.assertEqual(choice["message"]["content"], "AABBCC")++ def test_echo_with_no_prompt_echo_records_is_refused_by_name(self):+ # The engine answered half the opt-in: every generated record, no ECHO frame. The+ # prompt echo is the half `echo` asked for.+ base = _spawn_test_server(self, ScriptedEngine(chunks=("Hi",), echoes=()))+ status, error = _error_body(self, lambda: _post_completions(base, {+ "model": "test-model", "prompt": "Hi", "max_tokens": 2, "logprobs": 1,+ "echo": True}))+ self.assertEqual(status, 500)+ self.assertEqual(error["code"], "engine_logprob_records_incomplete")+ self.assertEqual(error["param"], "echo")+++class PinnedPrefixEchoTest(unittest.TestCase):+ """A KV slot holding a pin snapshot echoes prompt positions from the snapshot's length+ rather than from 0. Re-basing that run would attach every logprob to the wrong prompt+ token, so it is refused by name."""++ def test_a_pinned_prefix_is_refused_by_name(self):+ base = _spawn_test_server(self, _pinned_prefix_engine())+ status, error = _error_body(self, lambda: _post_completions(base, {+ "model": "test-model", "prompt": "abc", "logprobs": 1, "echo": True,+ "max_tokens": 2}))+ self.assertEqual(status, 503)+ self.assertEqual(error["code"], "engine_pinned_prefix_not_echoed")+ self.assertEqual(error["param"], "echo")++ def test_an_unpinned_prompt_control_is_served(self):+ base = _spawn_test_server(self, ScriptedEngine(+ chunks=("ok",),+ echoes=[(b"a", NAN, []), (b"b", -0.1, [(1, -0.1)])]))+ with _post_completions(base, {"model": "test-model", "prompt": "ab",+ "logprobs": 1, "echo": True,+ "max_tokens": 2}) as response:+ choice = json.load(response)["choices"][0]+ self.assertEqual(choice["text"], "abok")+++class ClientVisibleEngineFaultCodeTest(unittest.TestCase):+ """Each engine-fault path fails closed AND tells the client which fault. Every test+ reads the code and param out of the response body, because an exception with a good+ name that never reaches the wire is the defect rather than the fix. Each has a control+ that must come back 200, so "names the fault" can never be satisfied by a server that+ refuses everything.++ The codes say what the ENGINE did, never what this gateway did about it, so they+ survive a change of reaction."""++ def test_duplicate_candidate_names_itself_with_the_callers_own_param(self):+ # The same engine table is requested by two different parameters, and a client+ # branching on the code is told which of its own fields went unanswered.+ for post, body, param in (+ (_post_completions,+ {"model": "test-model", "prompt": "hi", "logprobs": 2, "max_tokens": 1},+ "logprobs"),+ (_post_chat,+ {"model": "test-model", "messages": [{"role": "user", "content": "hi"}],+ "logprobs": True, "top_logprobs": 2, "max_tokens": 1},+ "top_logprobs")):+ with self.subTest(param=param):+ base = _spawn_test_server(self, _duplicate_candidate_engine())+ status, error = _error_body(self, lambda: post(base, body))+ self.assertEqual(status, 500)+ self.assertEqual(error["code"], "engine_duplicate_logprob_candidate")+ self.assertEqual(error["param"], param)++ def test_distinct_candidates_control_is_served_on_both_surfaces(self):+ base = _spawn_test_server(self, _duplicate_candidate_engine(duplicate=False))+ with _post_completions(base, {"model": "test-model", "prompt": "hi",+ "logprobs": 2, "max_tokens": 1}) as response:+ body = json.load(response)+ self.assertEqual(len(body["choices"][0]["logprobs"]["top_logprobs"][0]), 2)+ base = _spawn_test_server(self, _duplicate_candidate_engine(duplicate=False))+ with _post_chat(base, {"model": "test-model",+ "messages": [{"role": "user", "content": "hi"}],+ "logprobs": True, "top_logprobs": 2,+ "max_tokens": 1}) as response:+ body = json.load(response)+ self.assertEqual(+ len(body["choices"][0]["logprobs"]["content"][0]["top_logprobs"]), 2)++ def _tail_engine(self, frame):+ """A real Engine over a fake process, so the frame is parsed by the dispatcher+ exactly as it is in production and the named error has to travel from the+ dispatcher thread to the waiting request thread and out onto the wire."""+ def respond(process, written):+ request_id = written.split()[1]+ process.stdout.feed(frame.replace(b"{id}", request_id))+ process.stdout.feed(b"DONE " + request_id + b" STAT 1 1 0 1 1 0\n")++ process = FakeProcess(respond)+ with patch("openai_server.subprocess.Popen", return_value=process):+ engine = Engine("glm", "model")+ self.addCleanup(engine.close)+ return engine++ def test_malformed_tail_names_itself_to_the_client(self):+ base = _spawn_test_server(+ self, self._tail_engine(b"DATA {id} 1 -0.5 1 3\nx\n"))+ status, error = _error_body(self, lambda: _post_completions(base, {+ "model": "test-model", "prompt": "hi", "logprobs": 1, "max_tokens": 1}))+ self.assertEqual(status, 500)+ self.assertEqual(error["code"], "engine_logprob_tail_malformed")+ self.assertEqual(error["param"], "logprobs")++ def test_well_formed_tail_control_is_served(self):+ # The control that matters most here: the relay must carry a good frame through+ # untouched, or "names the fault" is satisfied by a build that fails every+ # logprobs request.+ base = _spawn_test_server(+ self, self._tail_engine(b"DATA {id} 1 -0.5 1 3 -0.5\nx\n"))+ with _post_completions(base, {"model": "test-model", "prompt": "hi",+ "logprobs": 1, "max_tokens": 1}) as response:+ body = json.load(response)+ self.assertNotIn("error", body)+ self.assertEqual(body["choices"][0]["text"], "x")++ def test_an_unsupported_engine_names_its_refusal_in_the_body(self):+ base = _spawn_test_server(self, ScriptedEngine(supports_logprobs_echo=False))+ status, error = _error_body(self, lambda: _post_completions(base, {+ "model": "test-model", "prompt": "hi", "logprobs": 1}))+ self.assertEqual(status, 400)+ self.assertEqual(error["code"], "unsupported_parameter")+ self.assertEqual(error["param"], "logprobs")++ def test_the_same_engine_serves_a_plain_request(self):+ base = _spawn_test_server(self, ScriptedEngine(supports_logprobs_echo=False))+ with _post_completions(base, {"model": "test-model", "prompt": "hi"}) as response:+ body = json.load(response)+ self.assertEqual(body["choices"][0]["text"], "ok")+ self.assertIsNone(body["choices"][0]["logprobs"])+++class CapabilitySplitIndependenceTest(unittest.TestCase):+ """`supports_logprobs_echo` and `supports_tok_ids` gate two unrelated SUBMIT extension+ keys and must be settable one without the other. Both are `arch == "glm"` today, so+ nothing short of a double with the two flags set to different values can show the+ split is real rather than cosmetic."""++ def test_logprobs_is_refused_when_only_token_id_intake_is_available(self):+ base = _spawn_test_server(self, ScriptedEngine(supports_logprobs_echo=False,+ supports_tok_ids=True))+ status, error = _error_body(self, lambda: _post_completions(base, {+ "model": "test-model", "prompt": "hi", "logprobs": 1}))+ self.assertEqual(status, 400)+ self.assertEqual(error["param"], "logprobs")++ def test_logprobs_is_served_when_only_the_numeric_channel_is_available(self):+ base = _spawn_test_server(self, ScriptedEngine(supports_logprobs_echo=True,+ supports_tok_ids=False))+ with _post_completions(base, {"model": "test-model", "prompt": "hi",+ "logprobs": 1}) as response:+ choice = json.load(response)["choices"][0]+ self.assertEqual(choice["logprobs"]["tokens"], ["ok"])++ def test_an_engine_sets_the_two_flags_independently_of_each_other(self):+ # Set from the same arch check today, but each read site checks the one it means:+ # one flag forced off must not take the other with it.+ process = FakeProcess(lambda process, written: None)+ with patch("openai_server.subprocess.Popen", return_value=process):+ engine = Engine("glm", "model")+ self.addCleanup(engine.close)+ self.assertTrue(engine.supports_logprobs_echo)+ self.assertTrue(engine.supports_tok_ids)+ engine.supports_logprobs_echo = False+ self.assertTrue(engine.supports_tok_ids)+++class SubmitHeaderExtensionTest(unittest.TestCase):+ """The SUBMIT header, byte for byte. A request that opts into nothing must produce the+ header it produced before this channel existed, and an opted-in one must carry the+ extension keys behind the seventh numeric field."""++ def _engine(self, expected, frames):+ def respond(process, frame):+ self.assertEqual(frame, expected)+ process.stdout.feed(frames)++ process = FakeProcess(respond)+ with patch("openai_server.subprocess.Popen", return_value=process):+ engine = Engine("glm", "model")+ return engine, process++ def test_a_plain_request_submit_header_is_byte_identical(self):+ expected = b"SUBMIT 1 0 5 4 0.25 0.9\n" + b"hello\n"+ engine, process = self._engine(+ expected, b"DATA 1 2\nok\nDONE 1 STAT 1 2.5 0 1.0 5 0\n")+ engine.generate("hello", 4, 0.25, 0.9, [].append)+ engine.close()+ self.assertEqual(process.writes, [expected])++ def test_an_opted_in_request_carries_the_extension_behind_the_seventh_field(self):+ # coli_submit_parse reaches its key=value arm only after seven numeric fields, and+ # sending the extension on every request would make the plain-header test above+ # fail.+ expected = b"SUBMIT 1 0 5 4 0.25 0.9 0 logprobs=2\n" + b"hello\n"+ engine, process = self._engine(+ expected,+ b"ACCEPT 1 5\nECHO 1 1 0 nan 0\nh\n"+ b"DATA 1 2 -0.223144 1 3 -0.223144\nok\nDONE 1 STAT 1 2.5 0 1.0 5 0\n")+ stats = engine.generate("hello", 4, 0.25, 0.9, [].append, logprobs=2,+ gbytes_before_ext=True)+ engine.close()+ self.assertEqual(process.writes, [expected])+ self.assertEqual(stats["logprobs"]["generated"],+ [(b"ok", {"lp": -0.223144, "topk": [(3, -0.223144)]})])++ def test_the_seventh_field_is_sent_only_when_the_caller_asks_for_it(self):+ # It is a request, not a rule: engines that parse a header without that field+ # reject or mis-read one carrying it, so only a caller whose engine uses+ # coli_submit_parse asks for it. `pin` rides the same request.+ expected = b"SUBMIT 1 0 5 0 0.25 0.9 logprobs=1 pin=1\n" + b"hello\n"+ engine, process = self._engine(+ expected, b"ACCEPT 1 5\nECHO 1 1 0 nan 0\nh\nDONE 1 STAT 0 2.5 0 1.0 5 0\n")+ engine.generate("hello", 0, 0.25, 0.9, lambda _chunk: None, logprobs=1, pin=True,+ on_echo=lambda _record: None)+ engine.close()+ self.assertEqual(process.writes, [expected])+++class EngineExtensionCallSiteTest(unittest.TestCase):+ """Each of this server's call sites into Engine.generate() applies the same extension+ profile, so none can be left behind. For each: an opted-in request emits the extended+ header, and a plain request emits the base header."""++ def _header(self, path, body, engine_frames=None):+ """The SUBMIT header one HTTP request produces, taken off a fake engine process."""+ headers = []++ def respond(process, frame):+ headers.append(frame.split(b"\n", 1)[0])+ request_id = frame.split()[1]+ process.stdout.feed(engine_frames.replace(b"{id}", request_id)+ if engine_frames else+ b"ACCEPT " + request_id + b" 1\n"+ b"DATA " + request_id + b" 2\nok\n"+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")++ process = FakeProcess(respond)+ with patch("openai_server.subprocess.Popen", return_value=process):+ engine = Engine("glm", "model")+ self.addCleanup(engine.close)+ base = _spawn_test_server(self, engine)+ with _post_json(base, path, body) as response:+ response.read()+ return headers[0]++ OPTED_IN_FRAMES = (b"ACCEPT {id} 1\n"+ b"DATA {id} 2 -0.5 1 3 -0.5\nok\n"+ b"DONE {id} STAT 1 2.5 0 1.0 5 0\n")++ def test_non_streaming_completions_site(self):+ plain = self._header("/v1/completions", {"model": "test-model", "prompt": "hi"})+ self.assertNotIn(b"logprobs=", plain)+ opted = self._header("/v1/completions",+ {"model": "test-model", "prompt": "hi", "logprobs": 2},+ self.OPTED_IN_FRAMES)+ self.assertIn(b" 0 logprobs=2", opted)++ def test_non_streaming_chat_site(self):+ messages = [{"role": "user", "content": "hi"}]+ plain = self._header("/v1/chat/completions",+ {"model": "test-model", "messages": messages})+ self.assertNotIn(b"logprobs=", plain)+ opted = self._header("/v1/chat/completions",+ {"model": "test-model", "messages": messages,+ "logprobs": True, "top_logprobs": 1},+ self.OPTED_IN_FRAMES)+ self.assertIn(b" 0 logprobs=1", opted)++ def test_streaming_plain_site_keeps_the_base_header(self):+ # Streaming with logprobs is a named 400, so this site can only ever be reached+ # without the extension -- and must never grow it.+ header = self._header("/v1/completions",+ {"model": "test-model", "prompt": "hi", "stream": True})+ self.assertNotIn(b"logprobs=", header)+ status, error = _error_body(self, lambda: self._header(+ "/v1/completions",+ {"model": "test-model", "prompt": "hi", "stream": True, "logprobs": 1}))+ self.assertEqual((status, error["param"]), (400, "logprobs"))++ def test_streaming_chat_with_tools_site_keeps_the_base_header(self):+ messages = [{"role": "user", "content": "hi"}]+ tools = [{"type": "function",+ "function": {"name": "search",+ "parameters": {"type": "object", "properties": {}}}}]+ header = self._header("/v1/chat/completions",+ {"model": "test-model", "messages": messages,+ "tools": tools, "stream": True})+ self.assertNotIn(b"logprobs=", header)+ status, error = _error_body(self, lambda: self._header(+ "/v1/chat/completions",+ {"model": "test-model", "messages": messages, "tools": tools,+ "stream": True, "logprobs": True}))+ self.assertEqual((status, error["param"]), (400, "logprobs"))++ def test_the_messages_endpoint_ignores_a_logprobs_field(self):+ # `logprobs` is not in the Anthropic request shape, and this endpoint builds its+ # own translated request rather than forwarding the client's body, so the field+ # never reaches the options parser. A 200 with no logprobs anywhere, for either+ # endpoint's spelling of the field.+ for value in (True, 1):+ with self.subTest(value=value):+ base = _spawn_test_server(self, ScriptedEngine(chunks=("Hello",)))+ with _post_json(base, "/v1/messages",+ {"model": "test-model", "max_tokens": 4,+ "messages": [{"role": "user", "content": "hi"}],+ "logprobs": value}) as response:+ body = json.load(response)+ self.assertEqual(body["content"], [{"type": "text", "text": "Hello"}])+ self.assertNotIn("logprobs", json.dumps(body))++ def test_the_messages_endpoint_keeps_the_base_header(self):+ # /v1/messages sends no extension key, and nothing about this change may give it+ # one.+ header = self._header("/v1/messages",+ {"model": "test-model", "max_tokens": 4,+ "messages": [{"role": "user", "content": "hi"}]})+ self.assertEqual(header.split(b" ")[:2], [b"SUBMIT", b"1"])+ self.assertNotIn(b"logprobs=", header)+ self.assertNotIn(b"=", header)+++class DispatcherLogprobTailTest(unittest.TestCase):+ """A numeric tail is parsed only for the request that asked for the channel. A+ malformed one fails that request alone; a frame carrying fields a request never asked+ for is tolerated, as it was before the channel existed."""++ def _engine(self, respond):+ process = FakeProcess(respond)+ with patch("openai_server.subprocess.Popen", return_value=process):+ engine = Engine("glm", "model")+ self.addCleanup(engine.close)+ return engine, process++ def test_extra_fields_on_a_plain_requests_frame_are_tolerated(self):+ # The request never asked for the channel, so the tail is not parsed -- not even+ # to reject it. This is the base behaviour, restored.+ def respond(process, frame):+ request_id = frame.split()[1]+ process.stdout.feed(b"DATA " + request_id + b" 2 garbage fields here\nok\n"+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")++ engine, _process = self._engine(respond)+ chunks = []+ stats = engine.generate("hi", 4, 0.25, 0.9, chunks.append)+ self.assertEqual(chunks, ["ok"])+ self.assertNotIn("logprobs", stats)++ def _concurrent_fault(self, bad_frame):+ """Drive one plain request and one opted-in request over the same engine, with the+ opted-in request's fault arriving while the plain one is still in flight.++ The plain request is submitted first and left waiting for its DONE. The opted-in+ request's SUBMIT then triggers the bad frame, that request's terminal frame, and+ finally the plain request's. Returns `(plain result, opted-in exception)`."""+ first_submitted = threading.Event()++ def respond(process, frame):+ request_id = frame.split()[1]+ if b"logprobs=" in frame.split(b"\n", 1)[0]:+ process.stdout.feed(bad_frame.replace(b"{id}", request_id))+ process.stdout.feed(b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")+ process.stdout.feed(b"DONE 1 STAT 1 2.5 0 1.0 5 0\n")+ else:+ process.stdout.feed(b"DATA " + request_id + b" 2\nok\n")+ first_submitted.set()++ engine, _process = self._engine(respond)+ plain = {}++ def run_plain():+ chunks = []+ try:+ plain["stats"] = engine.generate("hi", 4, 0.25, 0.9, chunks.append)+ plain["chunks"] = chunks+ except Exception as error: # recorded, asserted by the caller+ plain["error"] = error++ thread = threading.Thread(target=run_plain, daemon=True)+ thread.start()+ self.assertTrue(first_submitted.wait(2))+ with self.assertRaises(APIError) as caught:+ engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=1,+ gbytes_before_ext=True)+ thread.join(timeout=2)+ self.assertFalse(thread.is_alive())+ return plain, caught.exception++ def test_a_malformed_tail_spares_a_request_that_is_in_flight(self):+ # The containment rule's whole point: not that the dispatcher survived the fault,+ # but that the request which never asked for the channel is untouched by it.+ # Failing every pending request instead leaves the plain request holding an error+ # it did not earn.+ plain, error = self._concurrent_fault(b"DATA {id} 2 -0.5 1 3\nok\n")+ self.assertEqual(error.status, 500)+ self.assertEqual(error.code, "engine_logprob_tail_malformed")+ self.assertNotIn("error", plain)+ self.assertEqual(plain["chunks"], ["ok"])+ self.assertEqual(plain["stats"]["completion_tokens"], 1)++ def test_a_malformed_echo_logprob_on_a_request_that_never_opted_in(self):+ # The `lp` field is read for EVERY ECHO frame, opted in or not, because the+ # closed-set scorer's own `logprob` key comes from it -- so unlike the tail, this+ # one is reachable by a request with no numeric channel at all. That is the row of+ # the four that was still reaching the dispatcher's blanket handler, which fails+ # every request in flight and stops the dispatcher for good. Here the bad frame+ # goes to the NON-opted-in request and a second request is in flight beside it.+ second_submitted = threading.Event()++ def respond(process, frame):+ request_id = frame.split()[1]+ if b"logprobs=" in frame.split(b"\n", 1)[0]:+ process.stdout.feed(b"DATA " + request_id + b" 2 -0.5 1 3 -0.5\nok\n")+ second_submitted.set()+ else:+ process.stdout.feed(b"ECHO " + request_id + b" 1 0 zz 0\nh\n")+ # The fault is delivered on the turn's terminal frame, so the engine has+ # to end the turn for the request to be answered at all.+ process.stdout.feed(+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")++ engine, _process = self._engine(respond)+ opted = {}++ def run_opted():+ chunks = []+ try:+ opted["stats"] = engine.generate(+ "hi", 4, 0.25, 0.9, chunks.append, logprobs=1,+ gbytes_before_ext=True, on_echo=lambda _r: None)+ opted["chunks"] = chunks+ except Exception as error: # recorded, asserted below+ opted["error"] = error++ thread = threading.Thread(target=run_opted, daemon=True)+ thread.start()+ self.assertTrue(second_submitted.wait(2))+ with self.assertRaises(APIError) as caught:+ engine.generate("hi", 4, 0.25, 0.9, [].append, on_echo=lambda _r: None)+ self.assertEqual(caught.exception.status, 500)+ self.assertEqual(caught.exception.code, "engine_logprob_tail_malformed")+ self.assertIsNone(engine.dispatcher_error)+ # The request beside it never saw the frame and is still being served.+ self.assertNotIn("error", opted)+ self.assertTrue(thread.is_alive())+ thread.join(timeout=0.1)++ def test_a_malformed_echo_position_spares_a_request_that_is_in_flight(self):+ # The ECHO `pos` field rides the same opted-in frame as the numeric tail and gets+ # the same containment. Raising inside the dispatcher instead kills it, and every+ # concurrent request on the engine fails with that request's fault.+ plain, error = self._concurrent_fault(b"ECHO {id} 1 x nan 0\nh\n")+ self.assertEqual(error.status, 500)+ # Its own code: the position and the numeric tail are different fields of the+ # frame, and a client branching on the code is told which one was unreadable.+ self.assertEqual(error.code, "engine_echo_position_malformed")+ self.assertEqual(error.param, "echo")+ self.assertNotIn("error", plain)+ self.assertEqual(plain["chunks"], ["ok"])++ def _poll(self, predicate, what):+ deadline = time.monotonic() + 2+ while time.monotonic() < deadline:+ if predicate():+ return+ time.sleep(0.005)+ self.fail(what)++ def test_a_recorded_fault_keeps_the_turn_open_until_its_terminal_frame(self):+ # The fault is found mid-turn and raised at the end of it. Delivering it when it+ # is found would unwind the request thread, and with it the scheduler admission,+ # while the engine is still generating and no CANCEL has been written -- the next+ # request would then submit into a pipe nobody is reading. The observable form of+ # "the turn is still the engine's" is that the pending entry is still there.+ release = threading.Event()++ def respond(process, frame):+ request_id = frame.split()[1]+ process.stdout.feed(b"DATA " + request_id + b" 2 -0.5 1 3\nok\n")++ def terminal():+ release.wait(2)+ process.stdout.feed(+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")++ threading.Thread(target=terminal, daemon=True).start()++ engine, process = self._engine(respond)+ result = {}++ def run():+ try:+ engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=1,+ gbytes_before_ext=True)+ except APIError as error:+ result["error"] = error++ thread = threading.Thread(target=run, daemon=True)+ thread.start()+ self._poll(lambda: (engine.pending.get("1") is not None+ and engine.pending["1"].failed is not None),+ "the fault was never recorded")+ # Recorded, held, and not yet delivered.+ self.assertIn("1", engine.pending)+ self.assertNotIn("error", result)+ self.assertTrue(thread.is_alive())+ release.set()+ thread.join(timeout=2)+ self.assertFalse(thread.is_alive())+ self.assertEqual(result["error"].code, "engine_logprob_tail_malformed")+ self.assertNotIn("1", engine.pending)+ # No CANCEL was written: the turn was never abandoned.+ self.assertEqual([w for w in process.writes if w.startswith(b"CANCEL")], [])++ def test_frames_for_a_failed_or_finished_request_are_ignored(self):+ # While the fault is recorded the entry stays, and that request's later frames are+ # routed to it and discarded; once its terminal frame has been delivered the entry+ # is gone, and anything further for that id belongs to no request. Neither may be+ # treated as a fault of its own: the dispatcher keeps running and the next request+ # on the same engine completes normally.+ def respond(process, frame):+ request_id = frame.split()[1]+ if b"logprobs=" in frame.split(b"\n", 1)[0]:+ process.stdout.feed(b"DATA " + request_id + b" 2 -0.5 1 3\nok\n")+ process.stdout.feed(b"DATA " + request_id + b" 2\nxx\n")+ process.stdout.feed(b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")+ # ...and two frames the engine sends after the turn it already ended,+ # which belong to no pending entry at all.+ process.stdout.feed(b"DATA " + request_id + b" 2\nyy\n")+ process.stdout.feed(b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")+ else:+ process.stdout.feed(b"DATA " + request_id + b" 2\nok\n"+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")++ engine, _process = self._engine(respond)+ with self.assertRaises(APIError) as caught:+ engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=1,+ gbytes_before_ext=True)+ self.assertEqual(caught.exception.code, "engine_logprob_tail_malformed")+ chunks = []+ stats = engine.generate("hi", 4, 0.25, 0.9, chunks.append)+ self.assertEqual(chunks, ["ok"])+ self.assertEqual(stats["completion_tokens"], 1)+ self.assertIsNone(engine.dispatcher_error)++ def _faulted_turn(self, terminal, cancelled=None, settle=1.0):+ """One opted-in request whose numeric tail is malformed, with `terminal` delivered+ only once the server answers the fault -- so the turn stays open long enough for+ the request thread's idle poll to run, which is where that answer is written.++ A timer delivers `terminal` anyway after `settle` seconds, so a build that answers+ nothing fails an assertion instead of hanging the suite. Returns `(raised exception+ or None, the frames the server wrote)`."""+ answered = threading.Event()++ def respond(process, frame):+ if not frame.startswith(b"SUBMIT"):+ answered.set() # STOP or CANCEL: the turn can end now+ process.stdout.feed(terminal.replace(b"{id}", frame.split()[1]))+ return+ request_id = frame.split()[1]+ process.stdout.feed(b"DATA " + request_id + b" 2 -0.5 1 3\nok\n")++ def fallback():+ if not answered.wait(settle):+ process.stdout.feed(terminal.replace(b"{id}", request_id))++ threading.Thread(target=fallback, daemon=True).start()++ engine, process = self._engine(respond)+ raised = None+ try:+ engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=1,+ gbytes_before_ext=True, cancelled=cancelled)+ except Exception as error: # the class is what the test asserts+ raised = error+ return raised, process.writes++ def test_a_cancellation_outranks_the_recorded_fault_on_a_terminal_done(self):+ # The engine may end a cancelled turn with DONE rather than ERROR CANCELLED. The+ # client left either way, so the turn is a cancellation, not a server failure --+ # which is what the scheduler books when anything but ClientCancelled comes out,+ # and what the metrics then count.+ raised, writes = self._faulted_turn(b"DONE {id} STAT 1 2.5 0 1.0 5 0\n",+ cancelled=lambda: True)+ self.assertIsInstance(raised, ClientCancelled)+ self.assertEqual([w for w in writes if w.startswith(b"CANCEL")], [b"CANCEL 1\n"])+ self.assertEqual([w for w in writes if w.startswith(b"STOP")], [])++ def test_a_cancellation_outranks_the_recorded_fault(self):+ # The client left mid-turn on a request that also carries a fault of its own. That+ # is answered the way it is answered on a healthy turn -- ClientCancelled, not a+ # 500 written to a socket that is already closed.+ raised, writes = self._faulted_turn(b"ERROR {id} CANCELLED\n",+ cancelled=lambda: True)+ self.assertIsInstance(raised, ClientCancelled)+ # The disconnect is what the engine was told about: CANCEL, not STOP.+ self.assertEqual([w for w in writes if w.startswith(b"CANCEL")], [b"CANCEL 1\n"])+ self.assertEqual([w for w in writes if w.startswith(b"STOP")], [])++ def test_any_other_terminal_error_keeps_the_fault_and_carries_the_engines_text(self):+ # The fault happened first and names the framing defect, so it is what the client+ # is told; the engine's own parting word is appended rather than dropped. Reporting+ # the engine's error instead would tell a client with an over-long prompt to+ # shorten it when the real defect was a frame this server could not read.+ raised, _writes = self._faulted_turn(b"ERROR {id} CONTEXT_EXCEEDED 9 8\n")+ self.assertIsInstance(raised, APIError)+ self.assertEqual(raised.status, 500)+ self.assertEqual(raised.code, "engine_logprob_tail_malformed")+ self.assertIn("malformed per-token logprob tail", raised.message)+ self.assertIn("The engine then reported:", raised.message)+ self.assertIn("CONTEXT_EXCEEDED", raised.message)++ def test_a_faulted_turn_is_stopped_once_and_then_drained(self):+ # The turn is doomed but still the engine's: the admission is held to its terminal+ # frame either way, and one STOP keeps it from spending the rest of the budget on a+ # response nobody will receive. Exactly one, and no CANCEL.+ raised, writes = self._faulted_turn(b"DONE {id} STAT 1 2.5 0 1.0 5 0\n")+ self.assertIsInstance(raised, APIError)+ self.assertEqual(raised.code, "engine_logprob_tail_malformed")+ self.assertEqual([w for w in writes if w.startswith(b"STOP")], [b"STOP 1\n"])+ self.assertEqual([w for w in writes if w.startswith(b"CANCEL")], [])++ def test_a_healthy_slow_turn_is_not_stopped(self):+ # The control for the fault guard, and it has to IDLE to be one: a fixture that+ # feeds every frame back to back never reaches the `queue.Empty` branch, which is+ # the only place the guard lives. This one leaves a gap wider than the 0.05 s poll+ # between DATA and DONE -- an ordinary inter-token interval on a large model -- so+ # a build that stops on every idle tick writes a STOP here and also skips the+ # decode for the rest of the turn, returning a 200 with an empty completion.+ def respond(process, frame):+ request_id = frame.split()[1]+ process.stdout.feed(b"DATA " + request_id + b" 2 -0.5 1 3 -0.5\nok\n")++ def terminal():+ time.sleep(0.15) # three idle polls, well under the ceiling+ process.stdout.feed(+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")++ threading.Thread(target=terminal, daemon=True).start()++ engine, process = self._engine(respond)+ chunks = []+ stats = engine.generate("hi", 4, 0.25, 0.9, chunks.append, logprobs=1,+ gbytes_before_ext=True)+ self.assertEqual([w for w in process.writes if w.startswith(b"STOP")], [])+ self.assertEqual([w for w in process.writes if w.startswith(b"CANCEL")], [])+ # ...and the turn still delivered, which is what a stray STOP would cost.+ self.assertEqual(chunks, ["ok"])+ self.assertEqual(stats["completion_tokens"], 1)++ def test_a_failed_fault_stop_write_answers_by_name_and_keeps_the_connection(self):+ # The STOP written for a recorded fault can itself fail at the pipe. An engine this+ # server cannot write to outranks a fault in the stream it cannot read: the request+ # is answered with a named error rather than having its connection dropped, which+ # is what an unhandled write error does. Written against this tree's own STOP write+ # so the pin survives the composition with the checked writer.+ class DeadOnStop:+ """The engine's stdin: accepts the SUBMIT, refuses the STOP."""+ def __init__(self, process):+ self.process = process++ def write(self, data):+ if data.startswith(b"STOP"):+ raise BrokenPipeError(32, "Broken pipe")+ return self.process.real_write(data)++ def flush(self):+ pass++ def respond(process, frame):+ request_id = frame.split()[1]+ process.stdout.feed(b"DATA " + request_id + b" 2 -0.5 1 3\nok\n")++ def terminal():+ time.sleep(0.15)+ process.stdout.feed(+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")++ threading.Thread(target=terminal, daemon=True).start()++ engine, process = self._engine(respond)+ process.real_write = process.write+ process.stdin = DeadOnStop(process)+ with self.assertRaises(Exception) as caught:+ engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=1,+ gbytes_before_ext=True)+ # The turn is over and nothing will ever answer it: the entry must not be left for+ # the dispatcher to deliver to a caller that has already unwound, nor to hold a+ # request id the next turn could be given.+ self.assertNotIn("1", engine.pending)+ self.assertEqual(engine.pending, {})+ # Named, and reaching the caller: an unraised write error would instead unwind+ # through the handler and close the socket with no status line on it.+ raised = caught.exception+ self.assertNotIsInstance(raised, BrokenPipeError)+ named = raised.code if isinstance(raised, APIError) else str(raised)+ self.assertTrue(named, "the failed STOP write must be reported by name")+ self.assertIn("STOP", str(raised).upper() + (raised.message.upper()+ if isinstance(raised, APIError) else ""))++ def test_the_malformed_tail_battery(self):+ # A required field absent or non-numeric, a k outside the engine's interface, or a+ # negative token id. Every one of them is a missing or unreadable field, never a+ # frame that merely carries more than the grammar names.+ for tail in (b"-0.5", b"-0.5 1 3", b"x 1 3 -0.5", b"-0.5 99 3 -0.5",+ b"-0.5 1 -3 -0.5", b"-0.5 x 3 -0.5"):+ with self.subTest(tail=tail):+ def respond(process, frame, tail=tail):+ request_id = frame.split()[1]+ process.stdout.feed(b"DATA " + request_id + b" 2 " + tail + b"\nok\n"+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")++ engine, _process = self._engine(respond)+ with self.assertRaises(APIError) as caught:+ engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=1,+ gbytes_before_ext=True)+ self.assertEqual(caught.exception.code, "engine_logprob_tail_malformed")++ def test_extra_trailing_fields_on_an_opted_in_frame_are_served(self):+ # The parser consumes the fields the grammar defines and ignores what follows. The+ # base tolerated a longer frame for every caller, and this parser now runs for+ # internal consumers of the channel that read only part of the record.+ def respond(process, frame):+ request_id = frame.split()[1]+ process.stdout.feed(+ b"DATA " + request_id + b" 2 -0.5 1 3 -0.5 9 -9.0\nok\n"+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")++ engine, _process = self._engine(respond)+ chunks = []+ stats = engine.generate("hi", 4, 0.25, 0.9, chunks.append, logprobs=1,+ gbytes_before_ext=True)+ self.assertEqual(chunks, ["ok"])+ self.assertEqual(stats["logprobs"]["generated"],+ [(b"ok", {"lp": -0.5, "topk": [(3, -0.5)]})])++ def test_the_closed_set_scoring_path_keeps_the_bases_frame_tolerance(self):+ # The other caller of the numeric channel reads `pos`/`logprob`/`text` and nothing+ # else, and its engine's ECHO frames may carry more than this parser names. Driven+ # through a real Engine and the real dispatcher, in that caller's own call shape,+ # because a FakeEngine cannot see a framing change at all.+ for label, echo_frame in (+ ("exact", b"ECHO {id} 1 0 nan 0\nh\n"),+ ("one extra trailing field", b"ECHO {id} 1 0 nan 0 7\nh\n"),+ ("a populated table plus an extra", b"ECHO {id} 1 0 -0.5 1 3 -0.5 7\nh\n")):+ with self.subTest(frame=label):+ def respond(process, frame, echo_frame=echo_frame):+ request_id = frame.split()[1]+ process.stdout.feed(echo_frame.replace(b"{id}", request_id))+ process.stdout.feed(b"DONE " + request_id + b" STAT 0 2.5 0 1.0 5 0\n")++ engine, _process = self._engine(respond)+ echoes = []+ engine.generate("hi", 0, 0.0, 1.0, lambda _chunk: None,+ logprobs=1, pin=True, on_echo=echoes.append)+ self.assertEqual(len(echoes), 1)+ self.assertEqual(echoes[0]["pos"], 0)+ self.assertEqual(echoes[0]["text"], "h")++ def test_a_well_formed_tail_reaches_the_caller(self):+ def respond(process, frame):+ request_id = frame.split()[1]+ process.stdout.feed(+ b"ACCEPT " + request_id + b" 1\n"+ b"ECHO " + request_id + b" 1 0 nan 0\nh\n"+ b"DATA " + request_id + b" 2 -0.5 2 3 -0.5 4 -1.25\nok\n"+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 5 0\n")++ engine, _process = self._engine(respond)+ echoes = []+ stats = engine.generate("hi", 4, 0.25, 0.9, [].append, logprobs=2,+ gbytes_before_ext=True, on_echo=echoes.append)+ self.assertEqual(stats["logprobs"]["generated"],+ [(b"ok", {"lp": -0.5, "topk": [(3, -0.5), (4, -1.25)]})])+ # The echo record keeps the scorer's three keys and their meanings, and gains the+ # payload and the numeric tail the logprobs surface needs.+ self.assertEqual(echoes[0]["pos"], 0)+ self.assertIsNone(echoes[0]["logprob"])+ self.assertEqual(echoes[0]["text"], "h")+ self.assertEqual(echoes[0]["bytes"], b"h")+ self.assertTrue(math.isnan(echoes[0]["lp"]))+ self.assertEqual(echoes[0]["topk"], [])++if __name__ == "__main__":unittest.main()