Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

test(openai): golden-fixture battery harness and speculative-gap frame audit

monotophic8a8018251ebb
diff --git a/c/tests/check_data_logprob_gaps.py b/c/tests/check_data_logprob_gaps.py
new file mode 100644
index 00000000..9de318e0
--- /dev/null
+++ b/c/tests/check_data_logprob_gaps.py
@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+"""Speculative-decoding gap check (U7a acceptance test 4).
+
+Asserts, over a captured raw engine-stdout transcript, that EVERY generated
+DATA frame of an opted-in request carries the per-token numeric channel
+("DATA <id> <n> <lp> <k> [tid tlp]*k") -- accepted draft tokens included.
+Speculatively-accepted draft tokens bypass the pick_tok call sites in the mux
+loop (packet Fork 5's implementation risk), so a gap would show up as a
+legacy 3-field DATA frame in the middle of an opted-in generation.
+
+Run recipe (orchestrator; needs a real model -- CPU or CUDA build):
+
+ # single KV slot + model drafts live = the speculative serve regime
+ cd c && make glm
+ SERVE=1 SERVE_BATCH=1 KV_SLOTS=1 DRAFT=2 CTX=4096 \
+ ./colibri <snapshot-dir> < submit.txt | tee engine_stdout.raw
+
+ where submit.txt contains one opted-in generation request, e.g. (prompt
+ "Hello" = 5 payload bytes, 64 new tokens, greedy, logprobs top-5):
+
+ SUBMIT 7 0 5 64 0 1 0 logprobs=5
+ Hello
+
+ (payload line follows the header; the trailing newline after the payload is
+ part of the framing). A speculative run needs an MTP-bearing container or
+ the n-gram fallback to actually accept drafts -- confirm acceptance in the
+ stderr log ("[MTP] ..." / spec acceptance lines) so the run genuinely
+ exercised the draft-accept emit path rather than trivially passing.
+
+Then:
+
+ python3 tests/check_data_logprob_gaps.py engine_stdout.raw --id 7 --topk 5
+
+Checks performed:
+ - every DATA frame for --id has exactly 5 + 2k header fields, a finite
+ float lp, and k == --topk (k may be < topk only if vocab < topk);
+ - ECHO frames (if the request also echoed) cover contiguous positions
+ 0..P-1, position 0 carrying "nan 0";
+ - payload framing (n bytes + newline) stays byte-exact throughout, so a
+ single malformed frame cannot hide by desynchronizing the parse.
+Exit 0 = no gaps; non-zero = at least one gap/malformed frame (listed).
+"""
+import argparse
+import math
+import sys
+
+
+def parse_frames(blob):
+ """Parse the serve-mux stdout byte stream into (header_fields, payload)."""
+ frames = []
+ i = 0
+ n = len(blob)
+ while i < n:
+ j = blob.find(b"\n", i)
+ if j < 0:
+ break
+ line = blob[i:j]
+ i = j + 1
+ fields = line.split()
+ if not fields:
+ continue
+ kind = fields[0]
+ if kind in (b"DATA", b"ECHO") and len(fields) >= 3:
+ try:
+ size = int(fields[2])
+ except ValueError:
+ frames.append((fields, None))
+ continue
+ payload = blob[i:i + size]
+ i += size
+ if i < n and blob[i:i + 1] == b"\n":
+ i += 1
+ else:
+ frames.append((fields, b"<BAD TERMINATOR>"))
+ continue
+ frames.append((fields, payload))
+ else:
+ frames.append((fields, None))
+ return frames
+
+
+def check(frames, request_id, topk):
+ rid = str(request_id).encode()
+ problems = []
+ data_frames = 0
+ echo_positions = []
+ for fields, payload in frames:
+ if len(fields) < 2 or fields[1] != rid:
+ continue
+ kind = fields[0]
+ if kind == b"DATA":
+ data_frames += 1
+ if len(fields) == 3:
+ problems.append(f"GAP: legacy 3-field DATA frame #{data_frames}"
+ f" (payload {payload!r}) has no logprob")
+ continue
+ try:
+ lp = float(fields[3])
+ k = int(fields[4])
+ except (ValueError, IndexError):
+ problems.append(f"malformed DATA numeric fields: {fields!r}")
+ continue
+ if not math.isfinite(lp):
+ problems.append(f"non-finite lp in DATA frame: {fields!r}")
+ if k != topk:
+ problems.append(f"DATA top-k {k} != requested {topk}: {fields!r}")
+ if len(fields) != 5 + 2 * k:
+ problems.append(f"DATA field count {len(fields)} != 5+2k: {fields!r}")
+ elif kind == b"ECHO":
+ try:
+ pos = int(fields[3])
+ except (ValueError, IndexError):
+ problems.append(f"malformed ECHO frame: {fields!r}")
+ continue
+ echo_positions.append(pos)
+ if pos == 0 and (fields[4] != b"nan" or fields[5] != b"0"):
+ problems.append(f"ECHO position 0 should carry 'nan 0': {fields!r}")
+ if pos > 0 and len(fields) < 6:
+ problems.append(f"short ECHO frame: {fields!r}")
+ if echo_positions and echo_positions != list(range(len(echo_positions))):
+ problems.append(f"ECHO positions not contiguous from 0: {echo_positions}")
+ return data_frames, len(echo_positions), problems
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("transcript", help="raw engine stdout capture")
+ parser.add_argument("--id", required=True, help="request id to audit")
+ parser.add_argument("--topk", type=int, required=True,
+ help="the SUBMIT logprobs=k value the request used")
+ args = parser.parse_args()
+ blob = open(args.transcript, "rb").read()
+ frames = parse_frames(blob)
+ data_frames, echo_frames, problems = check(frames, args.id, args.topk)
+ print(f"[gapcheck] id={args.id}: {data_frames} DATA frames, "
+ f"{echo_frames} ECHO frames")
+ if data_frames == 0:
+ problems.append("no DATA frames found for this id -- wrong id, or the "
+ "run produced nothing (check ERROR frames)")
+ for problem in problems:
+ print(f"[gapcheck] {problem}")
+ verdict = ("FAIL" if problems
+ else "PASS: every generated token carries a logprob value")
+ print(f"[gapcheck] {verdict}")
+ return 1 if problems else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/c/tests/golden_fixture_capture.py b/c/tests/golden_fixture_capture.py
new file mode 100644
index 00000000..9ae6424e
--- /dev/null
+++ b/c/tests/golden_fixture_capture.py
@@ -0,0 +1,200 @@
+#!/usr/bin/env python3
+"""Golden-fixture invariant harness (U7a acceptance test 1 / packet test-plan I1).
+
+Captures full HTTP responses for a battery of NON-logprobs requests against a
+RUNNING colibri server, normalizes the volatile fields (ids, timestamps), and
+byte-diffs two capture directories. The engine-channel change (U7a) is opt-in
+and no server request path opts in, so a pre-U7a capture and a post-U7a capture
+of the same battery on the same box/backend/model must be byte-identical --
+this script is the mechanism proving that, not reviewer inspection.
+
+Usage:
+ # 1. start the server on the CURRENT (pre-change) build, then:
+ python3 tests/golden_fixture_capture.py capture \
+ --base-url http://127.0.0.1:8000 --model <model-id> --out fixtures_pre
+
+ # 2. rebuild/restart on the post-change build (same box, same backend,
+ # same container, same CTX/KV_SLOTS env), then:
+ python3 tests/golden_fixture_capture.py capture \
+ --base-url http://127.0.0.1:8000 --model <model-id> --out fixtures_post
+
+ # 3. compare (exit 0 = byte-identical modulo normalized fields):
+ python3 tests/golden_fixture_capture.py diff fixtures_pre fixtures_post
+
+Battery: chat, chat+tools, chat streaming, completions without logprobs, and
+the error cases whose behavior must not move (seed 400, array-prompt 400,
+logprobs 400, out-of-range temperature 400) plus /v1/models.
+
+Normalization: every "id"/"created" field (recursively, and per SSE event) is
+replaced with a constant; nothing else is touched. Generation-bearing requests
+run at temperature 0. If a token-level diff still appears pre-vs-post on the
+SAME backend, triage it against the open engine-determinism finding
+(CUDA_VS_CPU_ARM_B_2026-08-17) before treating it as a U7a regression.
+"""
+import argparse
+import json
+import pathlib
+import sys
+import urllib.error
+import urllib.request
+
+NORMALIZED_KEYS = ("id", "created")
+
+
+def battery(model):
+ chat_messages = [
+ {"role": "system", "content": "You are a terse assistant."},
+ {"role": "user", "content": "Say the word 'fixture' and stop."},
+ ]
+ tools = [{
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "description": "Get the weather for a city.",
+ "parameters": {
+ "type": "object",
+ "properties": {"city": {"type": "string"}},
+ "required": ["city"],
+ },
+ },
+ }]
+ return [
+ ("chat_basic", "POST", "/v1/chat/completions",
+ {"model": model, "messages": chat_messages,
+ "max_tokens": 16, "temperature": 0}),
+ ("chat_tools", "POST", "/v1/chat/completions",
+ {"model": model, "messages": [
+ {"role": "user", "content": "What is the weather in Rome?"}],
+ "tools": tools, "tool_choice": "auto",
+ "max_tokens": 48, "temperature": 0}),
+ ("chat_stream", "POST", "/v1/chat/completions",
+ {"model": model, "messages": chat_messages,
+ "max_tokens": 16, "temperature": 0, "stream": True}),
+ ("completions_plain", "POST", "/v1/completions",
+ {"model": model, "prompt": "The capital of France is",
+ "max_tokens": 8, "temperature": 0}),
+ ("completions_stop", "POST", "/v1/completions",
+ {"model": model, "prompt": "Count: one, two,",
+ "max_tokens": 16, "temperature": 0, "stop": ["five"]}),
+ ("err_seed", "POST", "/v1/completions",
+ {"model": model, "prompt": "hello", "max_tokens": 1, "seed": 1234}),
+ ("err_array_prompt", "POST", "/v1/completions",
+ {"model": model, "prompt": [1, 2, 3], "max_tokens": 1}),
+ ("err_logprobs", "POST", "/v1/completions",
+ {"model": model, "prompt": "hello", "max_tokens": 1, "logprobs": 1}),
+ ("err_bad_temperature", "POST", "/v1/chat/completions",
+ {"model": model, "messages": chat_messages,
+ "max_tokens": 1, "temperature": 9}),
+ ("models", "GET", "/v1/models", None),
+ ]
+
+
+def normalize(obj):
+ if isinstance(obj, dict):
+ return {k: ("<normalized>" if k in NORMALIZED_KEYS else normalize(v))
+ for k, v in obj.items()}
+ if isinstance(obj, list):
+ return [normalize(v) for v in obj]
+ return obj
+
+
+def normalize_body(raw, content_type):
+ text = raw.decode("utf-8", "replace")
+ if "text/event-stream" in (content_type or ""):
+ events = []
+ for line in text.splitlines():
+ if not line.strip():
+ continue
+ if line.startswith("data:"):
+ payload = line[5:].strip()
+ if payload == "[DONE]":
+ events.append("[DONE]")
+ else:
+ try:
+ events.append(normalize(json.loads(payload)))
+ except ValueError:
+ events.append({"unparsed": payload})
+ else:
+ events.append({"raw_line": line})
+ return {"sse_events": events}
+ try:
+ return normalize(json.loads(text))
+ except ValueError:
+ return {"raw_text": text}
+
+
+def run_case(base_url, name, method, path, body):
+ data = None if body is None else json.dumps(body).encode()
+ request = urllib.request.Request(
+ base_url.rstrip("/") + path, data=data, method=method,
+ headers={"Content-Type": "application/json"})
+ try:
+ with urllib.request.urlopen(request, timeout=600) as response:
+ status = response.status
+ content_type = response.headers.get("Content-Type", "")
+ raw = response.read()
+ except urllib.error.HTTPError as error:
+ status = error.code
+ content_type = error.headers.get("Content-Type", "")
+ raw = error.read()
+ return {"case": name, "request": {"method": method, "path": path,
+ "body": body},
+ "status": status,
+ "body": normalize_body(raw, content_type)}
+
+
+def capture(args):
+ out = pathlib.Path(args.out)
+ out.mkdir(parents=True, exist_ok=True)
+ for name, method, path, body in battery(args.model):
+ record = run_case(args.base_url, name, method, path, body)
+ target = out / (name + ".json")
+ target.write_text(json.dumps(record, indent=2, sort_keys=True,
+ ensure_ascii=False) + "\n")
+ print(f"[capture] {name}: HTTP {record['status']} -> {target}")
+ print(f"[capture] done: {out}")
+ return 0
+
+
+def diff(args):
+ a, b = pathlib.Path(args.dir_a), pathlib.Path(args.dir_b)
+ names = sorted({p.name for p in a.glob("*.json")} |
+ {p.name for p in b.glob("*.json")})
+ failures = 0
+ for name in names:
+ fa, fb = a / name, b / name
+ if not fa.exists() or not fb.exists():
+ print(f"[diff] {name}: MISSING on one side")
+ failures += 1
+ continue
+ if fa.read_bytes() != fb.read_bytes():
+ print(f"[diff] {name}: DIFFERS")
+ failures += 1
+ else:
+ print(f"[diff] {name}: identical")
+ if failures:
+ print(f"[diff] FAIL: {failures}/{len(names)} cases differ")
+ return 1
+ print(f"[diff] PASS: {len(names)} cases byte-identical "
+ "(modulo normalized ids/timestamps)")
+ return 0
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ sub = parser.add_subparsers(dest="command", required=True)
+ p_capture = sub.add_parser("capture")
+ p_capture.add_argument("--base-url", default="http://127.0.0.1:8000")
+ p_capture.add_argument("--model", required=True)
+ p_capture.add_argument("--out", required=True)
+ p_capture.set_defaults(func=capture)
+ p_diff = sub.add_parser("diff")
+ p_diff.add_argument("dir_a")
+ p_diff.add_argument("dir_b")
+ p_diff.set_defaults(func=diff)
+ args = parser.parse_args()
+ return args.func(args)
+
+
+if __name__ == "__main__":
+ sys.exit(main())