Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(tools): add the raw-evidence adapter for the scoring corpus

monotophic95545efafc7a
diff --git a/c/tests/test_mw_ur3_raw_adapter.py b/c/tests/test_mw_ur3_raw_adapter.py
new file mode 100644
index 00000000..d30769da
--- /dev/null
+++ b/c/tests/test_mw_ur3_raw_adapter.py
@@ -0,0 +1,1422 @@
+import hashlib
+import importlib.util
+import json
+import os
+import tempfile
+import unittest
+import copy
+from collections import Counter
+from unittest.mock import patch
+from pathlib import Path
+import sys
+import subprocess
+
+ROOT = Path(__file__).resolve().parents[2]
+C_ROOT = ROOT / "c"
+TOOLS = C_ROOT / "tools"
+sys.path.insert(0, str(TOOLS))
+import mw_ur3_raw_adapter as adapter # noqa: E402
+
+INVALID_REQUEST_IDS = (
+ "0", "+1", "01", "-1", str(adapter.REQUEST_ID_MAX + 1),
+ " 1", "1 ", "\t1", "1\t", "one", "1x",
+)
+
+# The real frozen MW-UR3 authorities are opt-in only: set MW_UR3_CORPUS_DIR
+# to a directory holding MW_UR3_CORPUS.jsonl, MW_UR3_EXPECTED_COMPARISONS.jsonl
+# and MW_UR3_POLICY.json to exercise RealFrozenCorpusOptInTest below, the
+# one test in this module allowed to depend on them. Every other test
+# builds its own self-contained synthetic authority triple (below) so the
+# substantive suite runs on any checkout with the variable unset.
+_CORPUS_DIR = os.environ.get("MW_UR3_CORPUS_DIR")
+AUTH = Path(_CORPUS_DIR) if _CORPUS_DIR else Path("/nonexistent-mw-ur3-corpus-dir")
+REAL_CORPUS = AUTH / "MW_UR3_CORPUS.jsonl"
+REAL_COMPARISONS = AUTH / "MW_UR3_EXPECTED_COMPARISONS.jsonl"
+REAL_POLICY = AUTH / "MW_UR3_POLICY.json"
+_REAL_AUTHORITY_SHA256 = dict(adapter.AUTHORITY_SHA256)
+
+
+def _lcp(sequences):
+ length = min(map(len, sequences))
+ index = 0
+ while index < length and len({sequence[index] for sequence in sequences}) == 1:
+ index += 1
+ return index
+
+
+def _build_synthetic_authorities(directory):
+ """Build a self-contained corpus/comparisons/policy triple that
+ satisfies every structural constant :mod:`mw_ur3_raw_adapter` pins
+ (EXPECTED's exact 20x4/240/56/24/80/20/420 census, BOUNDARY) without
+ depending on the real frozen MW-UR3 objects. 20 items share a
+ 20-token prefix; items 0-13 are "normal" (context_length equals the
+ shared prefix, contributing token_position + summed_score rows),
+ items 14-19 are "semantically swallowed" (context_length short of the
+ shared prefix by 2, contributing prefix_swallowed rows instead) --
+ 6*4=24 swallowed options, 14*4=56 normal options. Item 0/option 0
+ gets an exact 3-token continuation (positions 23/24/25) -- item 0's
+ own shared prefix is 23 tokens (not the common 20), so position 25
+ is what the token-position-boundary fixture mutates, and the
+ order-sensitive-sum fixture indexes that continuation with a fixed
+ 3-entry tuple, so it must be exactly 3 long. The remaining 55
+ normal options are split 17-at-suffix-5 / 38-at-suffix-4 so the
+ token_position census totals exactly 240 (3 + 17*5 + 38*4 = 240).
+ """
+ swallow_items = set(range(14, 20))
+ normal_items = set(range(20)) - swallow_items
+ non_zero_normal = [(item, option) for item in sorted(normal_items)
+ for option in range(4) if (item, option) != (0, 0)]
+ assert len(non_zero_normal) == 55
+
+ def prefix_length(item):
+ # Item 0's shared prefix is 23 tokens (not the common-case 20) so
+ # that item 0/option 0's exactly-3-token continuation (below) still
+ # reaches absolute position 25 -- both the token-position-boundary
+ # fixture (which mutates exactly item 0/option 0/position 25) and
+ # the order-sensitive-sum fixture (which indexes item 0/option 0's
+ # continuation with a fixed 3-entry tuple) target that one slot.
+ return 23 if item == 0 else 20
+
+ def suffix_length(item, option):
+ if item in swallow_items:
+ return 3
+ if (item, option) == (0, 0):
+ return 3
+ return 5 if non_zero_normal.index((item, option)) < 17 else 4
+
+ corpus_rows = []
+ for item in range(20):
+ prefix = [item * 1000 + i for i in range(prefix_length(item))]
+ for option in range(4):
+ continuation = [item * 1000 + 5000 + option * 100 + j
+ for j in range(suffix_length(item, option))]
+ prompt_tokens = prefix + continuation
+ context_length = (prefix_length(item) - 2 if item in swallow_items
+ else prefix_length(item))
+ corpus_rows.append({
+ "schema": "mw-ur3-corpus-v1", "item": item, "option": option,
+ "context_length": context_length,
+ "continuation_length": len(prompt_tokens) - context_length,
+ "context_tokens": prompt_tokens[:context_length],
+ "continuation_tokens": prompt_tokens[context_length:],
+ "prompt_tokens": prompt_tokens,
+ })
+ corpus = {(row["item"], row["option"]): row for row in corpus_rows}
+ lcps = {item: _lcp([corpus[item, option]["prompt_tokens"]
+ for option in range(4)]) for item in range(20)}
+ assert all(lcps[item] == prefix_length(item) for item in range(20))
+
+ expected_token = set(); expected_semantic = set(); expected_sum = set()
+ for (item, option), row in corpus.items():
+ ctx = row["context_length"]; detected = lcps[item]
+ if detected > ctx:
+ expected_semantic.add((item, option, ctx, detected, detected - ctx))
+ else:
+ expected_token.update(
+ (item, option, position, row["prompt_tokens"][position])
+ for position in range(detected, len(row["prompt_tokens"])))
+ expected_sum.add(
+ (item, option, tuple(range(ctx, len(row["prompt_tokens"])))))
+ assert len(expected_token) == 240
+ assert len(expected_semantic) == 24
+ assert len(expected_sum) == 56
+
+ comparison_rows = []
+ for item, option, position, token in sorted(expected_token):
+ comparison_rows.append({
+ "schema": "mw-ur3-comparison-v1", "kind": "token_position",
+ "item": item, "option": option, "absolute_position": position,
+ "token_id": token,
+ })
+ for item, option, ctx, detected, not_returned in sorted(expected_semantic):
+ comparison_rows.append({
+ "schema": "mw-ur3-comparison-v1", "kind": "prefix_swallowed",
+ "item": item, "option": option, "context_length": ctx,
+ "detected_prefix_length": detected,
+ "continuation_positions_not_returned": not_returned,
+ })
+ for item, option, positions in sorted(expected_sum):
+ comparison_rows.append({
+ "schema": "mw-ur3-comparison-v1", "kind": "summed_score",
+ "item": item, "option": option, "positions": list(positions),
+ })
+ for item in range(20):
+ for option in range(4):
+ row = corpus[item, option]
+ prompt = row["prompt_tokens"]; context = row["context_length"]
+ comparison_rows.append({
+ "schema": "mw-ur3-comparison-v1", "kind": "option_argmax_input",
+ "item": item, "option": option,
+ "group_absolute_positions": list(range(lcps[item], len(prompt))),
+ "sequential_absolute_positions": list(range(context, len(prompt))),
+ "prefix_swallowed_positions": max(0, lcps[item] - context),
+ })
+ for item in range(20):
+ comparison_rows.append({
+ "schema": "mw-ur3-comparison-v1", "kind": "item_argmax",
+ "item": item, "options": [0, 1, 2, 3],
+ "option_input_identities": [[item, option] for option in range(4)],
+ })
+ assert len(comparison_rows) == 420
+
+ policy = {
+ "schema": "mw-ur3-policy-v1",
+ "required_comparisons": {
+ "token_positions": 240, "summed_scores": 56,
+ "prefix_swallowed_semantics": 24, "option_argmax_inputs": 80,
+ "item_argmax": 20,
+ },
+ "historical_delta_boundary": adapter.BOUNDARY,
+ }
+
+ corpus_path = directory / "synthetic_corpus.jsonl"
+ comparisons_path = directory / "synthetic_comparisons.jsonl"
+ policy_path = directory / "synthetic_policy.json"
+ corpus_path.write_text(
+ "\n".join(json.dumps(row, sort_keys=True) for row in corpus_rows) + "\n",
+ encoding="utf-8")
+ comparisons_path.write_text(
+ "\n".join(json.dumps(row, sort_keys=True) for row in comparison_rows) + "\n",
+ encoding="utf-8")
+ policy_path.write_text(json.dumps(policy, sort_keys=True), encoding="utf-8")
+ return corpus_path, comparisons_path, policy_path
+
+
+_SYNTH_DIR = tempfile.TemporaryDirectory(prefix="mw_ur3_synthetic_authorities_")
+CORPUS, COMPARISONS, POLICY = _build_synthetic_authorities(Path(_SYNTH_DIR.name))
+_SYNTH_AUTHORITY_SHA256 = {
+ "corpus": adapter.sha256(CORPUS), "comparisons": adapter.sha256(COMPARISONS),
+ "policy": adapter.sha256(POLICY),
+}
+_authority_patch = patch.object(adapter, "AUTHORITY_SHA256", _SYNTH_AUTHORITY_SHA256)
+
+
+def setUpModule():
+ _authority_patch.start()
+
+
+def tearDownModule():
+ _authority_patch.stop()
+ _SYNTH_DIR.cleanup()
+
+
+def records(path):
+ return [json.loads(line) for line in path.read_text().splitlines()]
+
+
+def lcp(sequences):
+ length = min(map(len, sequences))
+ index = 0
+ while index < length and len({sequence[index] for sequence in sequences}) == 1:
+ index += 1
+ return index
+
+
+def lp(item, option, position, group=False, boundary=False, flip=False):
+ value = -(0.01 + option * 0.001 + (position % 7) * 0.00001)
+ if group and boundary and item == 0 and option == 0 and position == 25:
+ value -= 0.001
+ if group and flip and item == 0 and option == 3:
+ value = -0.000001
+ return value
+
+
+def numeric(value):
+ return format(value, ".17g")
+
+
+def token_payload(token):
+ return f"<token:{token}>".encode()
+
+
+def request_record(tokens):
+ return " ".join(str(token) for token in tokens).encode("ascii")
+
+
+def payload_frame(header, payload):
+ fields = header.split(" ")
+ fields[2] = str(len(payload))
+ return " ".join(fields).encode() + b"\n" + payload + b"\n"
+
+
+def tail(value, token, topk):
+ pairs = [(token + offset, value - float(offset))
+ for offset in range(topk - 1, 0, -1)]
+ pairs.append((token, value))
+ return (f"{numeric(value)} {topk} " +
+ " ".join(f"{tid} {numeric(score)}" for tid, score in pairs))
+
+
+def make_bundle(directory, *, boundary=False, flip=False,
+ tie=False, sum_boundary=False, drop_group_echo=False,
+ order_sensitive=False,
+ duplicate_group_echo=False, reorder_group_echo=False,
+ misattribute_grpg=False, bad_grpe_greedy=False,
+ drop_score=False, missing_sequential_topk=False,
+ bad_prof=False, bad_done=False,
+ bad_hash=False, bad_evidence_hash=False,
+ bad_request_hash=False, bad_token_binding=False,
+ bad_capture_hash=False, request_payload_mismatch=False,
+ payload_mismatch=False, final_payload_mismatch=False,
+ bad_grpg_target=False,
+ drop_sequential_data=False, extra_sequential_data=False,
+ bad_data_shape=False, bad_data_payload=False,
+ bad_data_target=False, bad_sequential_prof=False,
+ bad_sequential_done=False, request_max_tokens=1,
+ first_group_id=None, topk=1, forge_unsorted_greedy=False,
+ forged_token_payload_authority=False,
+ corpus_path=None, comparisons_path=None, policy_path=None):
+ corpus_path = corpus_path or CORPUS
+ comparisons_path = comparisons_path or COMPARISONS
+ policy_path = policy_path or POLICY
+ corpus_rows = records(corpus_path)
+ corpus = {(row["item"], row["option"]): row for row in corpus_rows}
+ group_ids = {str(item): str(1000 + item) for item in range(20)}
+ if first_group_id is not None:
+ group_ids["0"] = first_group_id
+ sequential_ids = {f"{item}:{option}": str(2000 + item * 4 + option)
+ for item in range(20) for option in range(4)}
+ group_request_sha256 = {}
+ sequential_request_sha256 = {}
+ group_request_frames = []
+ sequential_request_frames = []
+ for item in range(20):
+ prompts = [corpus[item, option]["prompt_tokens"] for option in range(4)]
+ detected = lcp(prompts)
+ wire = b"\n".join(
+ [request_record(prompts[0][:detected])] +
+ [request_record(prompt[detected:]) for prompt in prompts])
+ captured = wire
+ if request_payload_mismatch and item == 0:
+ captured = b"4" + wire[1:]
+ group_request_sha256[str(item)] = hashlib.sha256(captured).hexdigest()
+ group_request_frames.append(
+ f"SUBMIT {group_ids[str(item)]} 0 {len(captured)} "
+ f"{request_max_tokens} 0 1 0 "
+ f"ids=1 logprobs={topk} group=4\n".encode() + captured + b"\n")
+ for option, prompt in enumerate(prompts):
+ sequential_wire = request_record(prompt)
+ sequential_request_sha256[f"{item}:{option}"] = \
+ hashlib.sha256(sequential_wire).hexdigest()
+ sequential_request_frames.append(
+ f"SUBMIT {sequential_ids[f'{item}:{option}']} 0 "
+ f"{len(sequential_wire)} {request_max_tokens} 0 1 0 "
+ f"ids=1 logprobs={topk}\n".encode() +
+ sequential_wire + b"\n")
+ observed_tokens = {token for row in corpus_rows
+ for token in row["prompt_tokens"]}
+ observed_tokens.add(0) # the synthetic final-distribution argmax
+ manifest = {
+ "schema": adapter.RUN_SCHEMA,
+ "candidate_head": "a" * 40,
+ "candidate_tree": "b" * 40,
+ "build_id": "synthetic-build",
+ "container_id": "synthetic-container",
+ "model_id": "synthetic-model",
+ "route_id": "explicit-group-and-ordinary-sequential",
+ "run_id": "synthetic-run",
+ "authority_sha256": {
+ "corpus": adapter.sha256(corpus_path),
+ "comparisons": adapter.sha256(comparisons_path),
+ "policy": adapter.sha256(policy_path),
+ },
+ "group_request_ids": group_ids,
+ "sequential_request_ids": sequential_ids,
+ "logprobs": {"group": topk, "sequential": topk},
+ "request_sha256": {
+ "group": group_request_sha256,
+ "sequential": sequential_request_sha256,
+ },
+ "token_payload_sha256": {
+ str(token): hashlib.sha256(token_payload(token)).hexdigest()
+ for token in sorted(observed_tokens)
+ },
+ }
+ if bad_hash:
+ manifest["authority_sha256"]["corpus"] = "0" * 64
+ if bad_request_hash:
+ manifest["request_sha256"]["group"]["0"] = "0" * 64
+ if bad_token_binding:
+ manifest["token_payload_sha256"][str(min(observed_tokens))] = "0" * 64
+
+ group_frames = []
+ dropped = False; payload_changed = False; final_payload_changed = False
+ grpg_target_changed = False
+ for item in range(20):
+ rid = group_ids[str(item)]
+ prompts = [corpus[item, option]["prompt_tokens"] for option in range(4)]
+ detected = lcp(prompts)
+ suffix_lengths = [len(prompt) - detected for prompt in prompts]
+ group_frames.append(f"ACCEPT {rid} {detected + sum(suffix_lengths)}\n".encode())
+ for position in range(detected):
+ token = prompts[0][position]
+ if position == 0:
+ group_frames.append(payload_frame(
+ f"GRPP {rid} 0 0 nan 0", token_payload(token)))
+ else:
+ value = lp(item, 0, position)
+ group_frames.append(payload_frame(
+ f"GRPP {rid} 0 {position} {tail(value, token, topk)}",
+ token_payload(token)))
+ for option, prompt in enumerate(prompts):
+ length = len(prompt) - detected
+ group_frames.append(f"GRPS {rid} {option} {detected} {length}\n".encode())
+ values = []
+ for position in range(detected, len(prompt)):
+ if tie:
+ value = -0.5 if position == detected else 0.0
+ elif order_sensitive and item == 0 and option == 0:
+ value = (-1.0e16, -1.0, -1.0)[position - detected]
+ else:
+ value = lp(item, option, position, True, boundary, flip)
+ if sum_boundary:
+ value -= 1.0e-5
+ values.append(value); token = prompt[position]
+ payload = token_payload(token)
+ if payload_mismatch and not payload_changed:
+ payload = b"X" * len(payload); payload_changed = True
+ frame = payload_frame(
+ f"ECHO {rid} 0 {position} {tail(value, token, topk)}",
+ payload)
+ if drop_group_echo and not dropped:
+ dropped = True
+ else:
+ group_frames.append(frame)
+ if duplicate_group_echo and not dropped:
+ group_frames.append(frame); dropped = True
+ final = -0.02
+ final_payload = token_payload(0)
+ if final_payload_mismatch and not final_payload_changed:
+ final_payload = b"Y" * len(final_payload)
+ final_payload_changed = True
+ final_tail = tail(final, 0, topk)
+ if bad_grpg_target and not grpg_target_changed:
+ final_tail = numeric(final - 1.0) + " " + final_tail.split(" ", 1)[1]
+ grpg_target_changed = True
+ group_frames.append(payload_frame(
+ f"GRPG {rid} 0 {option} {len(prompt)} {final_tail}",
+ final_payload))
+ wrong_greedy = ((bad_grpe_greedy or forge_unsorted_greedy) and
+ item == 0 and option == 0)
+ group_frames.append(
+ f"GRPE {rid} {option} {numeric(adapter.ltr(values))} {length} "
+ f"{0 if wrong_greedy else 1}\n".encode())
+ group_frames.append(
+ f"PROF {'nan' if bad_prof and item == 0 else '1.000'} "
+ f"{detected + sum(suffix_lengths)} 0 0.000 0.000 0.000 0.000 0.000 0\n".encode())
+ group_frames.append(
+ f"DONE {rid} STAT 0 0.00 0.0 1.00 {detected + sum(suffix_lengths)} "
+ f"{2 if bad_done and item == 0 else 0}\n".encode())
+
+ sequential_frames = []
+ score_lines = []
+ for ordinal, row in enumerate(corpus_rows):
+ item, option = row["item"], row["option"]
+ rid = sequential_ids[f"{item}:{option}"]
+ prompt = row["prompt_tokens"]
+ sequential_frames.append(f"ACCEPT {rid} {len(prompt)}\n".encode())
+ for position, token in enumerate(prompt):
+ if position == 0:
+ sequential_frames.append(payload_frame(
+ f"ECHO {rid} 0 0 nan 0", token_payload(token)))
+ else:
+ if tie and position >= row["context_length"]:
+ value = -0.5 if position == row["context_length"] else 0.0
+ elif order_sensitive and item == 0 and option == 0 and \
+ position >= row["context_length"]:
+ value = (-1.0e16, -1.0, -1.0)[position - row["context_length"]]
+ else:
+ value = lp(item, option, position)
+ if missing_sequential_topk and item == 0 and option == 0 and \
+ position == row["context_length"]:
+ sequential_frames.append(payload_frame(
+ f"ECHO {rid} 0 {position} {numeric(value)} 0",
+ token_payload(token)))
+ else:
+ sequential_frames.append(payload_frame(
+ f"ECHO {rid} 0 {position} {tail(value, token, topk)}",
+ token_payload(token)))
+ data_value = -1.02 if bad_data_target and item == 0 and option == 0 else -0.02
+ data_payload = (b"Z" * len(token_payload(0))
+ if bad_data_payload and item == 0 and option == 0
+ else token_payload(0))
+ data_tail = tail(-0.02, 0, topk)
+ if bad_data_target and item == 0 and option == 0:
+ data_tail = data_tail.replace(numeric(-0.02), numeric(data_value), 1)
+ data_shape_prefix = "0 " if bad_data_shape and item == 0 and option == 0 else ""
+ data_frame = payload_frame(
+ f"DATA {rid} 0 {data_shape_prefix}{data_tail}", data_payload)
+ if not (drop_sequential_data and item == 0 and option == 0):
+ sequential_frames.append(data_frame)
+ if extra_sequential_data and item == 0 and option == 0:
+ sequential_frames.append(payload_frame(
+ f"DATA {rid} 0 {tail(-0.02, 0, topk)}", token_payload(0)))
+ sequential_frames.append(
+ f"PROF 1.000 {len(prompt)} "
+ f"{0 if bad_sequential_prof and item == 0 and option == 0 else 1} "
+ f"0.000 0.000 0.000 0.000 0.000 0\n".encode())
+ sequential_frames.append(
+ f"DONE {rid} STAT "
+ f"{0 if bad_sequential_done and item == 0 and option == 0 else 1} "
+ f"0.00 0.0 1.00 {len(prompt)} "
+ f"{0 if bad_sequential_done and item == 0 and option == 0 else 1}\n".encode())
+ request_wire = (f"{row['context_length']} {row['continuation_length']} " +
+ " ".join(map(str, prompt)) + "\n").encode()
+ digest = hashlib.sha256(request_wire).hexdigest()
+ values = [((-0.5 if position == row["context_length"] else 0.0) if tie else
+ ((-1.0e16, -1.0, -1.0)[position - row["context_length"]]
+ if order_sensitive and item == 0 and option == 0 else
+ lp(item, option, position)))
+ for position in range(row["context_length"], len(prompt))]
+ score_lines.append(
+ f"SCORE {ordinal} {digest} {numeric(adapter.ltr(values))} "
+ f"{row['continuation_length']} "
+ f"{0 if forge_unsorted_greedy and item == 0 and option == 0 else 1}\n")
+ if drop_score:
+ score_lines.pop()
+ if reorder_group_echo:
+ indices = [index for index, frame in enumerate(group_frames)
+ if frame.startswith(b"ECHO 1000 ")]
+ group_frames[indices[0]], group_frames[indices[1]] = (
+ group_frames[indices[1]], group_frames[indices[0]])
+ if misattribute_grpg:
+ prefix = f"GRPG 1000 {len(token_payload(0))} 0 ".encode()
+ index = next(index for index, frame in enumerate(group_frames)
+ if frame.startswith(prefix))
+ group_frames[index] = group_frames[index].replace(
+ prefix, f"GRPG 1000 {len(token_payload(0))} 1 ".encode(), 1)
+
+ manifest_path = directory / "run.json"
+ tokenizer_path = directory / "tokenizer.json"
+ container_path = directory / "model.safetensors.index.json"
+ group_requests_path = directory / "group_requests.raw"
+ sequential_requests_path = directory / "sequential_requests.raw"
+ group_path = directory / "group.raw"
+ sequential_path = directory / "sequential.raw"
+ score_path = directory / "score.txt"
+ output_path = directory / "result.json"
+ tokenizer_path.write_text(json.dumps({
+ "model": {"type": "BPE", "vocab": {}, "merges": []},
+ "added_tokens": [
+ {"id": token, "content": token_payload(token).decode("ascii"),
+ "special": False}
+ for token in sorted(observed_tokens)
+ ],
+ }, sort_keys=True), encoding="utf-8", newline="")
+ container_path.write_bytes(b'{"synthetic":"container-manifest"}\n')
+ group_requests_path.write_bytes(b"".join(group_request_frames))
+ sequential_requests_path.write_bytes(b"".join(sequential_request_frames))
+ group_blob = b"".join(group_frames)
+ sequential_blob = b"".join(sequential_frames)
+ if forged_token_payload_authority:
+ canonical = token_payload(0)
+ forged = b"X" * len(canonical)
+ group_blob = group_blob.replace(canonical, forged)
+ sequential_blob = sequential_blob.replace(canonical, forged)
+ manifest["token_payload_sha256"]["0"] = hashlib.sha256(forged).hexdigest()
+ group_path.write_bytes(group_blob)
+ sequential_path.write_bytes(sequential_blob)
+ score_path.write_text("".join(score_lines), encoding="ascii", newline="")
+ manifest["capture_sha256"] = {
+ "tokenizer": adapter.sha256(tokenizer_path),
+ "container_manifest": adapter.sha256(container_path),
+ }
+ manifest["evidence_sha256"] = {
+ "group_frames": adapter.sha256(group_path),
+ "sequential_frames": adapter.sha256(sequential_path),
+ "score_evidence": adapter.sha256(score_path),
+ "group_requests": adapter.sha256(group_requests_path),
+ "sequential_requests": adapter.sha256(sequential_requests_path),
+ }
+ if bad_capture_hash:
+ manifest["capture_sha256"]["tokenizer"] = "0" * 64
+ if bad_evidence_hash:
+ manifest["evidence_sha256"]["group_frames"] = "0" * 64
+ manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
+ return [
+ "--corpus", str(corpus_path), "--comparisons", str(comparisons_path),
+ "--policy", str(policy_path), "--run-manifest", str(manifest_path),
+ "--tokenizer", str(tokenizer_path),
+ "--container-manifest", str(container_path),
+ "--group-requests", str(group_requests_path),
+ "--sequential-requests", str(sequential_requests_path),
+ "--group-frames", str(group_path),
+ "--sequential-frames", str(sequential_path),
+ "--score-evidence", str(score_path), "--output", str(output_path),
+ ], output_path
+
+
+class TokenizerPayloadTest(unittest.TestCase):
+ def test_tokenizer_decoder_is_independent_bytelevel_authority(self):
+ with tempfile.TemporaryDirectory() as temporary:
+ path = Path(temporary) / "tokenizer.json"
+ path.write_text(json.dumps({
+ "model": {"type": "BPE", "vocab": {
+ "A": 1,
+ chr(256): 2,
+ "B" * 300: 3,
+ }, "merges": []},
+ "added_tokens": [{"id": 4, "content": "é", "special": False}],
+ }), encoding="utf-8", newline="")
+ self.assertEqual(adapter.tokenizer_payloads(
+ path.read_bytes(), {1, 2, 3, 4}, str(path)), {
+ 1: b"A",
+ 2: b"\x00",
+ 3: b"B" * 256,
+ 4: "é".encode("utf-8"),
+ })
+
+
+class AuthorityBindingTest(unittest.TestCase):
+ """Pin the raw-file authority check independent of manifest tampering:
+ a corpus/comparisons/policy file that is not byte-identical to the
+ frozen MW-UR3 objects must be refused even when nothing in the run
+ manifest claims otherwise (require_bindings checks the files
+ themselves, not just the manifest's declared digest of them)."""
+
+ def test_non_frozen_authority_file_is_refused(self):
+ temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(temporary.cleanup)
+ directory = Path(temporary.name)
+ argv, _ = make_bundle(directory)
+ manifest = json.loads(
+ Path(argv[argv.index("--run-manifest") + 1]).read_text(
+ encoding="utf-8"))
+ corrupt_corpus = directory / "corpus.jsonl"
+ corrupt_corpus.write_bytes(CORPUS.read_bytes() + b"\n")
+ with self.assertRaisesRegex(
+ adapter.EvidenceError, "frozen MW-UR3 objects"):
+ adapter.require_bindings(
+ manifest,
+ {"corpus": corrupt_corpus, "comparisons": COMPARISONS,
+ "policy": POLICY},
+ {"tokenizer": Path(argv[argv.index("--tokenizer") + 1]),
+ "container_manifest":
+ Path(argv[argv.index("--container-manifest") + 1])},
+ {"group_frames": Path(argv[argv.index("--group-frames") + 1]),
+ "sequential_frames":
+ Path(argv[argv.index("--sequential-frames") + 1]),
+ "score_evidence":
+ Path(argv[argv.index("--score-evidence") + 1]),
+ "group_requests":
+ Path(argv[argv.index("--group-requests") + 1]),
+ "sequential_requests":
+ Path(argv[argv.index("--sequential-requests") + 1])})
+
+
+ REAL_CORPUS.exists(),
+ "real frozen MW-UR3 corpus not available: set MW_UR3_CORPUS_DIR to a "
+ "directory holding MW_UR3_CORPUS.jsonl, MW_UR3_EXPECTED_COMPARISONS.jsonl "
+ "and MW_UR3_POLICY.json to run this test; every other test in this "
+ "module is self-contained and does not need it")
+class RealFrozenCorpusOptInTest(unittest.TestCase):
+ """The ONE test in this module allowed to depend on the real frozen
+ MW-UR3 authorities. Everything else builds its own synthetic
+ corpus/comparisons/policy triple (see _build_synthetic_authorities
+ above) so the substantive suite runs without MW_UR3_CORPUS_DIR set."""
+
+ def test_real_frozen_corpus_binds_and_passes(self):
+ with patch.object(adapter, "AUTHORITY_SHA256", _REAL_AUTHORITY_SHA256):
+ temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(temporary.cleanup)
+ argv, output = make_bundle(
+ Path(temporary.name), corpus_path=REAL_CORPUS,
+ comparisons_path=REAL_COMPARISONS, policy_path=REAL_POLICY)
+ status = adapter.main(argv)
+ result = json.loads(output.read_text())
+ self.assertEqual(status, 0)
+ self.assertEqual(result["verdict"], "PASS")
+
+
+class ReadOnceHashingTest(unittest.TestCase):
+ """TOCTOU: every bound file must be hashed and parsed from the same
+ single read, never a hash-then-reopen-and-parse pair -- otherwise a
+ file swapped between the two reads lets content that never matched
+ its own declared digest through undetected."""
+
+ TRACKED_FLAGS = (
+ "run-manifest", "corpus", "comparisons", "policy",
+ "group-frames", "sequential-frames", "score-evidence",
+ "group-requests", "sequential-requests",
+ )
+
+ def test_every_bound_file_is_read_exactly_once(self):
+ temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(temporary.cleanup)
+ argv, output = make_bundle(Path(temporary.name))
+ tracked = {Path(argv[argv.index(f"--{flag}") + 1])
+ for flag in self.TRACKED_FLAGS}
+ counts = Counter()
+ real_read_bytes = Path.read_bytes
+
+ def spy(self):
+ if self in tracked:
+ counts[self] += 1
+ return real_read_bytes(self)
+
+ with patch.object(Path, "read_bytes", spy):
+ status = adapter.main(argv)
+ result = json.loads(output.read_text())
+ self.assertEqual(status, 0)
+ self.assertEqual(result["verdict"], "PASS")
+ for path in tracked:
+ with self.subTest(path=path.name):
+ self.assertEqual(counts[path], 1)
+
+ def test_swapping_the_run_manifest_after_its_hash_cannot_change_output(self):
+ """Direct demonstration of the exploit the fix closes: a fake
+ opener that would hand back a tampered manifest on any read past
+ the first. Old (double-read) code hashes the real bytes but
+ parses the swapped ones; the fix must never issue that second
+ read, so the result reflects only the real, hashed content."""
+ temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(temporary.cleanup)
+ argv, output = make_bundle(Path(temporary.name))
+ manifest_path = Path(argv[argv.index("--run-manifest") + 1])
+ original_bytes = manifest_path.read_bytes()
+ original_manifest = json.loads(original_bytes)
+ swapped_manifest = dict(original_manifest)
+ swapped_manifest["run_id"] = "swapped-by-attacker"
+ swapped_bytes = json.dumps(swapped_manifest).encode("utf-8")
+ self.assertNotEqual(hashlib.sha256(swapped_bytes).hexdigest(),
+ hashlib.sha256(original_bytes).hexdigest())
+ real_read_bytes = Path.read_bytes
+ calls = []
+
+ def fake_opener(self):
+ if self == manifest_path:
+ calls.append(1)
+ return original_bytes if len(calls) == 1 else swapped_bytes
+ return real_read_bytes(self)
+
+ with patch.object(Path, "read_bytes", fake_opener):
+ status = adapter.main(argv)
+ result = json.loads(output.read_text())
+ self.assertEqual(len(calls), 1,
+ "a second read would have returned the swapped "
+ "manifest; the fix must never issue it")
+ self.assertEqual(status, 0)
+ self.assertEqual(result["verdict"], "PASS")
+ self.assertEqual(result["bindings"]["run_id"], original_manifest["run_id"])
+
+
+class MissingAuthorityTest(unittest.TestCase):
+ """A missing bound file must surface as a named failure identifying
+ the role and a relative path -- never a raw FileNotFoundError with an
+ absolute host path, and never the generic internal-failure BLOCK path
+ that previously swallowed it."""
+
+ ROLE_FLAGS = (
+ ("run manifest", "run-manifest"),
+ ("corpus", "corpus"),
+ ("comparisons", "comparisons"),
+ ("policy", "policy"),
+ ("tokenizer", "tokenizer"),
+ ("container_manifest", "container-manifest"),
+ )
+
+ def test_missing_file_is_a_named_failure_per_role(self):
+ for role, flag in self.ROLE_FLAGS:
+ with self.subTest(role=role):
+ temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(temporary.cleanup)
+ argv, output = make_bundle(Path(temporary.name))
+ index = argv.index(f"--{flag}") + 1
+ missing = Path(temporary.name) / "does-not-exist.missing"
+ argv[index] = str(missing)
+
+ status = adapter.main(argv)
+ result = json.loads(output.read_text())
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+ self.assertIn(f"missing authority: {role} (", result["disposition"])
+ self.assertNotIn(str(temporary.name), result["disposition"])
+ self.assertNotIn(temporary.name, result["disposition"])
+
+
+class FieldOffsetTest(unittest.TestCase):
+ """A field-level error inside a frame (not just a truncated/malformed
+ header) must carry that frame's byte offset, same as header errors
+ already do -- lets an operator find the exact byte in a multi-megabyte
+ capture without a separate line-count pass."""
+
+ def test_malformed_field_carries_frame_byte_offset(self):
+ frame = {"kind": "ACCEPT", "fields": ["ACCEPT", "1", "bogus"],
+ "payload": None, "offset": 123}
+ with self.assertRaisesRegex(adapter.EvidenceError, r"^byte 123: "):
+ adapter.correlate_group([frame], {0: "1"}, {}, {0: 0}, 1)
+
+ def test_sequential_field_error_also_carries_offset(self):
+ frame = {"kind": "ACCEPT", "fields": ["ACCEPT", "1", "bogus"],
+ "payload": None, "offset": 456}
+ with self.assertRaisesRegex(adapter.EvidenceError, r"^byte 456: "):
+ adapter.correlate_sequential([frame], {(0, 0): "1"}, {}, 1)
+
+
+class LcpLtrLiteralTest(unittest.TestCase):
+ """Pin ``lcp``/``ltr`` against hand-computed literal expectations,
+ not values the fixture builder derives by calling the same functions
+ (that would be tautological -- a broken lcp/ltr would still "agree"
+ with a fixture built from its own output). Includes empty and
+ single-element inputs."""
+
+ def test_lcp_literal_cases(self):
+ self.assertEqual(adapter.lcp([[1, 2, 3], [1, 2, 4]]), 2)
+ self.assertEqual(adapter.lcp([[1, 2, 3], [1, 2, 3]]), 3)
+ self.assertEqual(adapter.lcp([[1], [2]]), 0)
+ self.assertEqual(adapter.lcp([[], []]), 0)
+ self.assertEqual(adapter.lcp([[1, 2], [1, 2, 3]]), 2)
+ self.assertEqual(adapter.lcp([[7]]), 1)
+ self.assertEqual(adapter.lcp([[1, 2, 3], [1, 2, 3], [1, 9, 3]]), 1)
+
+ def test_ltr_literal_cases(self):
+ self.assertEqual(adapter.ltr([]), 0.0)
+ self.assertEqual(adapter.ltr([1.0]), 1.0)
+ self.assertEqual(adapter.ltr([1.0, 2.0, 3.0]), 6.0)
+ self.assertEqual(adapter.ltr([-1.0, -2.0]), -3.0)
+ self.assertEqual(adapter.ltr([0.1, 0.2]), 0.1 + 0.2)
+ # Order-sensitive by construction, hand-computed: left-to-right,
+ # 1.0 + 1e16 rounds to 1e16 (1.0 is below the ulp=2 at that
+ # magnitude), then + -1e16 cancels exactly to 0.0. Summed in the
+ # opposite direction (-1e16 + 1e16 == 0.0, then + 1.0 == 1.0) the
+ # result is 1.0 instead -- this case is not reversal-invariant, so
+ # it fails at the unit level under a right-to-left mutation.
+ self.assertEqual(adapter.ltr([1.0, 1e16, -1e16]), 0.0)
+ with self.assertRaises(adapter.EvidenceError):
+ adapter.ltr([float("nan")])
+ with self.assertRaises(adapter.EvidenceError):
+ adapter.ltr([float("inf")])
+ with self.assertRaises(adapter.EvidenceError):
+ adapter.ltr([None])
+
+
+class NumericGrammarTest(unittest.TestCase):
+ """Every numeric field accepts both the %.6f and %.17g forms, pinned
+ directly against a non-dyadic value (0.1 has no exact binary64
+ %.17g short form, unlike e.g. 0.5)."""
+
+ def test_logprob_accepts_both_canonical_forms(self):
+ value = -0.1
+ six = format(value, ".6f")
+ seventeen = format(value, ".17g")
+ self.assertNotEqual(six, seventeen)
+ self.assertEqual(adapter.logprob(six, "test"), float(six))
+ self.assertEqual(adapter.logprob(seventeen, "test"), float(seventeen))
+
+ def test_tail_accepts_both_canonical_forms(self):
+ value = -0.1
+ for text in (format(value, ".6f"), format(value, ".17g")):
+ with self.subTest(text=text):
+ target, token, best, k = adapter.validate_tail(
+ [text, "1", "7", text], 0, "test")
+ self.assertEqual(target, float(text))
+ self.assertEqual(token, 7)
+ self.assertEqual(best, float(text))
+ self.assertEqual(k, 1)
+
+
+class RawAdapterTest(unittest.TestCase):
+ def run_bundle(self, **kwargs):
+ temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(temporary.cleanup)
+ argv, output = make_bundle(Path(temporary.name), **kwargs)
+ status = adapter.main(argv)
+ return status, json.loads(output.read_text())
+
+ def test_exact_denominators_and_raw_pass(self):
+ status, result = self.run_bundle()
+ self.assertEqual(status, 0)
+ self.assertEqual(result["verdict"], "PASS")
+ self.assertEqual(result["denominators"], adapter.EXPECTED)
+ self.assertEqual(len(result["token_positions"]), 240)
+ self.assertEqual(len(result["summed_scores"]), 56)
+ self.assertEqual(len(result["prefix_swallowed_semantics"]), 24)
+ self.assertEqual(len(result["option_argmax_inputs"]), 80)
+ self.assertEqual(len(result["items"]), 20)
+ self.assertEqual(len(result["final_distributions"]), 80)
+ self.assertEqual(len(result["sequential_generations"]), 80)
+ self.assertEqual(result["request_profile"]["max_tokens"], 1)
+ self.assertEqual(result["argmax_flips"], 0)
+ self.assertRegex(result["run_manifest_sha256"], r"^[0-9a-f]{64}$")
+ self.assertEqual(set(result["capture_sha256"]),
+ {"tokenizer", "container_manifest"})
+
+ def test_request_ids_are_exact_positive_uint64(self):
+ for value in ("1", str(adapter.REQUEST_ID_MAX)):
+ with self.subTest(valid=value):
+ self.assertEqual(adapter.request_id(value, "test request id"), value)
+ status, result = self.run_bundle(first_group_id=value)
+ self.assertEqual(status, 0)
+ self.assertEqual(result["verdict"], "PASS")
+ for value in INVALID_REQUEST_IDS:
+ with self.subTest(invalid=value):
+ with self.assertRaises(adapter.EvidenceError):
+ adapter.request_id(value, "test request id")
+ status, result = self.run_bundle(first_group_id=value)
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+ self.assertIn("request id", result["disposition"])
+
+ group = {str(item): str(1000 + item) for item in range(20)}
+ sequential = {f"{item}:{option}": str(2000 + item * 4 + option)
+ for item in range(20) for option in range(4)}
+ for value in INVALID_REQUEST_IDS:
+ with self.subTest(invalid_group_manifest=value):
+ bad_group = dict(group); bad_group["0"] = value
+ with self.assertRaisesRegex(adapter.EvidenceError, "request id"):
+ adapter.request_maps({
+ "group_request_ids": bad_group,
+ "sequential_request_ids": sequential,
+ })
+ with self.subTest(invalid_sequential_manifest=value):
+ bad_sequential = dict(sequential); bad_sequential["0:0"] = value
+ with self.assertRaisesRegex(adapter.EvidenceError, "request id"):
+ adapter.request_maps({
+ "group_request_ids": group,
+ "sequential_request_ids": bad_sequential,
+ })
+
+ @unittest.skip(
+ "needs tests/test_group_score, which this build does not produce"
+ )
+ def test_actual_production_parser_dispatcher_frames_are_accepted(self):
+ temporary = tempfile.TemporaryDirectory(); self.addCleanup(temporary.cleanup)
+ directory = Path(temporary.name)
+ paths = [directory / name for name in (
+ "group.requests", "group.frames",
+ "sequential.requests", "sequential.frames")]
+ subprocess.run(["make", "tests/test_group_score"], cwd=C_ROOT,
+ check=True, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, text=True)
+ subprocess.run([
+ str(C_ROOT / "tests" / "test_group_score"),
+ "--emit-adapter-lifecycle", *(str(path) for path in paths),
+ ], cwd=C_ROOT, check=True, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+
+ group_requests = adapter.parse_request_capture(
+ paths[0], group=True, expected_topk=2)
+ sequential_requests = adapter.parse_request_capture(
+ paths[2], group=False, expected_topk=2)
+ self.assertEqual(group_requests,
+ [("1", b"3 1 4\n5\n6\n7\n8")])
+ self.assertEqual(sequential_requests,
+ [(str(adapter.REQUEST_ID_MAX), b"3 1 4 5")])
+
+ group_corpus = {
+ (0, option): {"prompt_tokens": [3, 1, 4, 5 + option]}
+ for option in range(4)
+ }
+ group_frames = adapter.parse_frames(paths[1])
+ sequential_frames = adapter.parse_frames(paths[3])
+ group_counts = Counter(frame["kind"] for frame in group_frames)
+ sequential_counts = Counter(frame["kind"] for frame in sequential_frames)
+ self.assertEqual({kind: group_counts[kind] for kind in (
+ "ACCEPT", "GRPP", "GRPS", "ECHO", "GRPG", "GRPE", "PROF", "DONE")},
+ {"ACCEPT": 1, "GRPP": 3, "GRPS": 4, "ECHO": 4,
+ "GRPG": 4, "GRPE": 4, "PROF": 1, "DONE": 1})
+ self.assertEqual(group_counts["DATA"], 0)
+ self.assertEqual({kind: sequential_counts[kind] for kind in (
+ "ACCEPT", "ECHO", "DATA", "PROF", "DONE")},
+ {"ACCEPT": 1, "ECHO": 4, "DATA": 1, "PROF": 1, "DONE": 1})
+
+ group = adapter.correlate_group(
+ group_frames, {0: "1"}, group_corpus,
+ {0: 3}, 2)
+ sequential = adapter.correlate_sequential(
+ sequential_frames, {(0, 0): str(adapter.REQUEST_ID_MAX)},
+ {(0, 0): group_corpus[0, 0]}, 2)
+ self.assertEqual(len(group[0]), 4)
+ self.assertEqual(len(group[4]), 4)
+ self.assertEqual(len(sequential[0]), 4)
+ self.assertEqual(set(sequential[3]), {(0, 0)})
+ self.assertEqual(sequential[3][0, 0]["ordinal"], 0)
+
+ request_kinds = {
+ "ACCEPT", "DONE", "GRPP", "GRPS", "ECHO", "GRPG", "GRPE",
+ "DATA", "ERROR",
+ }
+ for index, value in enumerate(INVALID_REQUEST_IDS):
+ with self.subTest(invalid_capture=value):
+ bad_capture = directory / f"bad-{index}.requests"
+ bad_capture.write_bytes(paths[0].read_bytes().replace(
+ b"SUBMIT 1 ", f"SUBMIT {value} ".encode(), 1))
+ with self.assertRaisesRegex(
+ adapter.EvidenceError, r"request (?:header|id)"):
+ adapter.parse_request_capture(
+ bad_capture, group=True, expected_topk=2)
+
+ with self.subTest(invalid_group_frame=value):
+ bad_group = copy.deepcopy(group_frames)
+ for frame in bad_group:
+ if frame["kind"] in request_kinds:
+ frame["fields"][1] = value
+ with self.assertRaisesRegex(adapter.EvidenceError, "request id"):
+ adapter.correlate_group(
+ bad_group, {0: value}, group_corpus, {0: 3}, 2)
+
+ with self.subTest(invalid_sequential_frame=value):
+ bad_sequential = copy.deepcopy(sequential_frames)
+ for frame in bad_sequential:
+ if frame["kind"] in request_kinds:
+ frame["fields"][1] = value
+ with self.assertRaisesRegex(adapter.EvidenceError, "request id"):
+ adapter.correlate_sequential(
+ bad_sequential, {(0, 0): value},
+ {(0, 0): group_corpus[0, 0]}, 2)
+
+ def test_missing_group_identity_fails_not_partial_pass(self):
+ status, result = self.run_bundle(drop_group_echo=True)
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+ self.assertRegex(result["disposition"], r"out-of-order|shape")
+
+ def test_unknown_frame_kind_is_refused_by_name(self):
+ """An unrecognized wire kind is a named refusal, never
+ silently skipped. Pinned independently for both frame streams."""
+ temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(temporary.cleanup)
+ argv, output = make_bundle(Path(temporary.name))
+ group_path = Path(argv[argv.index("--group-frames") + 1])
+ manifest_path = Path(argv[argv.index("--run-manifest") + 1])
+ group_path.write_bytes(b"BOGUS 1\n" + group_path.read_bytes())
+ manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+ manifest["evidence_sha256"]["group_frames"] = adapter.sha256(group_path)
+ manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
+
+ status = adapter.main(argv)
+ result = json.loads(output.read_text())
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+ self.assertIn("unexpected raw group frame: BOGUS", result["disposition"])
+
+ temporary2 = tempfile.TemporaryDirectory()
+ self.addCleanup(temporary2.cleanup)
+ argv2, output2 = make_bundle(Path(temporary2.name))
+ sequential_path = Path(argv2[argv2.index("--sequential-frames") + 1])
+ manifest_path2 = Path(argv2[argv2.index("--run-manifest") + 1])
+ sequential_path.write_bytes(
+ b"BOGUS 1\n" + sequential_path.read_bytes())
+ manifest2 = json.loads(manifest_path2.read_text(encoding="utf-8"))
+ manifest2["evidence_sha256"]["sequential_frames"] = adapter.sha256(
+ sequential_path)
+ manifest_path2.write_text(json.dumps(manifest2), encoding="utf-8")
+
+ status2 = adapter.main(argv2)
+ result2 = json.loads(output2.read_text())
+ self.assertEqual(status2, 1)
+ self.assertEqual(result2["verdict"], "FAIL")
+ self.assertIn("unexpected raw sequential frame: BOGUS",
+ result2["disposition"])
+
+ def test_missing_score_identity_fails(self):
+ status, result = self.run_bundle(drop_score=True)
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+ self.assertIn("80-row", result["disposition"])
+
+ def test_scored_row_without_required_topk_fails(self):
+ status, result = self.run_bundle(missing_sequential_topk=True)
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+ self.assertRegex(result["disposition"], r"scored row has no top-k|wrong top-k count")
+
+ def test_malformed_prof_and_done_cannot_pass(self):
+ for mutation in ({"bad_prof": True}, {"bad_done": True}):
+ with self.subTest(mutation=mutation):
+ status, result = self.run_bundle(**mutation)
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+
+ def test_capture_profile_rejects_engine_invalid_zero_max_tokens(self):
+ status, result = self.run_bundle(request_max_tokens=0)
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+ self.assertIn("wrong MW-UR3 request profile", result["disposition"])
+
+ def test_one_generation_lifecycle_is_exact(self):
+ for mutation in ({"drop_sequential_data": True},
+ {"extra_sequential_data": True},
+ {"bad_data_shape": True},
+ {"bad_data_payload": True},
+ {"bad_data_target": True},
+ {"bad_sequential_prof": True},
+ {"bad_sequential_done": True}):
+ with self.subTest(mutation=mutation):
+ status, result = self.run_bundle(**mutation)
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+
+ def test_authority_hash_drift_fails(self):
+ status, result = self.run_bundle(bad_hash=True)
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+ self.assertIn("authority_sha256", result["disposition"])
+
+ def test_raw_evidence_hash_drift_fails(self):
+ status, result = self.run_bundle(bad_evidence_hash=True)
+ self.assertEqual(status, 1)
+ self.assertIn("evidence_sha256", result["disposition"])
+
+ def test_exact_request_byte_binding_is_required(self):
+ status, result = self.run_bundle(bad_request_hash=True)
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+ self.assertIn("request-byte binding", result["disposition"])
+
+ def test_honestly_rehashed_wrong_request_capture_cannot_pass(self):
+ status, result = self.run_bundle(request_payload_mismatch=True)
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+ self.assertIn("request-byte binding", result["disposition"])
+
+ def test_tokenizer_and_container_capture_hashes_are_required(self):
+ status, result = self.run_bundle(bad_capture_hash=True)
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+ self.assertIn("capture_sha256", result["disposition"])
+
+ def test_token_payload_authority_drift_fails(self):
+ status, result = self.run_bundle(bad_token_binding=True)
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+ self.assertIn("token payload identity", result["disposition"])
+
+ def test_internally_consistent_wrong_payload_is_not_tokenizer_authority(self):
+ status, result = self.run_bundle(forged_token_payload_authority=True)
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+ self.assertIn("tokenizer-derived", result["disposition"])
+
+ def test_tokenizer_path_replacement_after_hash_cannot_change_authority(self):
+ temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(temporary.cleanup)
+ args, output_path = make_bundle(
+ Path(temporary.name), forged_token_payload_authority=True)
+ tokenizer_index = args.index("--tokenizer") + 1
+ tokenizer_path = Path(args[tokenizer_index])
+ original = adapter.require_bindings
+
+ def replace_after_binding(*positional, **keywords):
+ result = original(*positional, **keywords)
+ tokenizer = json.loads(tokenizer_path.read_text(encoding="utf-8"))
+ record = next(record for record in tokenizer["added_tokens"]
+ if record["id"] == 0)
+ record["content"] = "X" * len(token_payload(0))
+ tokenizer_path.write_text(json.dumps(tokenizer, sort_keys=True),
+ encoding="utf-8", newline="")
+ return result
+
+ with patch.object(adapter, "require_bindings",
+ side_effect=replace_after_binding):
+ status = adapter.main(args)
+ result = json.loads(output_path.read_text(encoding="utf-8"))
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+ self.assertIn("tokenizer-derived", result["disposition"])
+
+ def test_group_sequential_payload_mismatch_cannot_pass(self):
+ status, result = self.run_bundle(payload_mismatch=True)
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+ self.assertRegex(result["disposition"], r"payload identity|payload mismatch")
+
+ def test_final_distribution_payload_mismatch_cannot_pass(self):
+ status, result = self.run_bundle(final_payload_mismatch=True)
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+ self.assertIn("group final", result["disposition"])
+
+ def test_final_distribution_target_must_be_argmax(self):
+ status, result = self.run_bundle(bad_grpg_target=True)
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+ self.assertIn("GRPG target is not its argmax", result["disposition"])
+
+ def test_unsorted_topk_recovers_argmax_by_value(self):
+ status, result = self.run_bundle(topk=2)
+ self.assertEqual(status, 0)
+ self.assertEqual(result["verdict"], "PASS")
+ self.assertEqual(result["logprobs"], {"group": 2, "sequential": 2})
+
+ def test_unsorted_first_entry_cannot_forge_greedy(self):
+ status, result = self.run_bundle(topk=2, forge_unsorted_greedy=True)
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+ self.assertRegex(result["disposition"], r"GRPE sum/shape|SCORE/ECHO")
+
+ def test_unexpected_input_failure_overwrites_stale_pass_fail_closed(self):
+ temporary = tempfile.TemporaryDirectory(); self.addCleanup(temporary.cleanup)
+ argv, output = make_bundle(Path(temporary.name))
+ output.write_text('{"verdict":"PASS","stale":true}\n')
+ manifest = Path(argv[argv.index("--run-manifest") + 1])
+ manifest.write_text("{", encoding="utf-8")
+ status = adapter.main(argv)
+ result = json.loads(output.read_text())
+ self.assertEqual(status, 2)
+ self.assertEqual(result["verdict"], "BLOCK")
+ self.assertNotIn("stale", result)
+
+ def test_duplicate_run_manifest_key_fails_closed(self):
+ temporary = tempfile.TemporaryDirectory(); self.addCleanup(temporary.cleanup)
+ argv, output = make_bundle(Path(temporary.name))
+ manifest = Path(argv[argv.index("--run-manifest") + 1])
+ wire = manifest.read_text(encoding="utf-8")
+ manifest.write_text('{"run_id":"shadow",' + wire[1:], encoding="utf-8")
+ status = adapter.main(argv)
+ result = json.loads(output.read_text())
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+ self.assertIn("duplicate JSON key", result["disposition"])
+
+ def test_group_duplicate_reorder_misattribution_and_greedy_bite(self):
+ for mutation in ({"duplicate_group_echo": True},
+ {"reorder_group_echo": True},
+ {"misattribute_grpg": True},
+ {"bad_grpe_greedy": True}):
+ with self.subTest(mutation=mutation):
+ status, result = self.run_bundle(**mutation)
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+
+ def test_exact_ties_choose_lowest_option_index(self):
+ status, result = self.run_bundle(tie=True)
+ self.assertEqual(status, 0)
+ self.assertEqual(result["verdict"], "PASS")
+ self.assertTrue(all(row["group_argmax"] == 0 and
+ row["sequential_argmax"] == 0 and
+ row["group_margin"] == 0.0 and
+ row["sequential_margin"] == 0.0
+ for row in result["items"]))
+
+ def test_sum_deltas_do_not_use_token_position_boundary(self):
+ status, result = self.run_bundle(sum_boundary=True)
+ self.assertEqual(status, 0)
+ self.assertEqual(result["verdict"], "PASS")
+ self.assertLessEqual(result["maximum_absolute_token_position_delta"],
+ adapter.BOUNDARY)
+ self.assertGreater(max(abs(row["delta"]) for row in result["summed_scores"]),
+ adapter.BOUNDARY)
+
+ def test_left_to_right_binary64_accumulation_is_load_bearing(self):
+ temporary = tempfile.TemporaryDirectory(); self.addCleanup(temporary.cleanup)
+ argv, output = make_bundle(Path(temporary.name), order_sensitive=True)
+
+ def right_to_left(values):
+ total = 0.0
+ for value in reversed(list(values)):
+ total += value
+ return total
+
+ with patch.object(adapter, "ltr", right_to_left):
+ status = adapter.main(argv)
+ result = json.loads(output.read_text())
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+ self.assertRegex(result["disposition"], r"GRPE sum|SCORE/ECHO")
+
+ def test_prefix_swallowed_rows_remain_raw_semantics(self):
+ status, result = self.run_bundle()
+ self.assertEqual(status, 0)
+ rows = result["prefix_swallowed_semantics"]
+ self.assertEqual(len(rows), 24)
+ self.assertTrue(any(row["shared_prefix_logprobs"] !=
+ row["sequential_logprobs"] for row in rows))
+
+ def test_token_position_boundary_is_stop_not_tolerance(self):
+ status, result = self.run_bundle(boundary=True)
+ self.assertEqual(status, 3)
+ self.assertEqual(result["verdict"], "STOP")
+ self.assertGreater(result["maximum_absolute_token_position_delta"],
+ adapter.BOUNDARY)
+
+ def test_argmax_flip_is_fail(self):
+ status, result = self.run_bundle(flip=True)
+ self.assertEqual(status, 1)
+ self.assertEqual(result["verdict"], "FAIL")
+ self.assertGreater(result["argmax_flips"], 0)
+
+ def test_internal_authority_denominator_and_identity_mutations_fail(self):
+ corpus_rows = records(CORPUS)
+ comparison_rows = records(COMPARISONS)
+ policy = json.loads(POLICY.read_text())
+ for kind in ("token_position", "summed_score", "prefix_swallowed",
+ "option_argmax_input", "item_argmax"):
+ mutated = copy.deepcopy(comparison_rows)
+ mutated.pop(next(index for index, row in enumerate(mutated)
+ if row["kind"] == kind))
+ with self.subTest(missing_kind=kind), self.assertRaises(adapter.EvidenceError):
+ adapter.validate_authorities(corpus_rows, mutated, policy)
+
+ duplicated = copy.deepcopy(comparison_rows)
+ first = next(row for row in duplicated if row["kind"] == "token_position")
+ victim = next(index for index, row in enumerate(duplicated)
+ if row["kind"] == "token_position" and row is not first)
+ duplicated[victim] = copy.deepcopy(first)
+ with self.assertRaisesRegex(adapter.EvidenceError, "duplicate"):
+ adapter.validate_authorities(corpus_rows, duplicated, policy)
+
+ swapped = copy.deepcopy(corpus_rows)
+ for left, right in ((swapped[0], swapped[1]),):
+ li, ri = left["context_length"], right["context_length"]
+ left["continuation_tokens"][0], right["continuation_tokens"][0] = (
+ right["continuation_tokens"][0], left["continuation_tokens"][0])
+ left["prompt_tokens"][li] = left["continuation_tokens"][0]
+ right["prompt_tokens"][ri] = right["continuation_tokens"][0]
+ with self.assertRaises(adapter.EvidenceError):
+ adapter.validate_authorities(swapped, comparison_rows, policy)
+
+ shrunken_policy = copy.deepcopy(policy)
+ shrunken_policy["required_comparisons"]["token_positions"] -= 1
+ shrunken = copy.deepcopy(comparison_rows)
+ shrunken.pop(next(index for index, row in enumerate(shrunken)
+ if row["kind"] == "token_position"))
+ with self.assertRaisesRegex(adapter.EvidenceError, "denominator"):
+ adapter.validate_authorities(corpus_rows, shrunken, shrunken_policy)
+
+ def _blank_evidence_file(self, argv, flag):
+ """Empty one evidence file in place and re-bind its manifest digest
+ so the bundle stays otherwise internally consistent -- used to
+ synthesize the "this engine build never emits this wire kind"
+ captures the per-family tests below need."""
+ path = Path(argv[argv.index(f"--{flag}") + 1])
+ path.write_bytes(b"")
+ manifest_path = Path(argv[argv.index("--run-manifest") + 1])
+ manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+ key = flag.replace("-", "_")
+ manifest["evidence_sha256"][key] = adapter.sha256(path)
+ manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
+
+ def test_family_support_direct(self):
+ """Unit-level pin on the helper itself, independent of the CLI
+ path: it reports presence per family and never raises."""
+ empty_score = b""
+ one_score = b"SCORE 0 " + b"a" * 64 + b" -0.1 1 1\n"
+ self.assertEqual(adapter.family_support([], empty_score), (False, False))
+ self.assertEqual(
+ adapter.family_support(
+ [{"kind": "GRPP", "fields": [], "payload": None, "offset": 0}],
+ empty_score),
+ (True, False))
+ self.assertEqual(adapter.family_support([], one_score), (False, True))
+ self.assertEqual(
+ adapter.family_support(
+ [{"kind": "GRPG", "fields": [], "payload": None, "offset": 0}],
+ one_score),
+ (True, True))
+
+ def test_neither_family_supported_is_honestly_unsupported(self):
+ """Combination 1/3 (none): a capture from an engine that
+ never emits group frames or score-evidence lines must not produce
+ a silent pass, nor a plain FAIL indistinguishable from a malformed
+ capture -- it gets its own named verdict, and neither family ran."""
+ temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(temporary.cleanup)
+ argv, output = make_bundle(Path(temporary.name))
+ self._blank_evidence_file(argv, "group-frames")
+ self._blank_evidence_file(argv, "score-evidence")
+
+ status = adapter.main(argv)
+ result = json.loads(output.read_text())
+ self.assertEqual(status, 4)
+ self.assertEqual(result["verdict"], "UNSUPPORTED")
+ self.assertEqual(result["group_checks"], "UNSUPPORTED")
+ self.assertEqual(result["score_checks"], "UNSUPPORTED")
+ self.assertEqual(result["checks_ran"], [])
+ self.assertIn("group frame kinds", result["disposition"])
+
+ def test_score_evidence_only_reports_group_unsupported(self):
+ """Combination 2/3 (evidence-only): score-evidence lines
+ present, group frames absent -- a capture that emits
+ score-evidence lines but no group frames. Must report
+ UNSUPPORTED for the group family (not FAIL,
+ and not an overall PASS), while the score family still runs."""
+ temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(temporary.cleanup)
+ argv, output = make_bundle(Path(temporary.name))
+ self._blank_evidence_file(argv, "group-frames")
+
+ status = adapter.main(argv)
+ result = json.loads(output.read_text())
+ self.assertEqual(status, 4)
+ self.assertEqual(result["verdict"], "UNSUPPORTED")
+ self.assertEqual(result["group_checks"], "UNSUPPORTED")
+ self.assertEqual(result["score_checks"], "PASS")
+ self.assertEqual(result["checks_ran"], ["score"])
+
+ def test_group_frames_only_reports_score_unsupported(self):
+ """Symmetric variant (group present, score-evidence absent):
+ the group family runs and passes on its own denominators, but the
+ overall verdict still cannot be a clean PASS since the score
+ family never ran."""
+ temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(temporary.cleanup)
+ argv, output = make_bundle(Path(temporary.name))
+ self._blank_evidence_file(argv, "score-evidence")
+
+ status = adapter.main(argv)
+ result = json.loads(output.read_text())
+ self.assertEqual(status, 4)
+ self.assertEqual(result["verdict"], "UNSUPPORTED")
+ self.assertEqual(result["group_checks"], "PASS")
+ self.assertEqual(result["score_checks"], "UNSUPPORTED")
+ self.assertEqual(result["checks_ran"], ["group"])
+ self.assertEqual(len(result["token_positions"]), 240)
+
+ def test_both_families_supported_is_the_only_path_to_pass(self):
+ """Combination 3/3 (both): the ordinary case every other test in
+ this module already exercises through run_bundle() -- restated
+ here as the explicit per-family contract: only two families that
+ both ran and passed produce group_checks == score_checks == PASS
+ and the overall PASS."""
+ status, result = self.run_bundle()
+ self.assertEqual(status, 0)
+ self.assertEqual(result["verdict"], "PASS")
+ self.assertEqual(result["group_checks"], "PASS")
+ self.assertEqual(result["score_checks"], "PASS")
+ self.assertEqual(result["checks_ran"], ["group", "score"])
+
+
+class AuthoritySha256LiteralPinTest(unittest.TestCase):
+ """The module's real-authority binding (AUTHORITY_SHA256) is exercised
+ end to end only by RealFrozenCorpusOptInTest above, which skips
+ whenever MW_UR3_CORPUS_DIR is not set -- true on every clean
+ checkout. This test needs no corpus: it pins the three digests the
+ module ships against literals stored here, independently of the
+ module's own source, so a silent edit to either side is caught even
+ with the real objects unavailable.
+
+ The `adapter` name imported at the top of this file has its
+ AUTHORITY_SHA256 replaced with a synthetic triple for the whole
+ module's test run (setUpModule/tearDownModule, above), so this test
+ loads a second, never-patched copy of the same file directly rather
+ than reading through that name."""
+
+ EXPECTED = {
+ "corpus": "d750ab07d611a140836f70ccba31fe28e586630d365131c4ce52e63120a07b6e",
+ "comparisons": "b4c73d22a0d5f7e595f7833f8b31c4669bafbc928fa21192b585df74455f3476",
+ "policy": "308734a31af638e912ccec790df07704d4d0f1d82b253353bddaa39aa261c56f",
+ }
+
+ @staticmethod
+ def _load_unpatched_adapter():
+ spec = importlib.util.spec_from_file_location(
+ "mw_ur3_raw_adapter_unpatched", TOOLS / "mw_ur3_raw_adapter.py")
+ fresh = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(fresh)
+ return fresh
+
+ def test_authority_sha256_matches_pinned_literals(self):
+ fresh = self._load_unpatched_adapter()
+ self.assertEqual(fresh.AUTHORITY_SHA256, self.EXPECTED)
+
+ def test_every_pinned_literal_is_64_lowercase_hex_characters(self):
+ for role, digest in self.EXPECTED.items():
+ with self.subTest(role=role):
+ self.assertEqual(len(digest), 64)
+ self.assertRegex(digest, r"^[0-9a-f]{64}$")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tools/mw_ur3_raw_adapter.py b/c/tools/mw_ur3_raw_adapter.py
new file mode 100644
index 00000000..1024960a
--- /dev/null
+++ b/c/tools/mw_ur3_raw_adapter.py
@@ -0,0 +1,1386 @@
+#!/usr/bin/env python3
+"""Validate the frozen MW-UR3 scoring corpus from raw engine evidence alone.
+
+This adapter is deliberately scoped below any higher-level response-layer
+HTTP surface. It consumes group frames (the batched multi-option wire
+kind), ordinary prompt-echo frames, and score-evidence lines, then joins
+them to the frozen corpus/comparison/policy identities. It never imports
+or fabricates a response-layer logprob shape of its own.
+
+The run manifest and raw request captures are capture authority, not
+analyzer-derived summaries. request_sha256 is checked against both the exact
+bytes submitted and the frozen corpus; token_payload_sha256 is produced
+independently from the run's frozen tokenizer. The tokenizer and container
+manifest are supplied as hashed capture inputs, and the result carries the
+run-manifest digest. Reconstructing any binding from output frames would
+defeat the identity proof and is not a valid capture.
+
+The group frames, and the score-evidence line format, are wire kinds that
+only some engine builds emit. Each is its own check family: a capture
+missing one wire kind is not a malformed capture, it is one this adapter
+cannot run that family's comparisons against, and that is reported by a
+named per-family ``UNSUPPORTED`` status (never silently degraded to a
+partial pass, and never conflated with a real FAIL) -- see
+``family_support`` and the ``group_checks``/``score_checks`` result
+fields.
+
+Process exit status, as implemented in ``main``:
+
+ 0 PASS result["verdict"] == "PASS"
+ 1 FAIL result["verdict"] == "FAIL", or any EvidenceError whose
+ verdict is neither BLOCK nor UNSUPPORTED
+ 2 BLOCK an EvidenceError raised with verdict="BLOCK", any other
+ uncaught exception (fail closed -- never emits a stale
+ PASS), or an OSError clearing a stale --output path
+ before the run starts
+ 3 STOP result["verdict"] == "STOP"
+ 4 UNSUPPORTED result["verdict"] == "UNSUPPORTED", or an EvidenceError
+ raised with verdict="UNSUPPORTED"
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import io
+import json
+import math
+import os
+import re
+import sys
+from collections import Counter, defaultdict
+from pathlib import Path
+
+SCHEMA = "mw-ur3-raw-result-v1"
+RUN_SCHEMA = "mw-ur3-raw-run-v1"
+CAPTURE_MAX_TOKENS = 1
+REQUEST_ID_MAX = (1 << 64) - 1
+TOKENIZER_ID_MAX = 1 << 21
+EXPECTED = {
+ "items": 20,
+ "options": 80,
+ "token_position": 240,
+ "summed_score": 56,
+ "prefix_swallowed": 24,
+ "option_argmax_input": 80,
+ "item_argmax": 20,
+ "register": 420,
+}
+BOUNDARY = 2.624e-05
+AUTHORITY_SHA256 = {
+ "corpus": "d750ab07d611a140836f70ccba31fe28e586630d365131c4ce52e63120a07b6e",
+ "comparisons": "b4c73d22a0d5f7e595f7833f8b31c4669bafbc928fa21192b585df74455f3476",
+ "policy": "308734a31af638e912ccec790df07704d4d0f1d82b253353bddaa39aa261c56f",
+}
+UINT = re.compile(r"(?:0|[1-9][0-9]*)\Z")
+NEG_FINITE = re.compile(
+ r"-(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:e[+-]?[0-9]+)?\Z"
+ r"|0(?:\.0+)?(?:e[+-]?[0-9]+)?\Z",
+ re.IGNORECASE,
+)
+
+
+class EvidenceError(Exception):
+ def __init__(self, message: str, verdict: str = "FAIL",
+ offset: int | None = None) -> None:
+ super().__init__(message)
+ self.verdict = verdict
+ # The byte offset of the frame header this error was raised while
+ # validating, when known -- lets a field-level error (a malformed
+ # number deep inside a frame, say) be traced back to the same
+ # header offset a truncated/malformed-header error already carries.
+ self.offset = offset
+
+
+GROUP_FRAME_KINDS = frozenset({"GRPP", "GRPG", "GRPS", "GRPE"})
+
+
+def family_support(group_frames: list[dict], score_evidence_raw: bytes
+ ) -> tuple[bool, bool]:
+ """Report, per check family, whether this capture's wire kinds let the
+ adapter even attempt that family -- never raises. ``group_frames`` is
+ the already-parsed group-frame stream; ``score_evidence_raw`` is the
+ raw score-evidence bytes (read once, not reopened) scanned for a bare
+ ``SCORE `` line prefix rather than the strict :func:`validate_score`
+ grammar, so a capture with no score-evidence lines at all is
+ distinguished from one with malformed lines -- the latter is a real
+ FAIL, not an UNSUPPORTED.
+ """
+ group_present = bool(
+ {frame["kind"] for frame in group_frames} & GROUP_FRAME_KINDS)
+ text = score_evidence_raw.decode("utf-8", errors="replace")
+ score_present = any(line.startswith("SCORE ") for line in text.splitlines())
+ return group_present, score_present
+
+
+def sha256(path: Path) -> str:
+ h = hashlib.sha256()
+ with path.open("rb") as handle:
+ for chunk in iter(lambda: handle.read(1 << 20), b""):
+ h.update(chunk)
+ return h.hexdigest()
+
+
+def sha256_bytes(data: bytes) -> str:
+ return hashlib.sha256(data).hexdigest()
+
+
+def tokenizer_payloads(raw: bytes, token_ids: set[int],
+ label: str = "tokenizer capture") -> dict[int, bytes]:
+ """Decode exact token payload bytes from the frozen ByteLevel tokenizer.
+
+ This mirrors ``tok_decode`` in ``c/tok.h`` for one token at a time: added
+ tokens render as their literal UTF-8 content, while model-vocabulary pieces
+ are mapped through the inverse GPT-2/ByteLevel byte table. The run manifest
+ is deliberately not an input to this authority.
+ """
+ root = load_json_bytes(raw, label)
+ if not isinstance(root, dict) or not isinstance(root.get("model"), dict):
+ raise EvidenceError("tokenizer capture has no model object")
+ vocab = root["model"].get("vocab")
+ added = root.get("added_tokens", [])
+ if not isinstance(vocab, dict) or not isinstance(added, list):
+ raise EvidenceError("tokenizer capture has malformed vocabulary")
+
+ id_to_piece: dict[int, str] = {}
+ for piece, token in vocab.items():
+ if not isinstance(piece, str) or type(token) is not int or \
+ not 0 <= token <= TOKENIZER_ID_MAX:
+ raise EvidenceError("tokenizer vocabulary entry is malformed")
+ if "\x00" in piece or token in id_to_piece:
+ raise EvidenceError("tokenizer vocabulary token ID is ambiguous")
+ id_to_piece[token] = piece
+
+ added_ids: set[int] = set()
+ for record in added:
+ if not isinstance(record, dict) or not {"id", "content"} <= set(record):
+ raise EvidenceError("tokenizer added-token entry is malformed")
+ token = record.get("id")
+ content = record.get("content")
+ if type(token) is not int or not 0 <= token <= TOKENIZER_ID_MAX or \
+ not isinstance(content, str) or \
+ "\x00" in content or token in added_ids:
+ raise EvidenceError("tokenizer added-token entry is malformed")
+ added_ids.add(token)
+ id_to_piece[token] = content
+
+ direct = set(range(33, 127)) | set(range(161, 173)) | set(range(174, 256))
+ cp_to_byte: dict[int, int] = {}
+ displaced = 0
+ for byte in range(256):
+ codepoint = byte if byte in direct else 256 + displaced
+ if byte not in direct:
+ displaced += 1
+ cp_to_byte[codepoint] = byte
+
+ payloads: dict[int, bytes] = {}
+ for token in token_ids:
+ if type(token) is not int or token < 0 or token not in id_to_piece:
+ raise EvidenceError(f"tokenizer has no captured token ID {token}")
+ piece = id_to_piece[token]
+ if token in added_ids:
+ payloads[token] = piece.encode("utf-8")[:256]
+ continue
+ decoded = bytearray()
+ for character in piece:
+ byte = cp_to_byte.get(ord(character))
+ if byte is not None:
+ decoded.append(byte)
+ payloads[token] = bytes(decoded[:256])
+ return payloads
+
+
+def load_json_bytes(raw: bytes, label: str):
+ def unique_object(pairs):
+ result = {}
+ for key, value in pairs:
+ if key in result:
+ raise EvidenceError(f"{label}: duplicate JSON key {key!r}")
+ result[key] = value
+ return result
+
+ try:
+ text = raw.decode("utf-8")
+ except UnicodeDecodeError as error:
+ raise EvidenceError(f"{label}: invalid UTF-8 JSON") from error
+ return json.loads(text, object_pairs_hook=unique_object)
+
+
+def load_json(path: Path):
+ return load_json_bytes(path.read_bytes(), str(path))
+
+
+def load_jsonl_bytes(raw: bytes, label: str):
+ try:
+ text = raw.decode("utf-8")
+ except UnicodeDecodeError as error:
+ raise EvidenceError(f"{label}: invalid UTF-8 JSONL") from error
+ rows = []
+ for number, line in enumerate(text.splitlines(keepends=True), 1):
+ if not line.endswith("\n") or "\r" in line or not line.strip():
+ raise EvidenceError(f"{label}:{number}: noncanonical JSONL record")
+ try:
+ rows.append(json.loads(line))
+ except json.JSONDecodeError as error:
+ raise EvidenceError(f"{label}:{number}: {error}") from error
+ return rows
+
+
+def load_jsonl(path: Path):
+ return load_jsonl_bytes(path.read_bytes(), str(path))
+
+
+def uint(text: str, label: str) -> int:
+ if not UINT.fullmatch(text):
+ raise EvidenceError(f"noncanonical {label}: {text!r}")
+ return int(text)
+
+
+def request_id(text: str, label: str) -> str:
+ """Return an exact canonical ID in the engine's positive uint64 domain."""
+ value = uint(text, label)
+ if not 1 <= value <= REQUEST_ID_MAX:
+ raise EvidenceError(f"out-of-domain {label}: {text!r}")
+ return str(value)
+
+
+def fixed_metric(text: str, places: int, label: str,
+ upper: float | None = None) -> float:
+ if not re.fullmatch(
+ rf"-?(?:0|[1-9][0-9]*)\.[0-9]{{{places}}}", text):
+ raise EvidenceError(f"noncanonical {label}: {text!r}")
+ value = float(text)
+ if not math.isfinite(value) or value < 0.0 or \
+ (upper is not None and value > upper):
+ raise EvidenceError(f"out-of-range {label}: {text!r}", "BLOCK")
+ return value
+
+
+def validate_prof(fields: list[str], label: str) -> tuple[int, int]:
+ if len(fields) != 10:
+ raise EvidenceError(f"malformed {label} PROF")
+ fixed_metric(fields[1], 3, f"{label} PROF wall")
+ prompt = uint(fields[2], f"{label} PROF prompt")
+ completion = uint(fields[3], f"{label} PROF completion")
+ for index, phase in enumerate(
+ ("disk", "wait", "matmul", "attention", "head"), 4):
+ fixed_metric(fields[index], 3, f"{label} PROF {phase}")
+ uint(fields[9], f"{label} PROF forwards")
+ return prompt, completion
+
+
+def validate_done(fields: list[str], label: str) -> tuple[int, int, int]:
+ if len(fields) != 9 or fields[2] != "STAT":
+ raise EvidenceError(f"malformed {label} DONE")
+ emitted = uint(fields[3], f"{label} DONE emitted")
+ fixed_metric(fields[4], 2, f"{label} DONE throughput")
+ fixed_metric(fields[5], 1, f"{label} DONE hit rate", 100.0)
+ fixed_metric(fields[6], 2, f"{label} DONE RSS")
+ prompt = uint(fields[7], f"{label} DONE prompt")
+ flag = uint(fields[8], f"{label} DONE length flag")
+ if flag not in (0, 1):
+ raise EvidenceError(f"invalid {label} DONE length flag: {flag}")
+ return emitted, prompt, flag
+
+
+def logprob(text: str, label: str, allow_nan: bool = False) -> float | None:
+ if allow_nan and text == "nan":
+ return None
+ if not NEG_FINITE.fullmatch(text):
+ raise EvidenceError(f"invalid {label}: {text!r}")
+ value = float(text)
+ if not math.isfinite(value) or value > 0.0:
+ raise EvidenceError(f"nonfinite/positive {label}: {text!r}", "BLOCK")
+ return value
+
+
+def validate_tail(fields: list[str], start: int, label: str,
+ allow_nan: bool = False,
+ expected_topk: int | None = None
+ ) -> tuple[float | None, int | None, float | None, int]:
+ if len(fields) < start + 2:
+ raise EvidenceError(f"truncated numeric tail: {label}")
+ target = logprob(fields[start], f"{label} target", allow_nan)
+ k = uint(fields[start + 1], f"{label} top-k count")
+ if k > 32 or len(fields) != start + 2 + 2 * k:
+ raise EvidenceError(f"wrong numeric-tail shape: {label}")
+ if expected_topk is not None and k != expected_topk:
+ raise EvidenceError(
+ f"wrong top-k count for {label}: expected {expected_topk}, got {k}")
+ if target is None and k != 0:
+ raise EvidenceError(f"NO_SCORE row has top-k entries: {label}")
+ if target is not None and k == 0:
+ raise EvidenceError(f"scored row has no top-k entries: {label}")
+ seen = set(); pairs = []
+ for index in range(k):
+ token = uint(fields[start + 2 + 2 * index], f"{label} top-k token")
+ if token in seen:
+ raise EvidenceError(f"duplicate top-k token: {label}")
+ seen.add(token)
+ value = logprob(fields[start + 3 + 2 * index],
+ f"{label} top-k logprob")
+ pairs.append((token, value))
+ if not pairs:
+ return target, None, None, k
+ # The engine deliberately emits the selected table in heap-slot order, not
+ # score order. Recover the engine's argmax by value, then its lowest-token-id
+ # tie rule; the first emitted entry has no rank meaning.
+ best_token, best_value = min(pairs, key=lambda pair: (-pair[1], pair[0]))
+ return target, best_token, best_value, k
+
+
+def parse_frames_bytes(raw: bytes, label: str) -> list[dict]:
+ stream = io.BytesIO(raw)
+ frames = []
+ payload_kinds = {"DATA", "ECHO", "GRPP", "GRPG", "TOOL"}
+ while stream.tell() < len(raw):
+ offset = stream.tell()
+ wire = stream.readline()
+ if not wire.endswith(b"\n"):
+ raise EvidenceError(f"{label}: truncated header at byte {offset}")
+ if b"\r" in wire:
+ raise EvidenceError(f"{label}: CR in header at byte {offset}")
+ try:
+ line = wire[:-1].decode("ascii")
+ except UnicodeDecodeError as error:
+ raise EvidenceError(f"{label}: non-ASCII header at byte {offset}") from error
+ if not line:
+ raise EvidenceError(f"{label}: empty header at byte {offset}")
+ fields = line.split(" ")
+ if not fields or not fields[0]:
+ raise EvidenceError(f"{label}: malformed header at byte {offset}")
+ kind = fields[0]
+ payload = None
+ if kind in payload_kinds:
+ if len(fields) < 3:
+ raise EvidenceError(f"{label}: truncated {kind} header")
+ size = uint(fields[2], f"{kind} payload size")
+ if size > 65536:
+ raise EvidenceError(f"{label}: oversized {kind} payload")
+ payload = stream.read(size)
+ if len(payload) != size or stream.read(1) != b"\n":
+ raise EvidenceError(f"{label}: truncated {kind} payload")
+ frames.append({"kind": kind, "fields": fields, "payload": payload,
+ "offset": offset})
+ return frames
+
+
+def parse_frames(path: Path) -> list[dict]:
+ return parse_frames_bytes(path.read_bytes(), str(path))
+
+
+def ltr(values) -> float:
+ total = 0.0
+ for value in values:
+ if value is None or not math.isfinite(value):
+ raise EvidenceError("nonfinite value in binary64 sum", "BLOCK")
+ total += value
+ if not math.isfinite(total):
+ raise EvidenceError("binary64 sum overflow", "BLOCK")
+ return total
+
+
+def lcp(sequences: list[list[int]]) -> int:
+ length = min(map(len, sequences))
+ index = 0
+ while index < length and len({sequence[index] for sequence in sequences}) == 1:
+ index += 1
+ return index
+
+
+def display_path(path: Path) -> str:
+ """A relative, non-host-identifying rendering of a bound path for error
+ text -- never the absolute host path a missing-file OSError would
+ otherwise leak."""
+ try:
+ shown = os.path.relpath(path)
+ except ValueError:
+ shown = None
+ return shown if shown and not shown.startswith("..") else path.name
+
+
+def read_authority(role: str, path: Path) -> bytes:
+ """Read one bound file's bytes exactly once (the same bytes are then
+ both hashed and parsed, to avoid a time-of-check/time-of-use gap
+ between the two). A missing file is a named failure identifying the
+ role and a relative path, never a raw host OSError with an absolute
+ path."""
+ try:
+ return path.read_bytes()
+ except FileNotFoundError:
+ raise EvidenceError(
+ f"missing authority: {role} ({display_path(path)})") from None
+ except OSError as error:
+ raise EvidenceError(
+ f"unreadable authority: {role} ({display_path(path)}): "
+ f"{error.strerror or error}") from None
+
+
+def require_bindings(manifest: dict, paths: dict[str, Path],
+ capture_paths: dict[str, Path],
+ evidence_paths: dict[str, Path],
+ capture_snapshots: dict[str, bytes] | None = None
+ ) -> tuple[dict, dict, dict, dict[str, bytes]]:
+ if manifest.get("schema") != RUN_SCHEMA:
+ raise EvidenceError(f"run manifest schema must be {RUN_SCHEMA}")
+ expected_keys = {
+ "schema", "candidate_head", "candidate_tree", "build_id",
+ "container_id", "model_id", "route_id", "run_id",
+ "authority_sha256", "capture_sha256", "evidence_sha256",
+ "group_request_ids", "sequential_request_ids", "logprobs",
+ "request_sha256", "token_payload_sha256",
+ }
+ if set(manifest) != expected_keys:
+ raise EvidenceError("run manifest field set is not exact")
+ required = ("candidate_head", "candidate_tree", "build_id", "container_id",
+ "model_id", "route_id", "run_id")
+ for key in required:
+ if not isinstance(manifest.get(key), str) or not manifest[key]:
+ raise EvidenceError(f"run manifest missing nonempty {key}")
+ for key in ("candidate_head", "candidate_tree"):
+ if not re.fullmatch(r"[0-9a-f]{40}", manifest[key]):
+ raise EvidenceError(f"run manifest {key} is not a canonical git object")
+
+ read_bytes: dict[str, bytes] = {}
+ for name, path in paths.items():
+ read_bytes[f"authority:{name}"] = read_authority(name, path)
+ hashes = {name: sha256_bytes(read_bytes[f"authority:{name}"]) for name in paths}
+ if hashes != AUTHORITY_SHA256:
+ raise EvidenceError("input authorities are not the frozen MW-UR3 objects")
+ if manifest.get("authority_sha256") != hashes:
+ raise EvidenceError("run manifest authority_sha256 is not exact")
+
+ snapshots = capture_snapshots or {}
+ if not set(snapshots) <= set(capture_paths):
+ raise EvidenceError("capture snapshot field set is not a subset")
+ capture_hashes = {}
+ for name, path in capture_paths.items():
+ read_bytes[f"capture:{name}"] = (
+ snapshots[name] if name in snapshots else read_authority(name, path))
+ capture_hashes[name] = sha256_bytes(read_bytes[f"capture:{name}"])
+ if manifest.get("capture_sha256") != capture_hashes:
+ raise EvidenceError("run manifest capture_sha256 is not exact")
+
+ for name, path in evidence_paths.items():
+ read_bytes[f"evidence:{name}"] = read_authority(name, path)
+ evidence_hashes = {
+ name: sha256_bytes(read_bytes[f"evidence:{name}"]) for name in evidence_paths}
+ if manifest.get("evidence_sha256") != evidence_hashes:
+ raise EvidenceError("run manifest evidence_sha256 is not exact")
+ return hashes, capture_hashes, evidence_hashes, read_bytes
+
+
+def validate_authorities(corpus_rows: list[dict], comparisons: list[dict],
+ policy: dict):
+ if policy.get("schema") != "mw-ur3-policy-v1":
+ raise EvidenceError("wrong MW-UR3 policy schema")
+ required = policy.get("required_comparisons", {})
+ policy_counts = {
+ "token_position": required.get("token_positions"),
+ "summed_score": required.get("summed_scores"),
+ "prefix_swallowed": required.get("prefix_swallowed_semantics"),
+ "option_argmax_input": required.get("option_argmax_inputs"),
+ "item_argmax": required.get("item_argmax"),
+ }
+ if policy_counts != {key: EXPECTED[key] for key in policy_counts}:
+ raise EvidenceError("policy denominator drift")
+ if policy.get("historical_delta_boundary") != BOUNDARY:
+ raise EvidenceError("policy stop boundary drift")
+
+ corpus = {}
+ for row in corpus_rows:
+ key = (row.get("item"), row.get("option"))
+ if row.get("schema") != "mw-ur3-corpus-v1" or key in corpus:
+ raise EvidenceError(f"duplicate/wrong corpus identity: {key}")
+ if row.get("context_length") != len(row.get("context_tokens", [])) or \
+ row.get("continuation_length") != len(row.get("continuation_tokens", [])) or \
+ row.get("prompt_tokens") != row.get("context_tokens", []) + row.get("continuation_tokens", []):
+ raise EvidenceError(f"corpus token/length mismatch: {key}")
+ corpus[key] = row
+ expected_options = {(item, option) for item in range(20) for option in range(4)}
+ if set(corpus) != expected_options or len(corpus_rows) != EXPECTED["options"]:
+ raise EvidenceError("corpus is not exact 20x4 set")
+
+ by_kind = defaultdict(list)
+ identities = set()
+ for row in comparisons:
+ kind = row.get("kind")
+ if row.get("schema") != "mw-ur3-comparison-v1" or kind not in policy_counts:
+ raise EvidenceError(f"unexpected comparison kind/schema: {kind}")
+ if kind == "token_position":
+ ident = (kind, row.get("item"), row.get("option"), row.get("absolute_position"))
+ else:
+ ident = (kind, row.get("item"), row.get("option"))
+ if ident in identities:
+ raise EvidenceError(f"duplicate comparison identity: {ident}")
+ identities.add(ident); by_kind[kind].append(row)
+ counts = Counter(row["kind"] for row in comparisons)
+ if len(comparisons) != EXPECTED["register"] or any(
+ counts[key] != value for key, value in policy_counts.items()):
+ raise EvidenceError(f"comparison census drift: {dict(counts)}")
+
+ lcps = {}
+ for item in range(20):
+ prompts = [corpus[item, option]["prompt_tokens"] for option in range(4)]
+ lcps[item] = lcp(prompts)
+ expected_token = set()
+ expected_semantic = set()
+ expected_sum = set()
+ for key, row in corpus.items():
+ item, option = key; ctx = row["context_length"]; detected = lcps[item]
+ if detected > ctx:
+ expected_semantic.add((item, option, ctx, detected, detected - ctx))
+ else:
+ expected_token.update(
+ (item, option, position, row["prompt_tokens"][position])
+ for position in range(detected, len(row["prompt_tokens"])))
+ expected_sum.add((item, option, tuple(range(ctx, len(row["prompt_tokens"])))))
+ got_token = {(row["item"], row["option"], row["absolute_position"], row["token_id"])
+ for row in by_kind["token_position"]}
+ got_semantic = {(row["item"], row["option"], row["context_length"],
+ row["detected_prefix_length"], row["continuation_positions_not_returned"])
+ for row in by_kind["prefix_swallowed"]}
+ got_sum = {(row["item"], row["option"], tuple(row["positions"]))
+ for row in by_kind["summed_score"]}
+ if got_token != expected_token or got_semantic != expected_semantic or got_sum != expected_sum:
+ raise EvidenceError("comparison identities disagree with corpus-derived LCP sets")
+ option_rows = {(row.get("item"), row.get("option")): row
+ for row in by_kind["option_argmax_input"]}
+ if set(option_rows) != expected_options or len(option_rows) != EXPECTED["options"]:
+ raise EvidenceError("option-argmax identities are not the exact 20x4 set")
+ for (item, option), row in option_rows.items():
+ prompt = corpus[item, option]["prompt_tokens"]
+ context = corpus[item, option]["context_length"]
+ if row.get("group_absolute_positions") != list(range(lcps[item], len(prompt))) or \
+ row.get("sequential_absolute_positions") != list(range(context, len(prompt))) or \
+ row.get("prefix_swallowed_positions") != max(0, lcps[item] - context):
+ raise EvidenceError(f"option-argmax position drift: {(item, option)}")
+ item_rows = {row.get("item"): row for row in by_kind["item_argmax"]}
+ if set(item_rows) != set(range(20)) or len(item_rows) != EXPECTED["items"]:
+ raise EvidenceError("item-argmax identities are not exact items 0..19")
+ for item, row in item_rows.items():
+ if row.get("options") != [0, 1, 2, 3] or \
+ row.get("option_input_identities") != [[item, option] for option in range(4)]:
+ raise EvidenceError(f"item-argmax input drift: {item}")
+ return corpus, by_kind, lcps
+
+
+def token_record(tokens: list[int]) -> bytes:
+ if not tokens:
+ raise EvidenceError("empty canonical token request")
+ return " ".join(map(str, tokens)).encode("ascii")
+
+
+def parse_request_capture_bytes(raw: bytes, label: str, *, group: bool,
+ expected_topk: int) -> list[tuple[str, bytes]]:
+ """Parse the exact canonical SUBMIT bytes written to the engine."""
+ records = []
+ stream = io.BytesIO(raw)
+ while True:
+ offset = stream.tell(); line = stream.readline()
+ if not line:
+ break
+ if not line.endswith(b"\n") or b"\r" in line:
+ raise EvidenceError(f"{label}:{offset}: noncanonical request header")
+ try:
+ fields = line[:-1].decode("ascii").split(" ")
+ except UnicodeDecodeError as error:
+ raise EvidenceError(f"{label}:{offset}: non-ASCII request header") from error
+ expected_fields = 11 if group else 10
+ if len(fields) != expected_fields or any(field == "" for field in fields) or \
+ fields[0] != "SUBMIT":
+ raise EvidenceError(f"{label}:{offset}: malformed request header")
+ rid = request_id(fields[1], "captured request id")
+ slot = uint(fields[2], "captured slot")
+ size = uint(fields[3], "captured payload size")
+ maximum = uint(fields[4], "captured max tokens")
+ gbytes = uint(fields[7], "captured grammar bytes")
+ if slot != 0 or maximum != CAPTURE_MAX_TOKENS or \
+ fields[5:7] != ["0", "1"] or \
+ gbytes != 0 or fields[8] != "ids=1" or \
+ fields[9] != f"logprobs={expected_topk}" or \
+ (group and fields[10] != "group=4"):
+ raise EvidenceError(f"{label}:{offset}: wrong MW-UR3 request profile")
+ payload = stream.read(size)
+ if len(payload) != size or stream.read(1) != b"\n":
+ raise EvidenceError(f"{label}:{offset}: truncated request payload")
+ records.append((rid, payload))
+ return records
+
+
+def parse_request_capture(path: Path, *, group: bool,
+ expected_topk: int) -> list[tuple[str, bytes]]:
+ return parse_request_capture_bytes(
+ path.read_bytes(), str(path), group=group, expected_topk=expected_topk)
+
+
+def request_topk_and_hashes(manifest: dict) -> tuple[int, int, dict, dict]:
+ """Extract and structurally validate the group/sequential topk and the
+ request-byte hash bindings the manifest is required to name -- shared
+ by both the group and the sequential request-capture validators so the
+ two can be run independently: group support does not gate
+ sequential validation, and vice versa."""
+ logprobs = manifest.get("logprobs")
+ if not isinstance(logprobs, dict) or set(logprobs) != {"group", "sequential"}:
+ raise EvidenceError("logprobs binding must name exact group/sequential paths")
+ group_topk, sequential_topk = logprobs["group"], logprobs["sequential"]
+ if type(group_topk) is not int or type(sequential_topk) is not int or \
+ not 1 <= group_topk <= 32 or not 1 <= sequential_topk <= 32:
+ raise EvidenceError("run logprobs bindings must be canonical integers 1..32")
+ bindings = manifest.get("request_sha256")
+ if not isinstance(bindings, dict) or set(bindings) != {"group", "sequential"}:
+ raise EvidenceError("request_sha256 must name exact group/sequential paths")
+ group_hashes, sequential_hashes = bindings["group"], bindings["sequential"]
+ if not isinstance(group_hashes, dict) or \
+ set(group_hashes) != {str(item) for item in range(20)}:
+ raise EvidenceError("group request hashes are not exact items 0..19")
+ expected_sequential = {f"{item}:{option}"
+ for item in range(20) for option in range(4)}
+ if not isinstance(sequential_hashes, dict) or \
+ set(sequential_hashes) != expected_sequential:
+ raise EvidenceError("sequential request hashes are not exact 20x4 set")
+ return group_topk, sequential_topk, group_hashes, sequential_hashes
+
+
+def validate_group_requests(gids: dict, corpus: dict, lcps: dict,
+ group_requests_raw: bytes, label: str,
+ group_topk: int, group_hashes: dict) -> None:
+ """Bind each raw group SUBMIT capture to the exact frozen token bytes."""
+ captured_group = parse_request_capture_bytes(
+ group_requests_raw, label, group=True, expected_topk=group_topk)
+ expected_group_order = [gids[item] for item in range(20)]
+ if [rid for rid, _ in captured_group] != expected_group_order:
+ raise EvidenceError("captured group requests are not the exact ordered 20-set")
+ for item in range(20):
+ prompts = [corpus[item, option]["prompt_tokens"] for option in range(4)]
+ detected = lcps[item]
+ group_wire = b"\n".join(
+ [token_record(prompts[0][:detected])] +
+ [token_record(prompt[detected:]) for prompt in prompts])
+ captured_wire = captured_group[item][1]
+ if captured_wire != group_wire or \
+ group_hashes[str(item)] != sha256_bytes(captured_wire):
+ raise EvidenceError(f"group request-byte binding mismatch item {item}")
+
+
+def validate_sequential_requests(sids: dict, corpus: dict,
+ sequential_requests_raw: bytes, label: str,
+ sequential_topk: int,
+ sequential_hashes: dict) -> None:
+ """Bind each raw sequential SUBMIT capture to the exact frozen token
+ bytes. Always run: today's engine emits ordinary sequential captures
+ regardless of group or score-evidence support."""
+ captured_sequential = parse_request_capture_bytes(
+ sequential_requests_raw, label, group=False, expected_topk=sequential_topk)
+ expected_sequential_order = [sids[item, option]
+ for item in range(20) for option in range(4)]
+ if [rid for rid, _ in captured_sequential] != expected_sequential_order:
+ raise EvidenceError("captured sequential requests are not the exact ordered 80-set")
+ for item in range(20):
+ prompts = [corpus[item, option]["prompt_tokens"] for option in range(4)]
+ for option, prompt in enumerate(prompts):
+ key = f"{item}:{option}"
+ captured_wire = captured_sequential[item * 4 + option][1]
+ if captured_wire != token_record(prompt) or \
+ sequential_hashes[key] != sha256_bytes(captured_wire):
+ raise EvidenceError(
+ f"sequential request-byte binding mismatch {item}/{option}")
+
+
+def request_maps(manifest: dict):
+ group = manifest.get("group_request_ids")
+ sequential = manifest.get("sequential_request_ids")
+ if not isinstance(group, dict) or set(group) != {str(i) for i in range(20)}:
+ raise EvidenceError("group_request_ids must be exact items 0..19")
+ expected_seq = {f"{i}:{o}" for i in range(20) for o in range(4)}
+ if not isinstance(sequential, dict) or set(sequential) != expected_seq:
+ raise EvidenceError("sequential_request_ids must be exact 20x4 set")
+ gids = {int(item): request_id(str(rid), "group request id")
+ for item, rid in group.items()}
+ sids = {tuple(map(int, key.split(":"))):
+ request_id(str(rid), "sequential request id")
+ for key, rid in sequential.items()}
+ all_ids = list(gids.values()) + list(sids.values())
+ if len(set(all_ids)) != len(all_ids):
+ raise EvidenceError("request IDs are not globally unique")
+ return gids, sids
+
+
+def correlate_group(frames: list[dict], gids: dict, corpus: dict, lcps: dict,
+ expected_topk: int):
+ expected_ids = set(gids.values())
+ accepted = {}; done = {}; prof = {}; prefixes = defaultdict(dict)
+ members = defaultdict(dict); current = {}
+ seen_ids = set(); open_rid = None; prof_seen = False
+ accept_order = []
+ expected_order = [gids[item] for item in sorted(gids)]
+ item_for_rid = {rid: item for item, rid in gids.items()}
+ for frame in frames:
+ try:
+ kind, fields = frame["kind"], frame["fields"]
+ if kind == "PROF":
+ if open_rid is None or current or len(fields) != 10 or prof_seen or \
+ set(members[open_rid]) != set(range(4)):
+ raise EvidenceError("misordered/malformed group PROF")
+ prof[open_rid] = validate_prof(fields, "group")
+ prof_seen = True
+ continue
+ if kind in {"HWINFO", "TIERS", "EMAP", "HITS", "STAT", "\x01\x01READY\x01\x01"}:
+ if open_rid is not None:
+ raise EvidenceError(f"unexpected {kind} inside group lifecycle")
+ continue
+ if kind in {"ACCEPT", "DONE", "GRPP", "GRPS", "ECHO", "GRPG", "GRPE", "ERROR"}:
+ if len(fields) < 2:
+ raise EvidenceError(f"truncated group frame {kind}")
+ rid = request_id(fields[1], f"{kind} request id"); seen_ids.add(rid)
+ if rid not in expected_ids:
+ raise EvidenceError(f"unexpected group request id {rid}")
+ else:
+ raise EvidenceError(f"unexpected raw group frame: {kind}")
+ if kind == "ERROR":
+ raise EvidenceError(f"group run contains ERROR for {rid}: {' '.join(fields[2:])}")
+ if kind == "ACCEPT":
+ if len(fields) != 3 or rid in accepted or open_rid is not None or \
+ len(accept_order) >= len(expected_order) or \
+ rid != expected_order[len(accept_order)]:
+ raise EvidenceError(f"duplicate/malformed group ACCEPT {rid}")
+ accepted[rid] = uint(fields[2], "group ACCEPT total")
+ accept_order.append(rid); open_rid = rid; prof_seen = False
+ elif kind == "GRPP":
+ if rid != open_rid or current or members[rid] or prof_seen or len(fields) < 6:
+ raise EvidenceError("misordered/truncated GRPP")
+ position = uint(fields[3], "GRPP position")
+ if position != len(prefixes[rid]):
+ raise EvidenceError(f"out-of-order GRPP position {rid}/{position}")
+ tail = validate_tail(fields, 4, "GRPP", allow_nan=position == 0,
+ expected_topk=0 if position == 0 else expected_topk)
+ if position in prefixes[rid]:
+ raise EvidenceError(f"duplicate GRPP position {rid}/{position}")
+ prefixes[rid][position] = (tail[0], frame["payload"], tail[1])
+ elif kind == "GRPS":
+ item = item_for_rid[rid]
+ member = uint(fields[2], "GRPS member") if len(fields) == 5 else -1
+ if rid != open_rid or len(fields) != 5 or current or prof_seen or \
+ len(prefixes[rid]) != lcps[item] or member != len(members[rid]):
+ raise EvidenceError("malformed GRPS")
+ if member in members[rid]:
+ raise EvidenceError(f"duplicate/nested GRPS {rid}/{member}")
+ members[rid][member] = {"ctx": uint(fields[3], "GRPS ctx"),
+ "len": uint(fields[4], "GRPS len"),
+ "echo": {},
+ "grpg": None, "grpe": None}
+ current[rid] = member
+ elif kind == "ECHO":
+ if rid != open_rid or rid not in current or len(fields) < 6:
+ raise EvidenceError(f"unattributed group ECHO {rid}")
+ position = uint(fields[3], "group ECHO position")
+ tail = validate_tail(fields, 4, "group ECHO",
+ expected_topk=expected_topk)
+ record = members[rid][current[rid]]
+ if record["grpg"] is not None or position != record["ctx"] + len(record["echo"]):
+ raise EvidenceError(f"out-of-order group ECHO {rid}/{position}")
+ if position in record["echo"]:
+ raise EvidenceError(f"duplicate group ECHO {rid}/{position}")
+ record["echo"][position] = (tail[0], frame["payload"], tail[1])
+ elif kind == "GRPG":
+ if rid != open_rid or rid not in current or len(fields) < 7:
+ raise EvidenceError(f"unattributed GRPG {rid}")
+ member = uint(fields[3], "GRPG member")
+ position = uint(fields[4], "GRPG position")
+ if member != current[rid] or member not in members[rid]:
+ raise EvidenceError(f"misattributed/duplicate GRPG {rid}/{member}")
+ record = members[rid][member]
+ if record["grpg"] is not None or \
+ len(record["echo"]) != record["len"] or \
+ position != record["ctx"] + record["len"]:
+ raise EvidenceError(f"misattributed/duplicate GRPG {rid}/{member}")
+ tail = validate_tail(fields, 5, "GRPG",
+ expected_topk=expected_topk)
+ if tail[0] != tail[2]:
+ raise EvidenceError(f"GRPG target is not its argmax {rid}/{member}")
+ record["grpg"] = (position, tail[0], tail[1], frame["payload"])
+ elif kind == "GRPE":
+ if rid != open_rid or rid not in current or len(fields) != 6:
+ raise EvidenceError(f"unattributed/malformed GRPE {rid}")
+ member = uint(fields[2], "GRPE member")
+ if member != current[rid] or members[rid][member]["grpg"] is None:
+ raise EvidenceError(f"misattributed GRPE {rid}/{member}")
+ score = logprob(fields[3], "GRPE score")
+ members[rid][member]["grpe"] = (
+ score, uint(fields[4], "GRPE len"), uint(fields[5], "GRPE greedy"))
+ del current[rid]
+ elif kind == "DONE":
+ if rid != open_rid or len(fields) != 9 or rid in done or current or \
+ not prof_seen or set(members[rid]) != set(range(4)):
+ raise EvidenceError(f"duplicate/malformed group DONE {rid}")
+ done[rid] = validate_done(fields, "group")
+ open_rid = None; prof_seen = False
+ except EvidenceError as error:
+ if error.offset is not None:
+ raise
+ raise EvidenceError(
+ f"byte {frame['offset']}: {error}", error.verdict,
+ offset=frame['offset']) from error
+ if current or open_rid is not None or seen_ids != expected_ids or accept_order != expected_order:
+ raise EvidenceError("incomplete group member lifecycle/request set")
+
+ values = {}; shared = {}; payloads = {}; shared_payloads = {}; finals = {}
+ for item, rid in gids.items():
+ detected = lcps[item]
+ prompts = [corpus[item, option]["prompt_tokens"] for option in range(4)]
+ lengths = [len(prompt) - detected for prompt in prompts]
+ if accepted.get(rid) != detected + sum(lengths):
+ raise EvidenceError(f"wrong group ACCEPT total item {item}")
+ if set(prefixes[rid]) != set(range(detected)):
+ raise EvidenceError(f"wrong GRPP positions item {item}")
+ if prefixes[rid].get(0, ("missing",))[0] is not None:
+ raise EvidenceError(f"GRPP pos0 is not NO_SCORE item {item}")
+ if set(members[rid]) != set(range(4)):
+ raise EvidenceError(f"wrong group member set item {item}")
+ shared[item] = {position: record[0]
+ for position, record in prefixes[rid].items()}
+ shared_payloads[item] = {position: record[1]
+ for position, record in prefixes[rid].items()}
+ for option in range(4):
+ record = members[rid][option]; prompt = prompts[option]
+ expected_positions = set(range(detected, len(prompt)))
+ if record["ctx"] != detected or record["len"] != lengths[option] or \
+ set(record["echo"]) != expected_positions:
+ raise EvidenceError(f"wrong group member shape item/option {item}/{option}")
+ if record["grpg"] is None or record["grpg"][0] != len(prompt):
+ raise EvidenceError(f"missing/wrong GRPG item/option {item}/{option}")
+ score = ltr(record["echo"][position][0]
+ for position in sorted(expected_positions))
+ expected_greedy = int(all(
+ record["echo"][position][2] == prompt[position]
+ for position in sorted(expected_positions)))
+ if record["grpe"] is None or record["grpe"][0] != score or \
+ record["grpe"][1] != lengths[option] or \
+ record["grpe"][2] != expected_greedy:
+ raise EvidenceError(f"GRPE sum/shape mismatch item/option {item}/{option}")
+ for position, evidence in record["echo"].items():
+ values[item, option, position] = evidence[0]
+ payloads[item, option, position] = evidence[1]
+ finals[item, option] = {
+ "absolute_position": record["grpg"][0],
+ "target_logprob": record["grpg"][1],
+ "token_id": record["grpg"][2],
+ "payload": record["grpg"][3],
+ }
+ summary = done.get(rid)
+ if prof.get(rid) != (accepted[rid], 0):
+ raise EvidenceError(f"wrong group PROF item {item}")
+ if summary is None or summary != (0, accepted[rid], 0):
+ raise EvidenceError(f"wrong group DONE item {item}")
+ return values, shared, payloads, shared_payloads, finals
+
+
+def correlate_sequential(frames: list[dict], sids: dict, corpus: dict,
+ expected_topk: int):
+ expected_ids = set(sids.values()); accepted = {}; done = {}; prof = {}
+ echoes = defaultdict(dict); generations = {}
+ data_count = defaultdict(int)
+ seen_ids = set(); open_rid = None; prof_seen = False; accept_order = []
+ expected_order = [sids[key] for key in sorted(sids)]
+ key_for_rid = {rid: key for key, rid in sids.items()}
+ for frame in frames:
+ try:
+ kind, fields = frame["kind"], frame["fields"]
+ if kind == "PROF":
+ if open_rid is None or len(fields) != 10 or prof_seen:
+ raise EvidenceError("misordered/malformed sequential PROF")
+ key = key_for_rid[open_rid]
+ if set(echoes[open_rid]) != set(range(len(corpus[key]["prompt_tokens"]))):
+ raise EvidenceError("sequential PROF before complete ECHO evidence")
+ prof[open_rid] = validate_prof(fields, "sequential")
+ prof_seen = True
+ continue
+ if kind in {"HWINFO", "TIERS", "EMAP", "HITS", "STAT", "\x01\x01READY\x01\x01"}:
+ continue
+ if kind in {"ACCEPT", "DONE", "ECHO", "DATA", "ERROR"}:
+ if len(fields) < 2:
+ raise EvidenceError(f"truncated sequential {kind}")
+ rid = request_id(fields[1], f"{kind} request id"); seen_ids.add(rid)
+ if rid not in expected_ids:
+ raise EvidenceError(f"unexpected sequential request id {rid}")
+ else:
+ raise EvidenceError(f"unexpected raw sequential frame: {kind}")
+ if kind == "ERROR":
+ raise EvidenceError(f"sequential run contains ERROR for {rid}")
+ if kind == "ACCEPT":
+ if len(fields) != 3 or rid in accepted or open_rid is not None or \
+ len(accept_order) >= len(expected_order) or \
+ rid != expected_order[len(accept_order)]:
+ raise EvidenceError(f"duplicate/malformed sequential ACCEPT {rid}")
+ accepted[rid] = uint(fields[2], "sequential ACCEPT total")
+ accept_order.append(rid); open_rid = rid; prof_seen = False
+ elif kind == "ECHO":
+ if rid != open_rid or prof_seen or len(fields) < 6:
+ raise EvidenceError("truncated sequential ECHO")
+ position = uint(fields[3], "sequential ECHO position")
+ if position != len(echoes[rid]):
+ raise EvidenceError(f"out-of-order sequential ECHO {rid}/{position}")
+ tail = validate_tail(
+ fields, 4, "sequential ECHO", allow_nan=position == 0,
+ expected_topk=0 if position == 0 else expected_topk)
+ if position in echoes[rid]:
+ raise EvidenceError(f"duplicate sequential ECHO {rid}/{position}")
+ echoes[rid][position] = (tail[0], frame["payload"], tail[1])
+ elif kind == "DATA":
+ if rid != open_rid or prof_seen or len(fields) < 6:
+ raise EvidenceError(f"misordered sequential DATA {rid}")
+ ordinal = data_count[rid]
+ if ordinal >= CAPTURE_MAX_TOKENS:
+ raise EvidenceError(f"wrong sequential DATA lifecycle {rid}/{ordinal}")
+ tail = validate_tail(fields, 3, "sequential DATA",
+ expected_topk=expected_topk)
+ if tail[0] != tail[2]:
+ raise EvidenceError(f"sequential DATA target is not its argmax {rid}")
+ generations[rid] = {
+ "ordinal": ordinal,
+ "target_logprob": tail[0],
+ "token_id": tail[1],
+ "payload": frame["payload"],
+ }
+ data_count[rid] += 1
+ elif kind == "DONE":
+ if rid != open_rid or len(fields) != 9 or rid in done or not prof_seen:
+ raise EvidenceError(f"duplicate/malformed sequential DONE {rid}")
+ done[rid] = validate_done(fields, "sequential")
+ open_rid = None; prof_seen = False
+ except EvidenceError as error:
+ if error.offset is not None:
+ raise
+ raise EvidenceError(
+ f"byte {frame['offset']}: {error}", error.verdict,
+ offset=frame['offset']) from error
+ if open_rid is not None or seen_ids != expected_ids or accept_order != expected_order:
+ raise EvidenceError("incomplete sequential request set")
+ values = {}; top_tokens = {}; payloads = {}; generated = {}
+ for key, rid in sids.items():
+ prompt = corpus[key]["prompt_tokens"]
+ if accepted.get(rid) != len(prompt) or set(echoes[rid]) != set(range(len(prompt))):
+ raise EvidenceError(f"wrong sequential ECHO shape {key}")
+ if echoes[rid][0][0] is not None:
+ raise EvidenceError(f"sequential pos0 is not NO_SCORE {key}")
+ summary = done.get(rid)
+ if prof.get(rid) != (len(prompt), CAPTURE_MAX_TOKENS):
+ raise EvidenceError(f"wrong sequential PROF {key}")
+ if data_count[rid] != CAPTURE_MAX_TOKENS or rid not in generations:
+ raise EvidenceError(f"missing sequential DATA lifecycle {key}")
+ if summary != (CAPTURE_MAX_TOKENS, len(prompt), 1):
+ raise EvidenceError(f"wrong sequential DONE {key}")
+ for position, evidence in echoes[rid].items():
+ values[key[0], key[1], position] = evidence[0]
+ payloads[key[0], key[1], position] = evidence[1]
+ top_tokens[key[0], key[1], position] = evidence[2]
+ generated[key] = generations[rid]
+ return values, top_tokens, payloads, generated
+
+
+def validate_payload_attribution(manifest: dict, tokenizer_raw: bytes,
+ tokenizer_label: str,
+ corpus: dict, lcps: dict,
+ sequential_payloads: dict, generations: dict,
+ group_payloads: dict | None = None,
+ shared_payloads: dict | None = None,
+ finals: dict | None = None,
+ ) -> tuple[list[dict], list[dict]]:
+ """Join every payload byte string to an independently bound token ID.
+
+ ``group_payloads``/``shared_payloads``/``finals`` are ``None`` when the
+ group check family is unsupported for this capture: the
+ sequential-side attribution (every prompt position, every generation)
+ still runs unconditionally, but the group/sequential cross-check and
+ the group-final-distribution binding are skipped rather than raising.
+ """
+ token_hashes = manifest.get("token_payload_sha256")
+ if not isinstance(token_hashes, dict):
+ raise EvidenceError("missing token_payload_sha256 identity bindings")
+ expected_tokens = {
+ token for row in corpus.values() for token in row["prompt_tokens"]
+ }
+ if finals is not None:
+ expected_tokens.update(record["token_id"] for record in finals.values())
+ expected_tokens.update(record["token_id"] for record in generations.values())
+ derived_payloads = tokenizer_payloads(
+ tokenizer_raw, expected_tokens, tokenizer_label)
+ if set(token_hashes) != {str(token) for token in expected_tokens} or any(
+ not isinstance(digest, str) or
+ not re.fullmatch(r"[0-9a-f]{64}", digest)
+ for digest in token_hashes.values()):
+ raise EvidenceError("token payload bindings are not the exact observed token set")
+ for token, payload in derived_payloads.items():
+ if token_hashes[str(token)] != sha256_bytes(payload):
+ raise EvidenceError(
+ f"token payload identity is not tokenizer-derived: {token}")
+
+ def bind(token: int, payload: bytes, label: str) -> None:
+ if payload != derived_payloads[token] or \
+ token_hashes[str(token)] != sha256_bytes(payload):
+ raise EvidenceError(f"token payload identity mismatch: {label}")
+
+ for item in range(20):
+ prompts = [corpus[item, option]["prompt_tokens"] for option in range(4)]
+ detected = lcps[item]
+ if group_payloads is not None:
+ for position in range(detected):
+ shared_payload = shared_payloads[item][position]
+ token = prompts[0][position]
+ bind(token, shared_payload, f"group shared {item}/{position}")
+ for option in range(4):
+ sequential_payload = sequential_payloads[item, option, position]
+ bind(token, sequential_payload,
+ f"sequential shared {item}/{option}/{position}")
+ if shared_payload != sequential_payload:
+ raise EvidenceError(
+ f"group/sequential shared payload mismatch {item}/{option}/{position}")
+ for option, prompt in enumerate(prompts):
+ for position in range(detected, len(prompt)):
+ group_payload = group_payloads[item, option, position]
+ sequential_payload = sequential_payloads[item, option, position]
+ token = prompt[position]
+ bind(token, group_payload,
+ f"group continuation {item}/{option}/{position}")
+ bind(token, sequential_payload,
+ f"sequential continuation {item}/{option}/{position}")
+ if group_payload != sequential_payload:
+ raise EvidenceError(
+ f"group/sequential payload mismatch {item}/{option}/{position}")
+ else:
+ for option, prompt in enumerate(prompts):
+ for position in range(len(prompt)):
+ sequential_payload = sequential_payloads[item, option, position]
+ bind(token := prompt[position], sequential_payload,
+ f"sequential {item}/{option}/{position}")
+
+ final_rows = []
+ if finals is not None:
+ for (item, option), record in sorted(finals.items()):
+ token = record["token_id"]
+ bind(token, record["payload"], f"group final {item}/{option}")
+ final_rows.append({
+ "item": item,
+ "option": option,
+ "absolute_position": record["absolute_position"],
+ "token_id": token,
+ "target_logprob": record["target_logprob"],
+ "payload_sha256": sha256_bytes(record["payload"]),
+ })
+ generation_rows = []
+ for (item, option), record in sorted(generations.items()):
+ token = record["token_id"]
+ bind(token, record["payload"], f"sequential generation {item}/{option}")
+ generation_rows.append({
+ "item": item,
+ "option": option,
+ "ordinal": record["ordinal"],
+ "token_id": token,
+ "target_logprob": record["target_logprob"],
+ "payload_sha256": sha256_bytes(record["payload"]),
+ })
+ return final_rows, generation_rows
+
+
+def validate_score_bytes(raw: bytes, label: str, corpus_rows: list[dict],
+ sequential: dict, sequential_top1: dict):
+ try:
+ text = raw.decode("ascii")
+ except UnicodeDecodeError as error:
+ raise EvidenceError(f"{label}: non-ASCII SCORE evidence") from error
+ rows = []
+ for number, line in enumerate(text.splitlines(keepends=True), 1):
+ fields = line.rstrip("\n").split(" ") if line.endswith("\n") else []
+ if len(fields) != 6 or fields[0] != "SCORE":
+ raise EvidenceError(f"{label}:{number}: malformed SCORE evidence")
+ ordinal = uint(fields[1], "SCORE ordinal")
+ if ordinal != len(rows) or not re.fullmatch(r"[0-9a-f]{64}", fields[2]):
+ raise EvidenceError(f"{label}:{number}: SCORE identity/order mismatch")
+ value = logprob(fields[3], "SCORE sum")
+ contlen = uint(fields[4], "SCORE continuation length")
+ greedy = uint(fields[5], "SCORE greedy")
+ if greedy not in (0, 1):
+ raise EvidenceError(f"{label}:{number}: bad SCORE greedy")
+ corpus = corpus_rows[ordinal]
+ wire = (f"{corpus['context_length']} {corpus['continuation_length']} " +
+ " ".join(map(str, corpus["prompt_tokens"])) + "\n").encode()
+ digest = hashlib.sha256(wire).hexdigest()
+ positions = range(corpus["context_length"], len(corpus["prompt_tokens"]))
+ expected = ltr(sequential[corpus["item"], corpus["option"], p]
+ for p in positions)
+ expected_greedy = int(all(
+ sequential_top1[corpus["item"], corpus["option"], position] ==
+ corpus["prompt_tokens"][position] for position in positions))
+ if fields[2] != digest or contlen != corpus["continuation_length"] or \
+ value != expected or greedy != expected_greedy:
+ raise EvidenceError(f"{label}:{number}: SCORE/ECHO mismatch")
+ rows.append(value)
+ if len(rows) != EXPECTED["options"]:
+ raise EvidenceError("SCORE evidence is not exact 80-row set")
+ return rows
+
+
+def validate_score(path: Path, corpus_rows: list[dict], sequential: dict,
+ sequential_top1: dict):
+ return validate_score_bytes(
+ path.read_bytes(), str(path), corpus_rows, sequential, sequential_top1)
+
+
+# "worst of the families" ranking: a genuine FAIL always outranks
+# an UNSUPPORTED family, a boundary STOP outranks a plain UNSUPPORTED, and
+# only two families that both actually ran and passed can ever produce the
+# overall PASS.
+_VERDICT_RANK = {"PASS": 0, "UNSUPPORTED": 1, "STOP": 2, "FAIL": 3}
+
+
+def build_group_comparisons(by_kind: dict, group: dict, sequential: dict,
+ shared: dict) -> dict:
+ """Join the group-dependent comparison kinds against the group and
+ sequential per-position values. Isolated from :func:`analyze` so the
+ "both families supported" and "group-only" paths share one oracle."""
+ token_rows = []
+ for row in by_kind["token_position"]:
+ key = (row["item"], row["option"], row["absolute_position"])
+ if key not in group or key not in sequential:
+ raise EvidenceError(f"missing token-position evidence {key}")
+ delta = group[key] - sequential[key]
+ token_rows.append({**row, "group_logprob": group[key],
+ "sequential_logprob": sequential[key], "delta": delta})
+ max_abs = max(abs(row["delta"]) for row in token_rows)
+
+ sum_rows = []
+ for row in by_kind["summed_score"]:
+ item, option = row["item"], row["option"]
+ positions = row["positions"]
+ gs = ltr(group[item, option, position] for position in positions)
+ ss = ltr(sequential[item, option, position] for position in positions)
+ sum_rows.append({**row, "group_score": gs, "sequential_score": ss,
+ "delta": gs - ss})
+
+ semantic_rows = []
+ for row in by_kind["prefix_swallowed"]:
+ item, option = row["item"], row["option"]
+ positions = list(range(row["context_length"], row["detected_prefix_length"]))
+ semantic_rows.append({**row, "positions": positions,
+ "shared_prefix_logprobs": [shared[item][position] for position in positions],
+ "sequential_logprobs": [sequential[item, option, position]
+ for position in positions]})
+
+ option_rows = []
+ option_scores = {}
+ for row in by_kind["option_argmax_input"]:
+ item, option = row["item"], row["option"]
+ gs = ltr(group[item, option, position]
+ for position in row["group_absolute_positions"])
+ ss = ltr(sequential[item, option, position]
+ for position in row["sequential_absolute_positions"])
+ option_scores[item, option] = (gs, ss)
+ option_rows.append({**row, "group_score": gs, "sequential_score": ss})
+
+ item_rows = []
+ flips = 0
+ for row in by_kind["item_argmax"]:
+ item = row["item"]
+ group_order = sorted(range(4), key=lambda option: (-option_scores[item, option][0], option))
+ seq_order = sorted(range(4), key=lambda option: (-option_scores[item, option][1], option))
+ ga, sa = group_order[0], seq_order[0]
+ gm = option_scores[item, ga][0] - option_scores[item, group_order[1]][0]
+ sm = option_scores[item, sa][1] - option_scores[item, seq_order[1]][1]
+ flip = ga != sa; flips += int(flip)
+ item_rows.append({**row, "group_argmax": ga, "sequential_argmax": sa,
+ "flip": flip, "group_margin": gm,
+ "sequential_margin": sm, "minimum_margin": min(gm, sm)})
+ return {"token_rows": token_rows, "sum_rows": sum_rows,
+ "semantic_rows": semantic_rows, "option_rows": option_rows,
+ "item_rows": item_rows, "max_abs": max_abs, "flips": flips}
+
+
+def analyze(args) -> dict:
+ paths = {"corpus": args.corpus, "comparisons": args.comparisons,
+ "policy": args.policy}
+ capture_paths = {"tokenizer": args.tokenizer,
+ "container_manifest": args.container_manifest}
+ evidence_paths = {"group_frames": args.group_frames,
+ "sequential_frames": args.sequential_frames,
+ "score_evidence": args.score_evidence,
+ "group_requests": args.group_requests,
+ "sequential_requests": args.sequential_requests}
+ run_manifest_raw = read_authority("run manifest", args.run_manifest)
+ run_manifest_sha256 = sha256_bytes(run_manifest_raw)
+ manifest = load_json_bytes(run_manifest_raw, str(args.run_manifest))
+ tokenizer_raw = read_authority("tokenizer", args.tokenizer)
+ authority_hashes, capture_hashes, evidence_hashes, read_bytes = require_bindings(
+ manifest, paths, capture_paths, evidence_paths,
+ capture_snapshots={"tokenizer": tokenizer_raw})
+ corpus_rows = load_jsonl_bytes(read_bytes["authority:corpus"], "corpus")
+ comparisons = load_jsonl_bytes(read_bytes["authority:comparisons"], "comparisons")
+ policy = load_json_bytes(read_bytes["authority:policy"], "policy")
+ corpus, by_kind, lcps = validate_authorities(corpus_rows, comparisons, policy)
+ gids, sids = request_maps(manifest)
+ group_topk, sequential_topk, group_hashes, sequential_hashes = \
+ request_topk_and_hashes(manifest)
+
+ group_frames_raw = read_bytes["evidence:group_frames"]
+ sequential_frames_raw = read_bytes["evidence:sequential_frames"]
+ score_evidence_raw = read_bytes["evidence:score_evidence"]
+ group_frames = parse_frames_bytes(group_frames_raw, str(args.group_frames))
+ sequential_frames = parse_frames_bytes(
+ sequential_frames_raw, str(args.sequential_frames))
+ group_supported, score_supported = family_support(
+ group_frames, score_evidence_raw)
+
+ # Sequential is always attempted: today's engine already emits ordinary
+ # ECHO/DATA frames and sequential SUBMIT captures regardless of group
+ # or score-evidence support.
+ validate_sequential_requests(
+ sids, corpus, read_bytes["evidence:sequential_requests"],
+ str(args.sequential_requests), sequential_topk, sequential_hashes)
+ sequential, sequential_top1, sequential_payloads, generations = correlate_sequential(
+ sequential_frames, sids, corpus, sequential_topk)
+
+ group = shared = group_payloads = shared_payloads = finals = None
+ if group_supported:
+ validate_group_requests(
+ gids, corpus, lcps, read_bytes["evidence:group_requests"],
+ str(args.group_requests), group_topk, group_hashes)
+ group, shared, group_payloads, shared_payloads, finals = correlate_group(
+ group_frames, gids, corpus, lcps, group_topk)
+
+ final_rows, generation_rows = validate_payload_attribution(
+ manifest, tokenizer_raw, str(args.tokenizer), corpus, lcps,
+ sequential_payloads, generations,
+ group_payloads=group_payloads, shared_payloads=shared_payloads,
+ finals=finals)
+
+ score_checks = "UNSUPPORTED"
+ if score_supported:
+ validate_score_bytes(
+ score_evidence_raw, str(args.score_evidence),
+ corpus_rows, sequential, sequential_top1)
+ score_checks = "PASS"
+
+ bindings = {key: manifest[key] for key in (
+ "candidate_head", "candidate_tree", "build_id", "container_id",
+ "model_id", "route_id", "run_id")}
+ common = {"schema": SCHEMA,
+ "bindings": bindings,
+ "run_manifest_sha256": run_manifest_sha256,
+ "authority_sha256": authority_hashes,
+ "capture_sha256": capture_hashes,
+ "evidence_sha256": evidence_hashes,
+ "request_sha256": manifest["request_sha256"],
+ "logprobs": manifest["logprobs"],
+ "request_profile": {"max_tokens": CAPTURE_MAX_TOKENS,
+ "temperature": 0, "top_p": 1,
+ "ids": 1, "grammar_bytes": 0},
+ "token_payload_sha256": manifest["token_payload_sha256"],
+ "token_position_stop_boundary": BOUNDARY,
+ "final_distributions": final_rows,
+ "sequential_generations": generation_rows}
+
+ if not group_supported:
+ group_checks = "UNSUPPORTED"
+ overall = max(group_checks, score_checks, key=_VERDICT_RANK.get)
+ disposition = (
+ "group frame kinds absent (UNSUPPORTED); "
+ f"score-evidence checks {score_checks.lower()} "
+ f"(ran: {'score' if score_supported else 'none'})")
+ return {**common, "verdict": overall, "disposition": disposition,
+ "group_checks": group_checks, "score_checks": score_checks,
+ "checks_ran": (["score"] if score_supported else [])}
+
+ joined = build_group_comparisons(by_kind, group, sequential, shared)
+ observed = {"items": len(joined["item_rows"]),
+ "options": len(joined["option_rows"]),
+ "token_position": len(joined["token_rows"]),
+ "summed_score": len(joined["sum_rows"]),
+ "prefix_swallowed": len(joined["semantic_rows"]),
+ "option_argmax_input": len(joined["option_rows"]),
+ "item_argmax": len(joined["item_rows"]),
+ "register": len(comparisons)}
+ if observed != EXPECTED:
+ raise EvidenceError(f"result denominator mismatch: {observed}")
+
+ group_checks = "PASS"
+ if joined["flips"]:
+ group_checks = "FAIL"
+ elif joined["max_abs"] > BOUNDARY:
+ group_checks = "STOP"
+ overall = max(group_checks, score_checks, key=_VERDICT_RANK.get)
+
+ if overall == "PASS":
+ disposition = "complete exact-authority raw-engine evidence"
+ elif group_checks == "FAIL":
+ disposition = "one or more item argmax flips"
+ elif group_checks == "STOP":
+ disposition = "paired token-position boundary exceeded; return raw evidence"
+ else:
+ disposition = (
+ "group checks passed but score-evidence lines are absent "
+ "(UNSUPPORTED); ran: group")
+
+ return {**common, "verdict": overall, "disposition": disposition,
+ "denominators": observed,
+ "group_checks": group_checks, "score_checks": score_checks,
+ "checks_ran": ["group"] + (["score"] if score_supported else []),
+ "maximum_absolute_token_position_delta": joined["max_abs"],
+ "argmax_flips": joined["flips"],
+ "token_positions": joined["token_rows"],
+ "summed_scores": joined["sum_rows"],
+ "prefix_swallowed_semantics": joined["semantic_rows"],
+ "option_argmax_inputs": joined["option_rows"],
+ "items": joined["item_rows"]}
+
+
+def parser() -> argparse.ArgumentParser:
+ result = argparse.ArgumentParser()
+ for name in ("corpus", "comparisons", "policy", "run-manifest",
+ "tokenizer", "container-manifest", "group-requests",
+ "sequential-requests", "group-frames", "sequential-frames",
+ "score-evidence"):
+ result.add_argument(f"--{name}", required=True, type=Path)
+ result.add_argument("--output", type=Path)
+ return result
+
+
+def main(argv=None) -> int:
+ args = parser().parse_args(argv)
+ if args.output:
+ try:
+ args.output.unlink(missing_ok=True)
+ except OSError as error:
+ sys.stderr.write(f"cannot clear stale evidence output: {error}\n")
+ return 2
+ try:
+ result = analyze(args)
+ status = {"PASS": 0, "FAIL": 1, "STOP": 3, "UNSUPPORTED": 4}[result["verdict"]]
+ except EvidenceError as error:
+ result = {"schema": SCHEMA, "verdict": error.verdict,
+ "disposition": str(error)}
+ status = {"BLOCK": 2, "UNSUPPORTED": 4}.get(error.verdict, 1)
+ except Exception as error: # fail closed: never preserve/emit a stale PASS
+ result = {"schema": SCHEMA, "verdict": "BLOCK",
+ "disposition": f"internal evidence failure: {type(error).__name__}: {error}"}
+ status = 2
+ wire = json.dumps(result, sort_keys=True, separators=(",", ":")) + "\n"
+ if args.output:
+ temporary = args.output.with_name(
+ f".{args.output.name}.tmp.{os.getpid()}")
+ try:
+ with temporary.open("x", encoding="utf-8", newline="") as handle:
+ handle.write(wire); handle.flush(); os.fsync(handle.fileno())
+ os.replace(temporary, args.output)
+ except OSError as error:
+ temporary.unlink(missing_ok=True)
+ sys.stderr.write(f"cannot commit evidence output: {error}\n")
+ return 2
+ else:
+ try:
+ sys.stdout.write(wire); sys.stdout.flush()
+ except OSError:
+ return 2
+ return status
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())