Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
move ablate-checker material out of this PR
monotophicfe50907a54db
diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex 9d2558bc..1ef3d3af 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -136,11 +136,6 @@ jobs:# run it -- the second half of #720.mkdir -p dist/toolscp c/tools/k3_tokenizer.py dist/tools/- # check_ablate_evidence.py verifies the ABLATE evidence that eval_glm.py- # (packaged above) produces. Nothing in the code names it -- its caller is a- # person with argparse arguments -- so pack_python's import walk cannot reach- # it, exactly like k3_tokenizer.py. Copied explicitly for the same reason.- cp c/tools/check_ablate_evidence.py dist/tools/cp c/coli dist/# Windows has nothing to click otherwise: `coli` is an extensionless# Python script, and the .exe files are engines that exit at once@@ -195,9 +190,6 @@ jobs:# Kimi K3 needs tokenizer.json synthesized from tiktoken.model; without this# script in the archive the engine ships but cannot be driven by coli chat.test -f tools/k3_tokenizer.py || { echo "FAIL: k3_tokenizer.py missing from archive"; exit 1; }- # Same position as k3_tokenizer.py: reachable only by a person, so only an- # explicit gate can prove it shipped.- test -f tools/check_ablate_evidence.py || { echo "FAIL: check_ablate_evidence.py missing from archive"; exit 1; }# E tutto cio' che coli raggiunge, import e sottoprocessi. Il controllo# qui sopra guarda l'unico file che il passo di copia aveva appena# scritto: conferma se stesso, ed e' il motivo per cui la v1.10.0 e'@@ -210,8 +202,6 @@ jobs:# fine. The check was wrong, not the artifact.python3 -c "import ast; ast.parse(open('tools/k3_tokenizer.py', encoding='utf-8').read())" \|| { echo "FAIL: packaged k3_tokenizer.py does not parse"; exit 1; }- python3 -c "import ast; ast.parse(open('tools/check_ablate_evidence.py', encoding='utf-8').read())" \- || { echo "FAIL: packaged check_ablate_evidence.py does not parse"; exit 1; }COLI_EXPECT="$SIBLINGS" python3 - <<'PYCHK'import importlib.machinery, importlib.util, json, os, sys, tempfilefrom family_registry import all_familiesdiff --git a/c/tests/test_check_ablate_evidence.py b/c/tests/test_check_ablate_evidence.pydeleted file mode 100644index 906cd552..00000000--- a/c/tests/test_check_ablate_evidence.py+++ /dev/null@@ -1,915 +0,0 @@-"""tools/check_ablate_evidence.py must accept only a complete, self-binding-ABLATE evidence artifact and reject every other input: a truncated or-replayed record stream, a header that does not bind the manifest or the-external config.json it is checked against, any field with the wrong-type, range, or key set at any of the four record kinds (header, item-header, target row, terminal completion), and a target row whose fields-contradict one another in a way the producer can never emit.--Checks enumerated from the source (`tools/check_ablate_evidence.py`,-read in full before writing this module) and covered below, grouped by-the function that performs them:--- `_checked_engine_text_size` / `_bounded_config_bytes`: the 256 MiB- inclusive engine text limit, both sides.-- `_reject_constant`, `_reject_duplicate_keys`, `_json_record`: no- NaN/Infinity JSON constants, no duplicate object keys, invalid- JSON/non-ASCII text rejected.-- `_config_identity`: empty file; invalid JSON; non-object root; each- of vocab_size/num_hidden_layers/n_routed_experts/first_k_dense_replace- missing or out of range.-- `_manifest_proof`: framing -- an empty manifest, an empty record, a- carriage return inside a record and an embedded NUL are refused, while- CRLF endings and a missing final newline are accepted and reduced to- the canonical form the engine binds; non-ASCII line; non-canonical integer grammar; too few fields;- every per-item field bound (item id, T, prompt, mode, cell count);- the mode/cell-count pairing rule; the field-count/denominator- arithmetic; every per-cell bound (layer, expert, applied-target,- mode-3 vs other-mode applied-target rule, duplicate cell);- out-of-vocabulary tokens; duplicate item ids across lines.-- `validate_ablate_evidence`: the evidence framing check; the header's- key set, type, and range checks; the header-vs-config identity- check; the header-vs-manifest-proof binding check; the item header's- key set, type, and manifest-order check; the target row's key set,- type, and identity checks; the three cross-field invariants (below);- the top-k list's shape, range, and- uniqueness checks; the terminal record's key set, bounds, and exact- content check; missing/extra/trailing records at every boundary.--Every fixture here is a literal artifact built by hand from the-module's documented wire schema (`coli-ablate/2`) and hashed with the-stdlib `hashlib` directly -- no expected value is produced by calling-the validator under test.-"""-import copy-import hashlib-import json-import pathlib-import subprocess-import sys-import tempfile-import unittest-from unittest import mock--from tools import check_ablate_evidence as ABLATE-from tools import engine_evidence--_DOMAIN = b"coli-ablate-manifest/2\n"---def _serialize(records):- return b"".join(- json.dumps(record, separators=(",", ":")).encode("ascii") + b"\n"- for record in records)---def _write(root, manifest_raw, evidence_raw, config_raw):- manifest = root / "manifest.txt"- evidence = root / "evidence.jsonl"- config = root / "config.json"- manifest.write_bytes(manifest_raw)- evidence.write_bytes(evidence_raw)- config.write_bytes(config_raw)- return manifest, evidence, config---def _run_cli(manifest, evidence, config):- return subprocess.run(- [sys.executable, str(pathlib.Path(ABLATE.__file__)),- str(manifest), str(evidence), "--config", str(config)],- stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)---class _GoldenFixture(unittest.TestCase):- """Shared two-item artifact, hand-derived from the documented schema.-- Manifest: item 1 (T=3, prompt=1, baseline, tokens 0,1,2) then item 2- (T=2, prompt=1, mode 1 with one ablated cell at layer 1/expert 2,- tokens 3,0). vocab=4, n_layers=4, first_dense=1, n_experts=5, so- topk = min(32, 4) = 4. Positions/gold are derived by hand from the- manifest's own tokens: item 1 has positions [0, 1] with gold tokens- 1 and 2; item 2 has position [0] with gold token 0.- """-- MANIFEST = b"1 3 1 0 0 0 1 2\n2 2 1 1 1 1 2 -1 3 0\n"- CONFIG = (b'{"vocab_size":4,"num_hidden_layers":4,'- b'"first_k_dense_replace":1,"n_routed_experts":5}\n')- MANIFEST_SHA256 = hashlib.sha256(_DOMAIN + MANIFEST).hexdigest()- CONFIG_SHA256 = hashlib.sha256(CONFIG).hexdigest()-- @classmethod- def golden_records(cls):- header = {- "t": "hdr", "schema": "coli-ablate/2", "vocab": 4, "topk": 4,- "n_layers": 4, "first_dense": 1, "n_experts": 5,- "config_sha256": cls.CONFIG_SHA256,- "manifest_sha256": cls.MANIFEST_SHA256,- "expected_items": 2, "expected_targets": 3,- }- item1_header = {"t": "ah", "item": 1, "mode": 0, "ncells": 0,- "T": 3, "n_prompt": 1, "cells": []}- row1 = {"t": "lg", "item": 1, "pos": 0, "gold": 1,- "nll": 0.2, "glogit": 1.0, "molo": 0.5, "mgn": 0.5,- "am": 1, "amlogit": 1.0, "logZ": 1.3, "corr": 1,- "tk": [[0, 0.1], [1, 1.0], [2, 0.3], [3, -0.2]]}- row2 = {"t": "lg", "item": 1, "pos": 1, "gold": 2,- "nll": 0.7, "glogit": 0.4, "molo": 0.9, "mgn": -0.5,- "am": 0, "amlogit": 0.9, "logZ": 1.1, "corr": 0,- "tk": [[0, 0.9], [1, 0.1], [2, 0.4], [3, -0.3]]}- item2_header = {"t": "ah", "item": 2, "mode": 1, "ncells": 1,- "T": 2, "n_prompt": 1, "cells": [[1, 2, -1]]}- row3 = {"t": "lg", "item": 2, "pos": 0, "gold": 0,- "nll": 0.0, "glogit": 2.0, "molo": -1e30, "mgn": 1e30,- "am": 0, "amlogit": 2.0, "logZ": 2.0, "corr": 1,- "tk": [[0, 2.0], [1, -1.0], [2, -2.0], [3, -3.0]]}- done = {"t": "done", "manifest_sha256": cls.MANIFEST_SHA256,- "completed_items": 2, "completed_targets": 3}- return [header, item1_header, row1, row2, item2_header, row3, done]-- def _reject(self, mutate, records=None):- """Apply `mutate` to a deep copy of the golden records and assert- the mutated artifact is refused."""- mutated = copy.deepcopy(records if records is not None- else self.golden_records())- mutate(mutated)- with tempfile.TemporaryDirectory() as tmp:- root = pathlib.Path(tmp)- manifest, evidence, config = _write(- root, self.MANIFEST, _serialize(mutated), self.CONFIG)- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE.validate_ablate_evidence(manifest, evidence, config)---class GoldenArtifactAcceptedTests(_GoldenFixture):- def test_valid_artifact_is_accepted_and_pass_line_is_exact(self):- with tempfile.TemporaryDirectory() as tmp:- root = pathlib.Path(tmp)- manifest, evidence, config = _write(- root, self.MANIFEST, _serialize(self.golden_records()),- self.CONFIG)- result = ABLATE.validate_ablate_evidence(- manifest, evidence, config)- self.assertEqual(result, {- "manifest_sha256": self.MANIFEST_SHA256,- "items": 2, "targets": 3,- })- cli = _run_cli(manifest, evidence, config)- self.assertEqual(cli.returncode, 0, cli.stderr.decode(errors="replace"))- self.assertEqual(cli.stderr, b"")- # print() terminates the line with the platform's newline, so a- # Windows child hands back CRLF; the pin is on the line's content.- self.assertEqual(- cli.stdout.replace(b"\r\n", b"\n"),- f"[ablate-evidence] PASS manifest={self.MANIFEST_SHA256} "- f"items=2 targets=3\n".encode("ascii"))---class TopkProducerCapAboveVocabFourTests(unittest.TestCase):- """The header `topk == min(32, vocab)` check only ever exercises the- "vocab is the binding constraint" side at `_GoldenFixture`'s vocab=4.- This fixture uses vocab=40 (above both 4 and the 32 cap) to pin the- other side: topk must be capped at 32, not left equal to vocab.- """-- MANIFEST = b"1 2 1 0 0 0 39\n"- CONFIG = (b'{"vocab_size":40,"num_hidden_layers":1,'- b'"first_k_dense_replace":0,"n_routed_experts":1}\n')- MANIFEST_SHA256 = hashlib.sha256(_DOMAIN + MANIFEST).hexdigest()- CONFIG_SHA256 = hashlib.sha256(CONFIG).hexdigest()-- @classmethod- def golden_records(cls, topk=32, tk_count=32):- header = {- "t": "hdr", "schema": "coli-ablate/2", "vocab": 40, "topk": topk,- "n_layers": 1, "first_dense": 0, "n_experts": 1,- "config_sha256": cls.CONFIG_SHA256,- "manifest_sha256": cls.MANIFEST_SHA256,- "expected_items": 1, "expected_targets": 1,- }- item1_header = {"t": "ah", "item": 1, "mode": 0, "ncells": 0,- "T": 2, "n_prompt": 1, "cells": []}- row = {"t": "lg", "item": 1, "pos": 0, "gold": 39,- "nll": 0.0, "glogit": 1.0, "molo": 0.5, "mgn": 0.5,- "am": 39, "amlogit": 1.0, "logZ": 1.3, "corr": 1,- "tk": [[i, -0.01 * i] for i in range(tk_count)]}- done = {"t": "done", "manifest_sha256": cls.MANIFEST_SHA256,- "completed_items": 1, "completed_targets": 1}- return [header, item1_header, row, done]-- def test_topk_capped_at_32_for_vocab_above_the_cap_is_accepted(self):- with tempfile.TemporaryDirectory() as tmp:- root = pathlib.Path(tmp)- manifest, evidence, config = _write(- root, self.MANIFEST, _serialize(self.golden_records()),- self.CONFIG)- result = ABLATE.validate_ablate_evidence(- manifest, evidence, config)- self.assertEqual(result, {- "manifest_sha256": self.MANIFEST_SHA256,- "items": 1, "targets": 1,- })-- def test_topk_left_uncapped_at_vocab_above_32_is_rejected(self):- # vocab=40 > 32, so header topk must be 32 -- not 40 (== vocab).- records = self.golden_records(topk=40, tk_count=40)- with tempfile.TemporaryDirectory() as tmp:- root = pathlib.Path(tmp)- manifest, evidence, config = _write(- root, self.MANIFEST, _serialize(records), self.CONFIG)- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE.validate_ablate_evidence(manifest, evidence, config)---class EvidenceFramingTests(_GoldenFixture):- """`validate_ablate_evidence`'s canonical-LF-JSONL framing check."""-- def _reject_raw(self, evidence_raw):- with tempfile.TemporaryDirectory() as tmp:- root = pathlib.Path(tmp)- manifest, evidence, config = _write(- root, self.MANIFEST, evidence_raw, self.CONFIG)- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE.validate_ablate_evidence(manifest, evidence, config)-- def test_empty_evidence_rejected(self):- self._reject_raw(b"")-- def test_evidence_missing_trailing_newline_rejected(self):- self._reject_raw(_serialize(self.golden_records())[:-1])-- def test_evidence_with_carriage_return_rejected(self):- self._reject_raw(_serialize(self.golden_records()).replace(- b"\n", b"\r\n", 1))-- def test_evidence_with_nul_byte_rejected(self):- self._reject_raw(_serialize(self.golden_records()) + b"\0")---class HeaderRecordTests(_GoldenFixture):- CASES = (- ("missing_key", lambda r: r[0].pop("topk")),- ("extra_key", lambda r: r[0].__setitem__("extra", 1)),- ("wrong_t", lambda r: r[0].__setitem__("t", "nope")),- ("wrong_schema", lambda r: r[0].__setitem__(- "schema", "coli-ablate/1")),- ("topk_wrong_type", lambda r: r[0].__setitem__("topk", "4")),- ("config_sha256_wrong_type", lambda r: r[0].__setitem__(- "config_sha256", 1)),- ("config_sha256_not_hex", lambda r: r[0].__setitem__(- "config_sha256", "z" * 64)),- ("manifest_sha256_wrong_type", lambda r: r[0].__setitem__(- "manifest_sha256", None)),- ("manifest_sha256_not_hex", lambda r: r[0].__setitem__(- "manifest_sha256", "0" * 63 + "g")),- ("vocab_out_of_range", lambda r: r[0].__setitem__("vocab", 0)),- ("n_layers_out_of_range", lambda r: r[0].__setitem__(- "n_layers", 0)),- ("first_dense_out_of_range", lambda r: r[0].__setitem__(- "first_dense", 99)),- ("n_experts_out_of_range", lambda r: r[0].__setitem__(- "n_experts", 0)),- ("expected_items_out_of_range", lambda r: r[0].__setitem__(- "expected_items", 0)),- ("expected_targets_out_of_range", lambda r: r[0].__setitem__(- "expected_targets", 0)),- ("topk_not_producer_exact", lambda r: r[0].__setitem__("topk", 3)),- ("vocab_identity_mismatch", lambda r: (- r[0].__setitem__("vocab", 5), r[0].__setitem__("topk", 5))),- ("n_layers_identity_mismatch", lambda r: r[0].__setitem__(- "n_layers", 2)),- ("first_dense_identity_mismatch", lambda r: r[0].__setitem__(- "first_dense", 0)),- ("n_experts_identity_mismatch", lambda r: r[0].__setitem__(- "n_experts", 6)),- ("config_sha256_identity_mismatch", lambda r: r[0].__setitem__(- "config_sha256", "0" * 64)),- ("manifest_sha256_binding_mismatch", lambda r: r[0].__setitem__(- "manifest_sha256", "1" * 64)),- ("expected_items_binding_mismatch", lambda r: r[0].__setitem__(- "expected_items", 99)),- ("expected_targets_binding_mismatch", lambda r: r[0].__setitem__(- "expected_targets", 99)),- )-- def test_header_field_checks(self):- for name, mutate in self.CASES:- with self.subTest(name=name):- self._reject(mutate)-- def test_topk_not_producer_exact_even_when_every_row_agrees_with_it(self):- # Isolates the header-level topk==min(32,vocab) check from the- # per-row "len(tk) == header['topk']" shape check: here every row's- # tk list is ALSO shrunk to match the wrong topk, so only the- # header-level producer-exactness check can catch the artifact.- def mutate(records):- records[0]["topk"] = 2- for record in records:- if record.get("t") == "lg":- record["tk"] = record["tk"][:2]- self._reject(mutate)---class ItemHeaderRecordTests(_GoldenFixture):- def test_missing_item_header_rejected(self):- self._reject(lambda r: r.__delitem__(slice(1, None)))-- def test_item_header_not_a_dict_rejected(self):- self._reject(lambda r: r.__setitem__(1, 5))-- def test_item_header_missing_key_rejected(self):- self._reject(lambda r: r[1].pop("ncells"))-- def test_item_header_extra_key_rejected(self):- self._reject(lambda r: r[1].__setitem__("extra", 1))-- def test_item_header_field_wrong_type_rejected(self):- self._reject(lambda r: r[1].__setitem__("item", "1"))-- def test_item_header_cells_not_a_list_rejected(self):- self._reject(lambda r: r[4].__setitem__("cells", {}))-- def test_item_header_cell_wrong_shape_rejected(self):- self._reject(lambda r: r[4].__setitem__("cells", [[1, 2]]))-- def test_item_header_cell_element_wrong_type_rejected(self):- self._reject(lambda r: r[4].__setitem__(- "cells", [[1, 2, "x"]]))-- def test_item_header_mismatch_vs_manifest_rejected(self):- self._reject(lambda r: r[1].__setitem__("T", 99))---class TargetRowRecordTests(_GoldenFixture):- def test_missing_target_row_rejected(self):- self._reject(lambda r: r.__delitem__(slice(2, None)))-- def test_row_not_a_dict_rejected(self):- self._reject(lambda r: r.__setitem__(2, 5))-- def test_row_missing_key_rejected(self):- self._reject(lambda r: r[2].pop("corr"))-- def test_row_extra_key_rejected(self):- self._reject(lambda r: r[2].__setitem__("extra", 1))-- def test_row_wrong_t_rejected(self):- self._reject(lambda r: r[2].__setitem__("t", "nope"))-- def test_row_item_mismatch_rejected(self):- self._reject(lambda r: r[2].__setitem__("item", 99))-- def test_row_pos_mismatch_rejected(self):- self._reject(lambda r: r[2].__setitem__("pos", 5))-- def test_row_gold_mismatch_rejected(self):- self._reject(lambda r: r[2].__setitem__("gold", 0))-- def test_row_am_wrong_type_rejected(self):- self._reject(lambda r: r[2].__setitem__("am", "1"))-- def test_row_am_out_of_range_rejected(self):- self._reject(lambda r: r[2].__setitem__("am", 4))-- def test_row_corr_wrong_type_rejected(self):- self._reject(lambda r: r[2].__setitem__("corr", "1"))-- def test_row_corr_out_of_range_rejected(self):- self._reject(lambda r: r[2].__setitem__("corr", 2))-- NUMERIC_FIELDS = ("nll", "glogit", "molo", "mgn", "amlogit", "logZ")-- def test_row_numeric_field_wrong_type_rejected(self):- for field in self.NUMERIC_FIELDS:- with self.subTest(field=field):- self._reject(lambda r, field=field: r[2].__setitem__(- field, "0"))-- def test_row_tk_not_a_list_rejected(self):- self._reject(lambda r: r[2].__setitem__("tk", 5))-- def test_row_tk_wrong_length_rejected(self):- self._reject(lambda r: r[2].__setitem__(- "tk", r[2]["tk"][:-1]))-- def test_row_tk_pair_wrong_shape_rejected(self):- self._reject(lambda r: r[2]["tk"].__setitem__(0, [0, 0.1, 9]))-- def test_row_tk_pair_id_wrong_type_rejected(self):- self._reject(lambda r: r[2]["tk"].__setitem__(0, ["0", 0.1]))-- def test_row_tk_pair_id_out_of_range_rejected(self):- self._reject(lambda r: r[2]["tk"].__setitem__(0, [4, 0.1]))-- def test_row_tk_pair_val_wrong_type_rejected(self):- self._reject(lambda r: r[2]["tk"].__setitem__(0, [0, "0.1"]))-- def test_row_tk_duplicate_ids_rejected(self):- self._reject(lambda r: r[2]["tk"].__setitem__(0, list(r[2]["tk"][1])))---class CrossFieldInvariantTests(_GoldenFixture):- """Invariants the engine's per-record emitter, `ablate_logit_record`- (and the row-writer `ablate_logit_line` it calls), guarantees for- every row it emits.-- - `nll >= 0`: `nll` is `-target_lp` (`ablate_logit_record`'s own- `gnll=-target_lp`), and `target_lp` is `delta - logse` where- `delta = lo[target] - r.max <= 0` (target's logit minus the row- max) and `logse = log(sum_i exp(lo[i]-max)) >= log(1) = 0` (the- max's own term contributes exp(0)=1 to that sum) -- the row-level- helpers this emitter builds on (`logprob_row_checked`/- `logprob_from_row_checked`). So `target_lp <= 0` always, hence- `nll >= 0` always.- - `corr == (am == gold)`: the emitter passes an `argmax==gold`- comparison directly as the `corr` argument to `ablate_logit_line`,- and `am` is that same argmax -- `corr` is never anything but that- comparison's result.- - `amlogit >= glogit`: `amlogit` is the row's own maximum logit and- `glogit` is one particular entry of that same row, so it can- never exceed the row's own maximum.-- Top-k ordering is deliberately NOT enforced: `tk` is unsorted on the- wire by design (`logit_topk_select`, documented as "deliberately not- a lowest-token-id tie rule").- """-- def test_nll_negative_rejected(self):- self._reject(lambda r: r[2].__setitem__("nll", -0.1))-- def test_nll_zero_accepted(self):- with tempfile.TemporaryDirectory() as tmp:- root = pathlib.Path(tmp)- records = self.golden_records()- records[2]["nll"] = 0.0- manifest, evidence, config = _write(- root, self.MANIFEST, _serialize(records), self.CONFIG)- ABLATE.validate_ablate_evidence(manifest, evidence, config)-- def test_corr_true_when_am_not_gold_rejected(self):- # am=1 == gold=1 in the golden row, so corr must be 1; forcing 0- # while leaving am/gold untouched breaks the agreement.- self._reject(lambda r: r[2].__setitem__("corr", 0))-- def test_corr_false_when_am_equals_gold_rejected(self):- # am=0 != gold=2 in row2 (index 3), so corr must be 0; forcing 1- # breaks the agreement the other way.- self._reject(lambda r: r[3].__setitem__("corr", 1))-- def test_amlogit_below_glogit_rejected(self):- self._reject(lambda r: r[2].__setitem__("amlogit", 0.5))-- def test_amlogit_equal_to_glogit_accepted(self):- # row1 already has amlogit == glogit == 1.0 (am == gold there);- # confirm the boundary itself -- not just values strictly above- # it -- is accepted.- with tempfile.TemporaryDirectory() as tmp:- root = pathlib.Path(tmp)- records = self.golden_records()- self.assertEqual(records[2]["amlogit"], records[2]["glogit"])- manifest, evidence, config = _write(- root, self.MANIFEST, _serialize(records), self.CONFIG)- ABLATE.validate_ablate_evidence(manifest, evidence, config)---class TerminalRecordTests(_GoldenFixture):- def test_missing_terminal_record_rejected(self):- self._reject(lambda r: r.__delitem__(slice(6, None)))-- def test_terminal_not_a_dict_rejected(self):- self._reject(lambda r: r.__setitem__(6, 5))-- def test_terminal_missing_key_rejected(self):- self._reject(lambda r: r[6].pop("completed_items"))-- def test_terminal_extra_key_rejected(self):- self._reject(lambda r: r[6].__setitem__("extra", 1))-- def test_terminal_completed_items_out_of_range_rejected(self):- self._reject(lambda r: r[6].__setitem__("completed_items", 0))-- def test_terminal_completed_targets_out_of_range_rejected(self):- self._reject(lambda r: r[6].__setitem__("completed_targets", 0))-- def test_terminal_wrong_t_rejected(self):- self._reject(lambda r: r[6].__setitem__("t", "nope"))-- def test_terminal_manifest_sha256_mismatch_rejected(self):- self._reject(lambda r: r[6].__setitem__("manifest_sha256", "1" * 64))-- def test_terminal_completed_items_mismatch_rejected(self):- self._reject(lambda r: r[6].__setitem__("completed_items", 1))-- def test_terminal_completed_targets_mismatch_rejected(self):- self._reject(lambda r: r[6].__setitem__("completed_targets", 1))-- def test_trailing_record_after_terminal_rejected(self):- self._reject(lambda r: r.append(dict(r[6])))---class TruncationReplayAndMismatchBiteTests(_GoldenFixture):- """Bite-style table close to the source's own producer-invariant- checks, rebuilt on this module's literal golden fixture instead of- an engine-produced one."""-- def test_named_mutations_all_refuse(self):- cases = (- ("missing_done", lambda r: r.__delitem__(6)),- ("truncated_last_row", lambda r: r.__setitem__(- 5, {"t": "lg", "item": 2})),- ("replayed_item_header", lambda r: r.insert(2, dict(r[1]))),- ("duplicate_done", lambda r: r.append(dict(r[6]))),- ("missing_target_row", lambda r: r.__delitem__(3)),- ("wrong_gold_downstream", lambda r: r[3].__setitem__(- "gold", 0)),- ("header_digest_forged", lambda r: r[0].__setitem__(- "manifest_sha256", "2" * 64)),- )- for name, mutate in cases:- with self.subTest(name=name):- self._reject(mutate)---class ManifestProofFramingTests(unittest.TestCase):- """`_manifest_proof`'s non-canonical-text framing checks."""-- ARGS = (4, 4, 1, 5) # vocab, n_layers, first_dense, n_experts-- def _reject(self, raw):- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE._manifest_proof(raw, *self.ARGS)-- def test_empty_manifest_rejected(self):- self._reject(b"")-- def _accept(self, raw):- return ABLATE._manifest_proof(raw, *self.ARGS)-- # The engine accepts a manifest saved with CRLF endings and one whose last- # line has no terminator, and digests the canonical form of either. A- # checker that refused them would reject files the producer really ran.- def test_manifest_missing_trailing_newline_accepted(self):- self._accept(b"1 3 1 0 0 0 1 2")-- def test_manifest_with_crlf_endings_accepted(self):- self._accept(b"1 3 1 0 0 0 1 2\r\n")-- def test_all_three_framings_bind_the_same_digest(self):- canonical = self._accept(b"1 3 1 0 0 0 1 2\n")["sha256"]- self.assertEqual(self._accept(b"1 3 1 0 0 0 1 2\r\n")["sha256"], canonical)- self.assertEqual(self._accept(b"1 3 1 0 0 0 1 2")["sha256"], canonical)-- def test_carriage_return_inside_a_record_rejected(self):- self._reject(b"1 3 1 0\r 0 0 1 2\n")-- def test_empty_line_inside_a_manifest_rejected(self):- self._reject(b"1 3 1 0 0 0 1 2\n\n2 2 1 0 0 0 1\n")-- def test_manifest_with_nul_byte_rejected(self):- self._reject(b"1 3 1 0 0 0 1 2\n\0")-- def test_manifest_non_ascii_line_rejected(self):- self._reject("1 3 1 0 0 0 1 é\n".encode("utf-8"))-- def test_manifest_control_byte_breaks_grammar_not_framing(self):- # A vertical tab embedded mid-line is not a canonical digit/space- # byte; it must be caught by the integer-grammar check, not- # silently absorbed as a line boundary bytes.splitlines() would- # not treat it as one either way (see the dedicated probe below).- self._reject(b"1 3\x0b1 0 0 0 1 2\n")-- def test_duplicate_item_id_across_lines_rejected(self):- self._reject(b"1 2 1 0 0 0 1\n1 2 1 0 0 0 1\n")-- def test_records_are_split_on_newline_only(self):- # The module now splits the canonical form on b"\n" rather than- # calling splitlines(), so no other byte can become a record- # boundary. bytes.splitlines() would additionally break on \r,- # which the canonical form no longer contains but which a future- # edit could reintroduce; splitting explicitly removes the- # question. These bytes must therefore stay inside one record and- # be caught by the integer-grammar check.- for value in (0x0B, 0x0C, 0x1C, 0x1D, 0x1E):- with self.subTest(byte=hex(value)):- raw = b"1 3 1 0 0 0 1" + bytes([value]) + b"2\n"- self._reject(raw)---class CanonicalManifestDigestTests(unittest.TestCase):- """The canonical rule, pinned against the engine by a literal digest.-- `c/tests/test_ablate_mode.c` asserts the same 64 characters for the same- manifest content. Two implementations that each only agreed with- themselves would both pass their own suites while disagreeing in the- field; a literal known answer on both sides is what rules that out.- """-- RECORD = b"0 3 2 0 0 1 2 3\n"- KNOWN = "c63a48c375b14ca60f26c7e3c5dd36b5929ffaf669a45511c93deee6e8bbd5ed"-- def test_known_answer_matches_the_engine(self):- digest = hashlib.sha256(- ABLATE.DOMAIN + engine_evidence.canonical_manifest_bytes(self.RECORD)- ).hexdigest()- self.assertEqual(digest, self.KNOWN)-- def test_every_accepted_framing_reaches_the_known_answer(self):- for raw in (self.RECORD, b"0 3 2 0 0 1 2 3\r\n", b"0 3 2 0 0 1 2 3"):- with self.subTest(raw=raw):- digest = hashlib.sha256(- ABLATE.DOMAIN- + engine_evidence.canonical_manifest_bytes(raw)).hexdigest()- self.assertEqual(digest, self.KNOWN)-- def test_canonicalization_refuses_what_the_engine_refuses(self):- for raw in (b"", b"\n", b"a\n\nb\n", b"a\rb\n", b"a\0b\n"):- with self.subTest(raw=raw):- with self.assertRaises(engine_evidence.ManifestFormError):- engine_evidence.canonical_manifest_bytes(raw)---class ManifestProofFieldBoundaryTests(unittest.TestCase):- """Per-field/per-cell/per-token bounds `_manifest_proof` enforces.-- Table built from `test_manifest_fixed_width_and_topology_c_python_parity`'s- Python-side expectations (each `expected` value here is the same- literal that method asserted, not something this module computed):- that method also cross-checked each case against a C test binary- this module does not build, so it is not this module's oracle to- carry (flagged separately, not absorbed here).- """-- def test_boundary_table(self):- i32 = ABLATE._INT32_MAX- i64 = ABLATE._INT64_MAX- sixteen = " ".join(f"{layer} 0 -1" for layer in range(1, 17))- cases = (- ("baseline_min", b"0 2 1 0 0 0 1\n", 4, 4, 1, 8, True),- ("fewer_than_five_fields", b"1 2\n", 4, 4, 1, 8, False),- ("item_max", f"{i64} 2 1 0 0 0 1\n".encode(),- 4, 4, 1, 8, True),- ("item_max_plus_1", f"{i64 + 1} 2 1 0 0 0 1\n".encode(),- 4, 4, 1, 8, False),- ("item_min_minus_1", b"-1 2 1 0 0 0 1\n", 4, 4, 1, 8, False),- ("T_min", b"7 2 1 0 0 0 1\n", 4, 4, 1, 8, True),- ("T_below_min", b"7 1 1 0 0 0\n", 4, 4, 1, 8, False),- ("T_max_incomplete", f"7 {i32} 1 0 0\n".encode(),- 4, 4, 1, 8, False),- ("T_max_plus_1", f"7 {i32 + 1} 1 0 0\n".encode(),- 4, 4, 1, 8, False),- ("prompt_max_incomplete", f"7 {i32} {i32 - 1} 0 0\n".encode(),- 4, 4, 1, 8, False),- ("prompt_max_plus_1", f"7 {i32} {i32 + 1} 0 0\n".encode(),- 4, 4, 1, 8, False),- ("prompt_min_minus_1", b"7 2 0 0 0 0 1\n",- 4, 4, 1, 8, False),- ("mode_max", b"7 2 1 3 1 1 2 3 0 1\n",- 4, 4, 1, 8, True),- ("mode_max_plus_1", b"7 2 1 4 1 1 2 -1 0 1\n",- 4, 4, 1, 8, False),- ("cells_max", f"7 2 1 1 16 {sixteen} 0 1\n".encode(),- 4, 17, 1, 8, True),- ("cells_max_plus_1", b"7 2 1 1 17 0 1\n",- 4, 17, 1, 8, False),- ("nonbaseline_zero", b"7 2 1 1 0 0 1\n",- 4, 4, 1, 8, False),- ("dense_layer", b"7 2 1 1 1 0 2 -1 0 1\n",- 4, 4, 1, 8, False),- ("layer_min", b"7 2 1 1 1 0 2 -1 0 1\n",- 4, 4, 0, 8, True),- ("layer_upper", b"7 2 1 1 1 3 2 -1 0 1\n",- 4, 4, 1, 8, True),- ("layer_engine_max", b"7 2 1 1 1 127 2 -1 0 1\n",- 4, 128, 0, 8, True),- ("layer_engine_max_plus_1", b"7 2 1 1 1 128 2 -1 0 1\n",- 4, 128, 0, 8, False),- ("source_upper", b"7 2 1 1 1 1 7 -1 0 1\n",- 4, 4, 1, 8, True),- ("source_min", b"7 2 1 1 1 1 0 -1 0 1\n",- 4, 4, 1, 8, True),- ("source_engine_max", b"7 2 1 1 1 1 4095 -1 0 1\n",- 4, 4, 1, 4096, True),- ("source_engine_max_plus_1", b"7 2 1 1 1 1 4096 -1 0 1\n",- 4, 4, 1, 4096, False),- ("target_upper", b"7 2 1 3 1 1 2 7 0 1\n",- 4, 4, 1, 8, True),- ("target_self_swap", b"7 2 1 3 1 1 2 2 0 1\n",- 4, 4, 1, 8, False),- ("target_signed_min",- f"7 2 1 3 1 1 2 {ABLATE._INT32_MIN} 0 1\n".encode(),- 4, 4, 1, 8, False),- ("target_engine_max", b"7 2 1 3 1 1 0 4095 0 1\n",- 4, 4, 1, 4096, True),- ("target_engine_max_plus_1", b"7 2 1 3 1 1 0 4096 0 1\n",- 4, 4, 1, 4096, False),- ("target_max_plus_1",- f"7 2 1 3 1 1 2 {i32 + 1} 0 1\n".encode(),- 4, 4, 1, 8, False),- ("duplicate_source", b"7 2 1 1 2 1 2 -1 1 2 -1 0 1\n",- 4, 4, 1, 8, False),- ("token_min", b"7 2 1 0 0 0 0\n", 1, 4, 1, 8, True),- ("token_upper", b"7 2 1 0 0 0 16777215\n",- 1 << 24, 4, 1, 8, True),- ("token_max_plus_1", b"7 2 1 0 0 0 16777216\n",- 1 << 24, 4, 1, 8, False),- ("vocab_max", b"7 2 1 0 0 0 1\n",- 1 << 24, 4, 1, 8, True),- ("vocab_max_plus_1", b"7 2 1 0 0 0 1\n",- (1 << 24) + 1, 4, 1, 8, False),- ("leading_zero_rejected", b"07 2 1 0 0 0 1\n",- 4, 4, 1, 8, False),- ("plus_sign_rejected", b"+7 2 1 0 0 0 1\n",- 4, 4, 1, 8, False),- ("double_space_rejected", b"7 2 1 0 0 0 1\n",- 4, 4, 1, 8, False),- ("trailing_space_rejected", b"7 2 1 0 0 0 1 \n",- 4, 4, 1, 8, False),- )- for (name, raw, vocab, layers, first_dense, experts,- expected) in cases:- with self.subTest(name=name):- if expected:- ABLATE._manifest_proof(- raw, vocab, layers, first_dense, experts)- else:- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE._manifest_proof(- raw, vocab, layers, first_dense, experts)---class ConfigIdentityTests(unittest.TestCase):- CONFIG = (b'{"vocab_size":4,"num_hidden_layers":4,'- b'"first_k_dense_replace":1,"n_routed_experts":5}\n')-- def test_engine_text_size_rejects_non_int_length(self):- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE._checked_engine_text_size(True, "config")- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE._checked_engine_text_size(1.0, "config")-- def test_engine_byte_limit_is_inclusive_and_enforced_both_sides(self):- engine_limit = 256 << 20- self.assertEqual(ABLATE._ENGINE_TEXT_MAX_BYTES, engine_limit)- self.assertEqual(- ABLATE._checked_engine_text_size(engine_limit, "config"),- engine_limit)- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE._checked_engine_text_size(engine_limit + 1, "config")-- with tempfile.TemporaryDirectory() as tmp:- root = pathlib.Path(tmp)- ablate_config = root / "ablate-config.json"- ablate_config.write_bytes(self.CONFIG)- with mock.patch.object(- ABLATE, "_ENGINE_TEXT_MAX_BYTES", len(self.CONFIG)):- identity = ABLATE._config_identity(ablate_config)- self.assertEqual(identity["vocab"], 4)- self.assertEqual(- identity["config_sha256"],- hashlib.sha256(self.CONFIG).hexdigest())- ablate_config.write_bytes(self.CONFIG + b" ")- with self.assertRaisesRegex(- ABLATE.AblateEvidenceError, "256 MiB"):- ABLATE._config_identity(ablate_config)-- def _reject(self, raw):- with tempfile.TemporaryDirectory() as tmp:- path = pathlib.Path(tmp) / "config.json"- path.write_bytes(raw)- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE._config_identity(path)-- def test_empty_config_rejected(self):- self._reject(b"")-- def test_invalid_json_config_rejected(self):- self._reject(b"{not json}\n")-- def test_non_object_root_rejected(self):- self._reject(b"[1,2,3]\n")-- def test_vocab_size_missing_rejected(self):- self._reject(b'{"num_hidden_layers":4,"first_k_dense_replace":1,'- b'"n_routed_experts":5}\n')-- def test_vocab_size_out_of_range_rejected(self):- self._reject(b'{"vocab_size":0,"num_hidden_layers":4,'- b'"first_k_dense_replace":1,"n_routed_experts":5}\n')-- def test_num_hidden_layers_out_of_range_rejected(self):- self._reject(b'{"vocab_size":4,"num_hidden_layers":0,'- b'"first_k_dense_replace":1,"n_routed_experts":5}\n')-- def test_n_routed_experts_out_of_range_rejected(self):- self._reject(b'{"vocab_size":4,"num_hidden_layers":4,'- b'"first_k_dense_replace":1,"n_routed_experts":0}\n')-- def test_first_k_dense_replace_out_of_range_rejected(self):- self._reject(b'{"vocab_size":4,"num_hidden_layers":4,'- b'"first_k_dense_replace":5,"n_routed_experts":5}\n')---class JsonHelperTests(unittest.TestCase):- def test_duplicate_json_key_rejected(self):- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE._json_record(b'{"a":1,"a":2}', "record 1")-- def test_nan_constant_rejected(self):- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE._json_record(b'{"nll":NaN}', "record 1")-- def test_infinity_constant_rejected(self):- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE._json_record(b'{"nll":Infinity}', "record 1")-- def test_negative_infinity_constant_rejected(self):- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE._json_record(b'{"nll":-Infinity}', "record 1")-- def test_non_ascii_bytes_rejected(self):- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE._json_record(b"\xff", "record 1")-- def test_malformed_json_rejected(self):- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE._json_record(b"{not json}", "record 1")---class FixedWidthHelperTests(unittest.TestCase):- def test_bounded_int_rejects_bool_disguised_as_int(self):- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE._bounded_int(True, "x", 1, 10)-- def test_int64_max_is_the_literal_signed_64_bit_bound(self):- # Pinned by literal, not derived, so a future refactor of the- # module's own (1 << 63) - 1 expression cannot silently drift.- self.assertEqual(ABLATE._INT64_MAX, 9223372036854775807)-- def test_fixed_width_helpers_and_derived_count_boundaries(self):- for value in (ABLATE._INT64_MIN, ABLATE._INT64_MAX):- self.assertEqual(ABLATE._manifest_i64(str(value), 1), value)- for value in (ABLATE._INT64_MIN - 1, ABLATE._INT64_MAX + 1):- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE._manifest_i64(str(value), 1)- for value in (ABLATE._INT32_MIN, ABLATE._INT32_MAX):- self.assertEqual(ABLATE._bounded_int(- value, "int32", ABLATE._INT32_MIN, ABLATE._INT32_MAX), value)- for value in (ABLATE._INT32_MIN - 1, ABLATE._INT32_MAX + 1):- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE._bounded_int(- value, "int32", ABLATE._INT32_MIN, ABLATE._INT32_MAX)- self.assertEqual(- ABLATE._count_add(0, ABLATE._INT64_MAX, "count"),- ABLATE._INT64_MAX)- self.assertEqual(- ABLATE._count_add(ABLATE._INT64_MAX, 0, "count"),- ABLATE._INT64_MAX)- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE._count_add(ABLATE._INT64_MAX, 1, "count")- for label in ("expected_items", "expected_targets",- "completed_items", "completed_targets"):- self.assertEqual(- ABLATE._bounded_int(1, label, 1, ABLATE._INT64_MAX), 1)- self.assertEqual(ABLATE._bounded_int(- ABLATE._INT64_MAX, label, 1, ABLATE._INT64_MAX),- ABLATE._INT64_MAX)- for value in (0, ABLATE._INT64_MAX + 1):- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE._bounded_int(value, label, 1, ABLATE._INT64_MAX)-- def test_retained_long_max_plus_one_artifact_is_incomplete(self):- manifest_raw = b"9223372036854775808 2 1 0 0 0 1\n"- digest = hashlib.sha256(_DOMAIN + manifest_raw).hexdigest()- self.assertEqual(- digest,- "988a1cf2ddc812f38138e51eecfebb2ba0c9980e31b4c7183396716f114d6538")- with tempfile.TemporaryDirectory() as tmp:- root = pathlib.Path(tmp)- config_raw = (b'{"vocab_size":2,"num_hidden_layers":4,'- b'"first_k_dense_replace":1,"n_routed_experts":8}\n')- records = (- {"t": "hdr", "schema": "coli-ablate/2", "vocab": 2,- "topk": 2, "n_layers": 4, "first_dense": 1,- "n_experts": 8,- "config_sha256": hashlib.sha256(config_raw).hexdigest(),- "manifest_sha256": digest,- "expected_items": 1, "expected_targets": 1},- {"t": "ah", "item": 9223372036854775808, "mode": 0,- "ncells": 0, "T": 2, "n_prompt": 1, "cells": []},- {"t": "lg", "item": 9223372036854775808, "pos": 0,- "gold": 1, "nll": 0, "glogit": 0, "molo": 0,- "mgn": 0, "am": 1, "amlogit": 0, "logZ": 0,- "corr": 1, "tk": [[1, 0], [0, -1]]},- {"t": "done", "manifest_sha256": digest,- "completed_items": 1, "completed_targets": 1},- )- manifest, evidence, config = _write(- root, manifest_raw, _serialize(records), config_raw)- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE.validate_ablate_evidence(manifest, evidence, config)- cli = _run_cli(manifest, evidence, config)- self.assertNotEqual(cli.returncode, 0)- self.assertIn(b"[ablate-evidence] INCOMPLETE:", cli.stderr)---if __name__ == "__main__":- unittest.main()diff --git a/c/tests/test_eval_glm.py b/c/tests/test_eval_glm.pyindex fd153199..eef6e766 100644--- a/c/tests/test_eval_glm.py+++ b/c/tests/test_eval_glm.py@@ -26,8 +26,7 @@ that performs them:is exactly the defect dev's plain ``line[0] in "-0123456789"`` filterdoes not catch (differential bite, below).- `score_request_wire`: strict ASCII/LF request grammar, the per-record- SHA-256 digest, and the inclusive 256 MiB engine text limit shared with- `check_ablate_evidence.py`.+ SHA-256 digest, and the inclusive 256 MiB engine text limit.- `completion_error`: the exact zero-exit/complete-count/positive-tokendenominator that alone passes.- `main`: incremental durability (one written+flushed row per completed@@ -42,9 +41,6 @@ Deferred (need a live binary this module does not have access to):- `test_c_emitted_c17g_corpus_is_canonical`, which drives``test_logprob_wire --score-c17g-fixture`` (a binary produced by adifferent part of this project's build, not present here).-- the ABLATE-block stdout probes (`c_manifest_accepts`/- `test_complete_production_fixture_is_strict_json`-style checks): out of- scope for this module (owned by `test_check_ablate_evidence.py`).No model is run by the committed tests -- every case here drives`eval_glm.py` against an injected stand-in for the direct engine launch@@ -75,18 +71,11 @@ _spec = importlib.util.spec_from_file_location(EVAL = importlib.util.module_from_spec(_spec)_spec.loader.exec_module(EVAL)-_ablate_spec = importlib.util.spec_from_file_location(- "check_ablate_evidence_under_test", TOOLS / "check_ablate_evidence.py")-ABLATE = importlib.util.module_from_spec(_ablate_spec)-_ablate_spec.loader.exec_module(ABLATE)-class EvalGlmEvidenceTests(unittest.TestCase):BANNER = ("== GLM C engine (glm_moe_dsa), cache=64 experts/layer | ""compute experts@4-bit dense@8-bit | idot: neon-i8mm ==")- CONFIG = (b'{"vocab_size":4,"num_hidden_layers":4,'- b'"first_k_dense_replace":1,"n_routed_experts":5}\n')@staticmethoddef loaded(state="ACTIVE", draft=1, layers=78, experts=256,@@ -759,39 +748,16 @@ class EvalGlmEvidenceTests(unittest.TestCase):self.assertIn("MEAN acc_norm", self.last_stdout)def test_python_engine_byte_limits_are_inclusive_and_preallocation(self):- # The 256 MiB inclusive engine-text limit is shared between- # this module and check_ablate_evidence.py; checked on both- # sides of the shared constant/helper.engine_limit = 256 << 20- self.assertEqual(ABLATE._ENGINE_TEXT_MAX_BYTES, engine_limit)self.assertEqual(EVAL._ENGINE_TEXT_MAX_BYTES, engine_limit)- self.assertEqual(- ABLATE._checked_engine_text_size(engine_limit, "config"),- engine_limit)self.assertEqual(EVAL._checked_engine_text_size(engine_limit, "SCORE"),engine_limit)- with self.assertRaises(ABLATE.AblateEvidenceError):- ABLATE._checked_engine_text_size(engine_limit + 1, "config")with self.assertRaises(EVAL.EvidenceError):EVAL._checked_engine_text_size(engine_limit + 1, "SCORE")with tempfile.TemporaryDirectory() as tmp:root = pathlib.Path(tmp)- ablate_config = root / "ablate-config.json"- ablate_config.write_bytes(self.CONFIG)- with mock.patch.object(- ABLATE, "_ENGINE_TEXT_MAX_BYTES", len(self.CONFIG)):- identity = ABLATE._config_identity(ablate_config)- self.assertEqual(identity["vocab"], 4)- self.assertEqual(- identity["config_sha256"],- hashlib.sha256(self.CONFIG).hexdigest())- ablate_config.write_bytes(self.CONFIG + b" ")- with self.assertRaisesRegex(- ABLATE.AblateEvidenceError, "256 MiB"):- ABLATE._config_identity(ablate_config)-score_config = root / "config.json"score_raw = b'{"vocab_size":4}\n'score_config.write_bytes(score_raw)diff --git a/c/tests/test_pack_python.py b/c/tests/test_pack_python.pyindex e2f0375a..fee5a4fd 100644--- a/c/tests/test_pack_python.py+++ b/c/tests/test_pack_python.py@@ -259,42 +259,5 @@ class PackPythonDataFileTests(unittest.TestCase):self.assertIn("1 Python files and 1 data files", buf.getvalue())-class HumanOnlyToolsShipExplicitly(unittest.TestCase):- """Tools that only a person invokes are invisible to the import walk.-- `pack_python.py` computes the archive from what `coli` reaches, imports and- subprocesses alike. A script whose caller is a person typing argparse- arguments is reached by nobody in the code, so static analysis cannot put- it in the archive -- `k3_tokenizer.py` has always been in that position and- `release.yml` copies it by hand. `check_ablate_evidence.py` joined it: it- verifies the ABLATE evidence that the packaged `eval_glm.py` produces, so a- release that ships the producer without the verifier is half a tool.-- This pins both halves of that decision on the real tree: the walk does NOT- reach the file (if it ever does, the explicit copy became redundant and this- test should be retired), and the workflow copies AND gates it, the way it- does for the tokenizer helper. A copy without a gate is how v1.10.0 shipped- four broken commands green (#1296)."""-- HUMAN_ONLY = ("k3_tokenizer.py", "check_ablate_evidence.py")-- def test_the_walk_does_not_reach_them(self):- paths = PACK.needed(HERE.parent)- for name in self.HUMAN_ONLY:- self.assertTrue((TOOLS / name).is_file(), name)- self.assertNotIn(TOOLS / name, paths,- f"{name} is now reached by the walk; drop its "- f"explicit copy from release.yml and this pin")-- def test_release_copies_and_gates_them(self):- release = (HERE.parent.parent / ".github" / "workflows"- / "release.yml").read_text(encoding="utf-8")- for name in self.HUMAN_ONLY:- self.assertIn(f"cp c/tools/{name} dist/tools/", release,- f"release.yml no longer copies {name}")- self.assertIn(f"test -f tools/{name} ||", release,- f"release.yml copies {name} but does not gate on it")--if __name__ == "__main__":unittest.main()diff --git a/c/tools/check_ablate_evidence.py b/c/tools/check_ablate_evidence.pydeleted file mode 100644index 0f8486ed..00000000--- a/c/tools/check_ablate_evidence.py+++ /dev/null@@ -1,403 +0,0 @@-#!/usr/bin/env python3-"""Validate one complete ABLATE evidence artifact against a config.json."""--import argparse-import hashlib-import json-import math-import pathlib-import re-import sys--# The canonical-manifest rule is shared with the engine and with the other-# evidence checkers, so it lives in one module. Imported both ways because this-# file is run as a script from the engine directory and imported as part of the-# tools package by the tests.-try:- from tools.engine_evidence import (- ManifestFormError, canonical_manifest_bytes)-except ImportError: # run directly: tools/ is on the path- from engine_evidence import ManifestFormError, canonical_manifest_bytes---DOMAIN = b"coli-ablate-manifest/2\n"-_INT = re.compile(r"(?:0|[1-9][0-9]*|-[1-9][0-9]*)")-_SHA256 = re.compile(r"[0-9a-f]{64}")--# The wire schema (``coli-ablate/2``) is a fixed-width LP64 domain, chosen so-# a producer and a checker on different host ABIs agree byte-for-byte: every-# manifest integer and every completion counter is signed 64-bit, while a-# value the engine narrows to C ``int`` (a layer, an expert, a token id) is-# signed 32-bit.-_INT32_MIN = -(1 << 31)-_INT32_MAX = (1 << 31) - 1-_INT64_MIN = -(1 << 63)-_INT64_MAX = (1 << 63) - 1-_ENGINE_VOCAB_MAX = 1 << 24-_ENGINE_LAYERS_MAX = 128-_ENGINE_EXPERTS_MAX = 4096-_ENGINE_TEXT_MAX_BYTES = 256 << 20---class AblateEvidenceError(ValueError):- """The artifact cannot prove a complete ABLATE denominator."""---def _checked_engine_text_size(length, label):- if type(length) is not int or not 0 <= length <= _ENGINE_TEXT_MAX_BYTES:- raise AblateEvidenceError(- f"{label} exceeds the inclusive 256 MiB engine limit")- return length---def _reject_constant(value):- raise AblateEvidenceError(f"non-JSON constant: {value}")---def _reject_duplicate_keys(pairs):- result = {}- for key, value in pairs:- if key in result:- raise AblateEvidenceError(f"duplicate JSON key: {key}")- result[key] = value- return result---def _json_record(raw, label):- try:- return json.loads(- raw.decode("ascii"), parse_constant=_reject_constant,- object_pairs_hook=_reject_duplicate_keys)- except (UnicodeDecodeError, json.JSONDecodeError,- AblateEvidenceError) as exc:- raise AblateEvidenceError(f"invalid {label} JSON: {exc}") from exc---def _bounded_config_bytes(config_path):- path = pathlib.Path(config_path)- with path.open("rb") as source:- source.seek(0, 2)- length = source.tell()- _checked_engine_text_size(length, "external config.json")- source.seek(0)- raw = source.read(_ENGINE_TEXT_MAX_BYTES + 1)- _checked_engine_text_size(len(raw), "external config.json")- return raw---def _config_identity(config_path):- raw = _bounded_config_bytes(config_path)- if not raw:- raise AblateEvidenceError("external config.json is empty")- try:- config = json.loads(- raw.decode("utf-8"), parse_constant=_reject_constant,- object_pairs_hook=_reject_duplicate_keys)- except (UnicodeDecodeError, json.JSONDecodeError,- AblateEvidenceError) as exc:- raise AblateEvidenceError(f"external config.json is invalid: {exc}") from exc- if not isinstance(config, dict):- raise AblateEvidenceError("external config.json root is not an object")- try:- identity = {- "vocab": _bounded_int(- config["vocab_size"], "config vocab_size",- 1, _ENGINE_VOCAB_MAX),- "n_layers": _bounded_int(- config["num_hidden_layers"], "config num_hidden_layers",- 1, _ENGINE_LAYERS_MAX),- "n_experts": _bounded_int(- config["n_routed_experts"], "config n_routed_experts",- 1, _ENGINE_EXPERTS_MAX),- }- identity["first_dense"] = _bounded_int(- config["first_k_dense_replace"],- "config first_k_dense_replace", 0, identity["n_layers"])- except (KeyError, AblateEvidenceError) as exc:- raise AblateEvidenceError(- "external config.json topology is incomplete or invalid") from exc- identity["config_sha256"] = hashlib.sha256(raw).hexdigest()- return identity---def _bounded_int(value, label, minimum, maximum):- if type(value) is not int or not minimum <= value <= maximum:- raise AblateEvidenceError(- f"{label} is outside {minimum}..{maximum}")- return value---def _manifest_i64(text, line_number):- try:- value = int(text)- except ValueError as exc:- raise AblateEvidenceError(- f"manifest line {line_number} integer is too large") from exc- return _bounded_int(- value, f"manifest line {line_number} integer",- _INT64_MIN, _INT64_MAX)---def _count_add(current, increment, label):- _bounded_int(current, label, 0, _INT64_MAX)- _bounded_int(increment, f"{label} increment", 0, _INT64_MAX)- if increment > _INT64_MAX - current:- raise AblateEvidenceError(f"{label} exceeds signed 64-bit domain")- return current + increment---def _manifest_proof(raw, vocab, n_layers, first_dense, n_experts):- # The engine accepts a manifest saved with either line ending, and with or- # without a terminator on its last line, and digests the canonical form- # rather than the bytes on disk. Reproduce that here from the shared rule,- # or this checker would reject a file the producer ran and would compute a- # different digest for one it accepted.- try:- raw = canonical_manifest_bytes(raw)- except ManifestFormError as exc:- raise AblateEvidenceError(f"manifest is not canonical text: {exc}") from exc- _bounded_int(vocab, "external vocabulary", 1, _ENGINE_VOCAB_MAX)- _bounded_int(n_layers, "external n_layers", 1, _ENGINE_LAYERS_MAX)- _bounded_int(first_dense, "header first_dense", 0, n_layers)- _bounded_int(n_experts, "external n_experts", 1, _ENGINE_EXPERTS_MAX)- items = []- seen = set()- item_count = targets = 0- for line_number, raw_line in enumerate(raw[:-1].split(b"\n"), 1):- try:- text = raw_line.decode("ascii")- except UnicodeDecodeError as exc:- raise AblateEvidenceError(- f"manifest line {line_number} is not ASCII") from exc- parts = text.split(" ")- if (not parts or any(not _INT.fullmatch(part) for part in parts) or- " ".join(parts) != text):- raise AblateEvidenceError(- f"manifest line {line_number} is not canonical integer grammar")- values = [_manifest_i64(part, line_number) for part in parts]- if len(values) < 5:- raise AblateEvidenceError(f"manifest line {line_number} is truncated")- item, length, prompt, mode, cells = values[:5]- if (item < 0 or item in seen or- not 2 <= length <= _INT32_MAX or prompt < 1 or- prompt >= length or mode not in range(4) or- cells not in range(17) or- (mode == 0 and cells != 0) or- (mode != 0 and cells == 0)):- raise AblateEvidenceError(- f"manifest line {line_number} has invalid fields/denominator")- expected = 5 + 3 * cells + length- if len(values) != expected:- raise AblateEvidenceError(- f"manifest line {line_number} has invalid fields/denominator")- triples = []- source_cells = set()- cursor = 5- for _ in range(cells):- layer, expert, applied = values[cursor:cursor + 3]- cursor += 3- if (not first_dense <= layer < n_layers or- not 0 <= expert < n_experts or- not _INT32_MIN <= applied <= _INT32_MAX or- (mode == 3 and- (not 0 <= applied < n_experts or applied == expert)) or- (mode != 3 and applied != -1) or- (layer, expert) in source_cells):- raise AblateEvidenceError(- f"manifest line {line_number} has invalid cell")- source_cells.add((layer, expert))- triples.append([layer, expert, applied])- tokens = values[cursor:]- if any(token < 0 or token >= vocab for token in tokens):- raise AblateEvidenceError(- f"manifest line {line_number} has out-of-vocabulary token")- seen.add(item)- positions = range(prompt - 1, length - 1)- row_targets = length - prompt- item_count = _count_add(item_count, 1, "manifest item count")- targets = _count_add(targets, row_targets, "manifest target count")- items.append({- "item": item, "T": length, "n_prompt": prompt,- "mode": mode, "ncells": cells, "cells": triples,- "positions": positions, "tokens": tuple(tokens),- })- if item_count <= 0 or targets <= 0:- raise AblateEvidenceError("manifest denominator is not positive")- return {- "sha256": hashlib.sha256(DOMAIN + raw).hexdigest(),- "items": tuple(items), "item_count": item_count, "targets": targets,- }---def validate_ablate_evidence(manifest_path, evidence_path, config_path):- identity = _config_identity(config_path)- manifest_raw = pathlib.Path(manifest_path).read_bytes()- evidence_raw = pathlib.Path(evidence_path).read_bytes()- if (not evidence_raw or not evidence_raw.endswith(b"\n") or- b"\r" in evidence_raw or b"\0" in evidence_raw):- raise AblateEvidenceError("evidence is not nonempty canonical LF JSONL")- records = [_json_record(line, f"record {index}")- for index, line in enumerate(evidence_raw.splitlines(), 1)]- if not records:- raise AblateEvidenceError("evidence has no header")- header = records[0]- if (not isinstance(header, dict) or set(header) != {- "t", "schema", "vocab", "topk", "manifest_sha256",- "n_layers", "first_dense", "n_experts",- "config_sha256",- "expected_items", "expected_targets"} or- header.get("t") != "hdr" or header.get("schema") != "coli-ablate/2" or- type(header.get("topk")) is not int or- not isinstance(header.get("config_sha256"), str) or- not _SHA256.fullmatch(header["config_sha256"]) or- not isinstance(header.get("manifest_sha256"), str) or- not _SHA256.fullmatch(header["manifest_sha256"])):- raise AblateEvidenceError("header keys or values are not exact")- try:- _bounded_int(header["vocab"], "header vocabulary",- 1, _ENGINE_VOCAB_MAX)- if header["topk"] != min(32, header["vocab"]):- raise AblateEvidenceError("header topk is not producer-exact")- _bounded_int(header["n_layers"], "header n_layers",- 1, _ENGINE_LAYERS_MAX)- _bounded_int(header["first_dense"], "header first_dense",- 0, header["n_layers"])- _bounded_int(header["n_experts"], "header n_experts",- 1, _ENGINE_EXPERTS_MAX)- _bounded_int(header["expected_items"], "header expected_items",- 1, _INT64_MAX)- _bounded_int(header["expected_targets"], "header expected_targets",- 1, _INT64_MAX)- except (KeyError, AblateEvidenceError) as exc:- raise AblateEvidenceError(- f"header keys or values are not exact: {exc}") from exc- if any(header[key] != identity[key] for key in (- "vocab", "n_layers", "first_dense", "n_experts",- "config_sha256")):- raise AblateEvidenceError(- "header does not match the external config identity")- proof = _manifest_proof(- manifest_raw, identity["vocab"], identity["n_layers"],- identity["first_dense"], identity["n_experts"])- if (header["manifest_sha256"] != proof["sha256"] or- header["expected_items"] != proof["item_count"] or- header["expected_targets"] != proof["targets"]):- raise AblateEvidenceError("header does not bind the source manifest proof")-- cursor = 1- completed_items = completed_targets = 0- logit_keys = {- "t", "item", "pos", "gold", "nll", "glogit", "molo", "mgn",- "am", "amlogit", "logZ", "corr", "tk",- }- for expected in proof["items"]:- if cursor >= len(records):- raise AblateEvidenceError("missing item header")- item_header = records[cursor]- cursor += 1- if (not isinstance(item_header, dict) or set(item_header) != {- "t", "item", "mode", "ncells", "T", "n_prompt", "cells"} or- any(type(item_header.get(key)) is not int for key in (- "item", "mode", "ncells", "T", "n_prompt")) or- not isinstance(item_header.get("cells"), list) or- any(not isinstance(cell, list) or len(cell) != 3 or- any(type(value) is not int for value in cell)- for cell in item_header["cells"]) or- item_header != {key: expected[key] for key in (- "item", "mode", "ncells", "T", "n_prompt", "cells")} |- {"t": "ah"}):- raise AblateEvidenceError("item header does not match manifest order")- for position in expected["positions"]:- if cursor >= len(records):- raise AblateEvidenceError("missing target row")- row = records[cursor]- cursor += 1- if (not isinstance(row, dict) or set(row) != logit_keys or- row.get("t") != "lg" or type(row.get("item")) is not int or- row["item"] != expected["item"] or- type(row.get("pos")) is not int or row["pos"] != position or- type(row.get("gold")) is not int or- row["gold"] != expected["tokens"][position + 1] or- type(row.get("am")) is not int or- row["am"] not in range(header["vocab"]) or- type(row.get("corr")) is not int or row["corr"] not in (0, 1)):- raise AblateEvidenceError("target row identity/fields are invalid")- for field in ("nll", "glogit", "molo", "mgn", "amlogit", "logZ"):- if (type(row.get(field)) not in (int, float) or- not math.isfinite(row[field])):- raise AblateEvidenceError(f"target row {field} is nonfinite")- # These three hold for every row the producer can emit: nll is a- # negated log-probability (always <= 0 before negation); corr is- # defined as the argmax/gold agreement, not sampled separately;- # and amlogit is the row's own max logit, so no field can exceed- # it -- least of all the gold token's own logit.- if row["nll"] < 0:- raise AblateEvidenceError("target row nll is negative")- if row["corr"] != int(row["am"] == row["gold"]):- raise AblateEvidenceError(- "target row corr does not match its am/gold agreement")- if row["amlogit"] < row["glogit"]:- raise AblateEvidenceError(- "target row amlogit is below glogit")- topk = row.get("tk")- if (not isinstance(topk, list) or len(topk) != header["topk"] or- any(not isinstance(pair, list) or len(pair) != 2 or- type(pair[0]) is not int or not 0 <= pair[0] < header["vocab"] or- type(pair[1]) not in (int, float) or not math.isfinite(pair[1])- for pair in topk) or- len({pair[0] for pair in topk}) != len(topk)):- raise AblateEvidenceError("target row top-k is invalid")- completed_targets = _count_add(- completed_targets, 1, "completed target count")- completed_items = _count_add(- completed_items, 1, "completed item count")-- if cursor >= len(records):- raise AblateEvidenceError("missing terminal completion record")- done = records[cursor]- cursor += 1- if (not isinstance(done, dict) or set(done) != {- "t", "manifest_sha256", "completed_items", "completed_targets"}):- raise AblateEvidenceError("terminal completion proof is invalid")- try:- _bounded_int(done["completed_items"], "done completed_items",- 1, _INT64_MAX)- _bounded_int(done["completed_targets"], "done completed_targets",- 1, _INT64_MAX)- except (KeyError, AblateEvidenceError) as exc:- raise AblateEvidenceError("terminal completion proof is invalid") from exc- if done != {"t": "done", "manifest_sha256": proof["sha256"],- "completed_items": completed_items,- "completed_targets": completed_targets}:- raise AblateEvidenceError("terminal completion proof is invalid")- if cursor != len(records):- raise AblateEvidenceError("records follow terminal completion proof")- if (completed_items != proof["item_count"] or- completed_targets != proof["targets"]):- raise AblateEvidenceError("completed denominator does not match manifest")- return {- "manifest_sha256": proof["sha256"], "items": completed_items,- "targets": completed_targets,- }---def main(argv=None):- parser = argparse.ArgumentParser(description=__doc__)- parser.add_argument("manifest")- parser.add_argument("evidence")- parser.add_argument("--config", required=True,- help="independently supplied loaded-model config.json")- args = parser.parse_args(argv)- try:- result = validate_ablate_evidence(- args.manifest, args.evidence, args.config)- except (OSError, AblateEvidenceError) as exc:- print(f"[ablate-evidence] INCOMPLETE: {exc}", file=sys.stderr)- return 1- print(f"[ablate-evidence] PASS manifest={result['manifest_sha256']} "- f"items={result['items']} targets={result['targets']}")- return 0---if __name__ == "__main__":- sys.exit(main())