Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

move ablate-checker material out of this PR

monotophicfe50907a54db
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 9d2558bc..1ef3d3af 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -136,11 +136,6 @@ jobs:
# run it -- the second half of #720.
mkdir -p dist/tools
cp c/tools/k3_tokenizer.py dist/tools/
- # check_ablate_evidence.py verifies the ABLATE evidence that eval_glm.py
- # (packaged above) produces. Nothing in the code names it -- its caller is a
- # person with argparse arguments -- so pack_python's import walk cannot reach
- # it, exactly like k3_tokenizer.py. Copied explicitly for the same reason.
- cp c/tools/check_ablate_evidence.py dist/tools/
cp c/coli dist/
# Windows has nothing to click otherwise: `coli` is an extensionless
# Python script, and the .exe files are engines that exit at once
@@ -195,9 +190,6 @@ jobs:
# Kimi K3 needs tokenizer.json synthesized from tiktoken.model; without this
# script in the archive the engine ships but cannot be driven by coli chat.
test -f tools/k3_tokenizer.py || { echo "FAIL: k3_tokenizer.py missing from archive"; exit 1; }
- # Same position as k3_tokenizer.py: reachable only by a person, so only an
- # explicit gate can prove it shipped.
- test -f tools/check_ablate_evidence.py || { echo "FAIL: check_ablate_evidence.py missing from archive"; exit 1; }
# E tutto cio' che coli raggiunge, import e sottoprocessi. Il controllo
# qui sopra guarda l'unico file che il passo di copia aveva appena
# scritto: conferma se stesso, ed e' il motivo per cui la v1.10.0 e'
@@ -210,8 +202,6 @@ jobs:
# fine. The check was wrong, not the artifact.
python3 -c "import ast; ast.parse(open('tools/k3_tokenizer.py', encoding='utf-8').read())" \
|| { echo "FAIL: packaged k3_tokenizer.py does not parse"; exit 1; }
- python3 -c "import ast; ast.parse(open('tools/check_ablate_evidence.py', encoding='utf-8').read())" \
- || { echo "FAIL: packaged check_ablate_evidence.py does not parse"; exit 1; }
COLI_EXPECT="$SIBLINGS" python3 - <<'PYCHK'
import importlib.machinery, importlib.util, json, os, sys, tempfile
from family_registry import all_families
diff --git a/c/tests/test_check_ablate_evidence.py b/c/tests/test_check_ablate_evidence.py
deleted file mode 100644
index 906cd552..00000000
--- a/c/tests/test_check_ablate_evidence.py
+++ /dev/null
@@ -1,915 +0,0 @@
-"""tools/check_ablate_evidence.py must accept only a complete, self-binding
-ABLATE evidence artifact and reject every other input: a truncated or
-replayed record stream, a header that does not bind the manifest or the
-external config.json it is checked against, any field with the wrong
-type, range, or key set at any of the four record kinds (header, item
-header, target row, terminal completion), and a target row whose fields
-contradict one another in a way the producer can never emit.
-
-Checks enumerated from the source (`tools/check_ablate_evidence.py`,
-read in full before writing this module) and covered below, grouped by
-the function that performs them:
-
-- `_checked_engine_text_size` / `_bounded_config_bytes`: the 256 MiB
- inclusive engine text limit, both sides.
-- `_reject_constant`, `_reject_duplicate_keys`, `_json_record`: no
- NaN/Infinity JSON constants, no duplicate object keys, invalid
- JSON/non-ASCII text rejected.
-- `_config_identity`: empty file; invalid JSON; non-object root; each
- of vocab_size/num_hidden_layers/n_routed_experts/first_k_dense_replace
- missing or out of range.
-- `_manifest_proof`: framing -- an empty manifest, an empty record, a
- carriage return inside a record and an embedded NUL are refused, while
- CRLF endings and a missing final newline are accepted and reduced to
- the canonical form the engine binds; non-ASCII line; non-canonical integer grammar; too few fields;
- every per-item field bound (item id, T, prompt, mode, cell count);
- the mode/cell-count pairing rule; the field-count/denominator
- arithmetic; every per-cell bound (layer, expert, applied-target,
- mode-3 vs other-mode applied-target rule, duplicate cell);
- out-of-vocabulary tokens; duplicate item ids across lines.
-- `validate_ablate_evidence`: the evidence framing check; the header's
- key set, type, and range checks; the header-vs-config identity
- check; the header-vs-manifest-proof binding check; the item header's
- key set, type, and manifest-order check; the target row's key set,
- type, and identity checks; the three cross-field invariants (below);
- the top-k list's shape, range, and
- uniqueness checks; the terminal record's key set, bounds, and exact
- content check; missing/extra/trailing records at every boundary.
-
-Every fixture here is a literal artifact built by hand from the
-module's documented wire schema (`coli-ablate/2`) and hashed with the
-stdlib `hashlib` directly -- no expected value is produced by calling
-the validator under test.
-"""
-import copy
-import hashlib
-import json
-import pathlib
-import subprocess
-import sys
-import tempfile
-import unittest
-from unittest import mock
-
-from tools import check_ablate_evidence as ABLATE
-from tools import engine_evidence
-
-_DOMAIN = b"coli-ablate-manifest/2\n"
-
-
-def _serialize(records):
- return b"".join(
- json.dumps(record, separators=(",", ":")).encode("ascii") + b"\n"
- for record in records)
-
-
-def _write(root, manifest_raw, evidence_raw, config_raw):
- manifest = root / "manifest.txt"
- evidence = root / "evidence.jsonl"
- config = root / "config.json"
- manifest.write_bytes(manifest_raw)
- evidence.write_bytes(evidence_raw)
- config.write_bytes(config_raw)
- return manifest, evidence, config
-
-
-def _run_cli(manifest, evidence, config):
- return subprocess.run(
- [sys.executable, str(pathlib.Path(ABLATE.__file__)),
- str(manifest), str(evidence), "--config", str(config)],
- stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
-
-
-class _GoldenFixture(unittest.TestCase):
- """Shared two-item artifact, hand-derived from the documented schema.
-
- Manifest: item 1 (T=3, prompt=1, baseline, tokens 0,1,2) then item 2
- (T=2, prompt=1, mode 1 with one ablated cell at layer 1/expert 2,
- tokens 3,0). vocab=4, n_layers=4, first_dense=1, n_experts=5, so
- topk = min(32, 4) = 4. Positions/gold are derived by hand from the
- manifest's own tokens: item 1 has positions [0, 1] with gold tokens
- 1 and 2; item 2 has position [0] with gold token 0.
- """
-
- MANIFEST = b"1 3 1 0 0 0 1 2\n2 2 1 1 1 1 2 -1 3 0\n"
- CONFIG = (b'{"vocab_size":4,"num_hidden_layers":4,'
- b'"first_k_dense_replace":1,"n_routed_experts":5}\n')
- MANIFEST_SHA256 = hashlib.sha256(_DOMAIN + MANIFEST).hexdigest()
- CONFIG_SHA256 = hashlib.sha256(CONFIG).hexdigest()
-
- @classmethod
- def golden_records(cls):
- header = {
- "t": "hdr", "schema": "coli-ablate/2", "vocab": 4, "topk": 4,
- "n_layers": 4, "first_dense": 1, "n_experts": 5,
- "config_sha256": cls.CONFIG_SHA256,
- "manifest_sha256": cls.MANIFEST_SHA256,
- "expected_items": 2, "expected_targets": 3,
- }
- item1_header = {"t": "ah", "item": 1, "mode": 0, "ncells": 0,
- "T": 3, "n_prompt": 1, "cells": []}
- row1 = {"t": "lg", "item": 1, "pos": 0, "gold": 1,
- "nll": 0.2, "glogit": 1.0, "molo": 0.5, "mgn": 0.5,
- "am": 1, "amlogit": 1.0, "logZ": 1.3, "corr": 1,
- "tk": [[0, 0.1], [1, 1.0], [2, 0.3], [3, -0.2]]}
- row2 = {"t": "lg", "item": 1, "pos": 1, "gold": 2,
- "nll": 0.7, "glogit": 0.4, "molo": 0.9, "mgn": -0.5,
- "am": 0, "amlogit": 0.9, "logZ": 1.1, "corr": 0,
- "tk": [[0, 0.9], [1, 0.1], [2, 0.4], [3, -0.3]]}
- item2_header = {"t": "ah", "item": 2, "mode": 1, "ncells": 1,
- "T": 2, "n_prompt": 1, "cells": [[1, 2, -1]]}
- row3 = {"t": "lg", "item": 2, "pos": 0, "gold": 0,
- "nll": 0.0, "glogit": 2.0, "molo": -1e30, "mgn": 1e30,
- "am": 0, "amlogit": 2.0, "logZ": 2.0, "corr": 1,
- "tk": [[0, 2.0], [1, -1.0], [2, -2.0], [3, -3.0]]}
- done = {"t": "done", "manifest_sha256": cls.MANIFEST_SHA256,
- "completed_items": 2, "completed_targets": 3}
- return [header, item1_header, row1, row2, item2_header, row3, done]
-
- def _reject(self, mutate, records=None):
- """Apply `mutate` to a deep copy of the golden records and assert
- the mutated artifact is refused."""
- mutated = copy.deepcopy(records if records is not None
- else self.golden_records())
- mutate(mutated)
- with tempfile.TemporaryDirectory() as tmp:
- root = pathlib.Path(tmp)
- manifest, evidence, config = _write(
- root, self.MANIFEST, _serialize(mutated), self.CONFIG)
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE.validate_ablate_evidence(manifest, evidence, config)
-
-
-class GoldenArtifactAcceptedTests(_GoldenFixture):
- def test_valid_artifact_is_accepted_and_pass_line_is_exact(self):
- with tempfile.TemporaryDirectory() as tmp:
- root = pathlib.Path(tmp)
- manifest, evidence, config = _write(
- root, self.MANIFEST, _serialize(self.golden_records()),
- self.CONFIG)
- result = ABLATE.validate_ablate_evidence(
- manifest, evidence, config)
- self.assertEqual(result, {
- "manifest_sha256": self.MANIFEST_SHA256,
- "items": 2, "targets": 3,
- })
- cli = _run_cli(manifest, evidence, config)
- self.assertEqual(cli.returncode, 0, cli.stderr.decode(errors="replace"))
- self.assertEqual(cli.stderr, b"")
- # print() terminates the line with the platform's newline, so a
- # Windows child hands back CRLF; the pin is on the line's content.
- self.assertEqual(
- cli.stdout.replace(b"\r\n", b"\n"),
- f"[ablate-evidence] PASS manifest={self.MANIFEST_SHA256} "
- f"items=2 targets=3\n".encode("ascii"))
-
-
-class TopkProducerCapAboveVocabFourTests(unittest.TestCase):
- """The header `topk == min(32, vocab)` check only ever exercises the
- "vocab is the binding constraint" side at `_GoldenFixture`'s vocab=4.
- This fixture uses vocab=40 (above both 4 and the 32 cap) to pin the
- other side: topk must be capped at 32, not left equal to vocab.
- """
-
- MANIFEST = b"1 2 1 0 0 0 39\n"
- CONFIG = (b'{"vocab_size":40,"num_hidden_layers":1,'
- b'"first_k_dense_replace":0,"n_routed_experts":1}\n')
- MANIFEST_SHA256 = hashlib.sha256(_DOMAIN + MANIFEST).hexdigest()
- CONFIG_SHA256 = hashlib.sha256(CONFIG).hexdigest()
-
- @classmethod
- def golden_records(cls, topk=32, tk_count=32):
- header = {
- "t": "hdr", "schema": "coli-ablate/2", "vocab": 40, "topk": topk,
- "n_layers": 1, "first_dense": 0, "n_experts": 1,
- "config_sha256": cls.CONFIG_SHA256,
- "manifest_sha256": cls.MANIFEST_SHA256,
- "expected_items": 1, "expected_targets": 1,
- }
- item1_header = {"t": "ah", "item": 1, "mode": 0, "ncells": 0,
- "T": 2, "n_prompt": 1, "cells": []}
- row = {"t": "lg", "item": 1, "pos": 0, "gold": 39,
- "nll": 0.0, "glogit": 1.0, "molo": 0.5, "mgn": 0.5,
- "am": 39, "amlogit": 1.0, "logZ": 1.3, "corr": 1,
- "tk": [[i, -0.01 * i] for i in range(tk_count)]}
- done = {"t": "done", "manifest_sha256": cls.MANIFEST_SHA256,
- "completed_items": 1, "completed_targets": 1}
- return [header, item1_header, row, done]
-
- def test_topk_capped_at_32_for_vocab_above_the_cap_is_accepted(self):
- with tempfile.TemporaryDirectory() as tmp:
- root = pathlib.Path(tmp)
- manifest, evidence, config = _write(
- root, self.MANIFEST, _serialize(self.golden_records()),
- self.CONFIG)
- result = ABLATE.validate_ablate_evidence(
- manifest, evidence, config)
- self.assertEqual(result, {
- "manifest_sha256": self.MANIFEST_SHA256,
- "items": 1, "targets": 1,
- })
-
- def test_topk_left_uncapped_at_vocab_above_32_is_rejected(self):
- # vocab=40 > 32, so header topk must be 32 -- not 40 (== vocab).
- records = self.golden_records(topk=40, tk_count=40)
- with tempfile.TemporaryDirectory() as tmp:
- root = pathlib.Path(tmp)
- manifest, evidence, config = _write(
- root, self.MANIFEST, _serialize(records), self.CONFIG)
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE.validate_ablate_evidence(manifest, evidence, config)
-
-
-class EvidenceFramingTests(_GoldenFixture):
- """`validate_ablate_evidence`'s canonical-LF-JSONL framing check."""
-
- def _reject_raw(self, evidence_raw):
- with tempfile.TemporaryDirectory() as tmp:
- root = pathlib.Path(tmp)
- manifest, evidence, config = _write(
- root, self.MANIFEST, evidence_raw, self.CONFIG)
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE.validate_ablate_evidence(manifest, evidence, config)
-
- def test_empty_evidence_rejected(self):
- self._reject_raw(b"")
-
- def test_evidence_missing_trailing_newline_rejected(self):
- self._reject_raw(_serialize(self.golden_records())[:-1])
-
- def test_evidence_with_carriage_return_rejected(self):
- self._reject_raw(_serialize(self.golden_records()).replace(
- b"\n", b"\r\n", 1))
-
- def test_evidence_with_nul_byte_rejected(self):
- self._reject_raw(_serialize(self.golden_records()) + b"\0")
-
-
-class HeaderRecordTests(_GoldenFixture):
- CASES = (
- ("missing_key", lambda r: r[0].pop("topk")),
- ("extra_key", lambda r: r[0].__setitem__("extra", 1)),
- ("wrong_t", lambda r: r[0].__setitem__("t", "nope")),
- ("wrong_schema", lambda r: r[0].__setitem__(
- "schema", "coli-ablate/1")),
- ("topk_wrong_type", lambda r: r[0].__setitem__("topk", "4")),
- ("config_sha256_wrong_type", lambda r: r[0].__setitem__(
- "config_sha256", 1)),
- ("config_sha256_not_hex", lambda r: r[0].__setitem__(
- "config_sha256", "z" * 64)),
- ("manifest_sha256_wrong_type", lambda r: r[0].__setitem__(
- "manifest_sha256", None)),
- ("manifest_sha256_not_hex", lambda r: r[0].__setitem__(
- "manifest_sha256", "0" * 63 + "g")),
- ("vocab_out_of_range", lambda r: r[0].__setitem__("vocab", 0)),
- ("n_layers_out_of_range", lambda r: r[0].__setitem__(
- "n_layers", 0)),
- ("first_dense_out_of_range", lambda r: r[0].__setitem__(
- "first_dense", 99)),
- ("n_experts_out_of_range", lambda r: r[0].__setitem__(
- "n_experts", 0)),
- ("expected_items_out_of_range", lambda r: r[0].__setitem__(
- "expected_items", 0)),
- ("expected_targets_out_of_range", lambda r: r[0].__setitem__(
- "expected_targets", 0)),
- ("topk_not_producer_exact", lambda r: r[0].__setitem__("topk", 3)),
- ("vocab_identity_mismatch", lambda r: (
- r[0].__setitem__("vocab", 5), r[0].__setitem__("topk", 5))),
- ("n_layers_identity_mismatch", lambda r: r[0].__setitem__(
- "n_layers", 2)),
- ("first_dense_identity_mismatch", lambda r: r[0].__setitem__(
- "first_dense", 0)),
- ("n_experts_identity_mismatch", lambda r: r[0].__setitem__(
- "n_experts", 6)),
- ("config_sha256_identity_mismatch", lambda r: r[0].__setitem__(
- "config_sha256", "0" * 64)),
- ("manifest_sha256_binding_mismatch", lambda r: r[0].__setitem__(
- "manifest_sha256", "1" * 64)),
- ("expected_items_binding_mismatch", lambda r: r[0].__setitem__(
- "expected_items", 99)),
- ("expected_targets_binding_mismatch", lambda r: r[0].__setitem__(
- "expected_targets", 99)),
- )
-
- def test_header_field_checks(self):
- for name, mutate in self.CASES:
- with self.subTest(name=name):
- self._reject(mutate)
-
- def test_topk_not_producer_exact_even_when_every_row_agrees_with_it(self):
- # Isolates the header-level topk==min(32,vocab) check from the
- # per-row "len(tk) == header['topk']" shape check: here every row's
- # tk list is ALSO shrunk to match the wrong topk, so only the
- # header-level producer-exactness check can catch the artifact.
- def mutate(records):
- records[0]["topk"] = 2
- for record in records:
- if record.get("t") == "lg":
- record["tk"] = record["tk"][:2]
- self._reject(mutate)
-
-
-class ItemHeaderRecordTests(_GoldenFixture):
- def test_missing_item_header_rejected(self):
- self._reject(lambda r: r.__delitem__(slice(1, None)))
-
- def test_item_header_not_a_dict_rejected(self):
- self._reject(lambda r: r.__setitem__(1, 5))
-
- def test_item_header_missing_key_rejected(self):
- self._reject(lambda r: r[1].pop("ncells"))
-
- def test_item_header_extra_key_rejected(self):
- self._reject(lambda r: r[1].__setitem__("extra", 1))
-
- def test_item_header_field_wrong_type_rejected(self):
- self._reject(lambda r: r[1].__setitem__("item", "1"))
-
- def test_item_header_cells_not_a_list_rejected(self):
- self._reject(lambda r: r[4].__setitem__("cells", {}))
-
- def test_item_header_cell_wrong_shape_rejected(self):
- self._reject(lambda r: r[4].__setitem__("cells", [[1, 2]]))
-
- def test_item_header_cell_element_wrong_type_rejected(self):
- self._reject(lambda r: r[4].__setitem__(
- "cells", [[1, 2, "x"]]))
-
- def test_item_header_mismatch_vs_manifest_rejected(self):
- self._reject(lambda r: r[1].__setitem__("T", 99))
-
-
-class TargetRowRecordTests(_GoldenFixture):
- def test_missing_target_row_rejected(self):
- self._reject(lambda r: r.__delitem__(slice(2, None)))
-
- def test_row_not_a_dict_rejected(self):
- self._reject(lambda r: r.__setitem__(2, 5))
-
- def test_row_missing_key_rejected(self):
- self._reject(lambda r: r[2].pop("corr"))
-
- def test_row_extra_key_rejected(self):
- self._reject(lambda r: r[2].__setitem__("extra", 1))
-
- def test_row_wrong_t_rejected(self):
- self._reject(lambda r: r[2].__setitem__("t", "nope"))
-
- def test_row_item_mismatch_rejected(self):
- self._reject(lambda r: r[2].__setitem__("item", 99))
-
- def test_row_pos_mismatch_rejected(self):
- self._reject(lambda r: r[2].__setitem__("pos", 5))
-
- def test_row_gold_mismatch_rejected(self):
- self._reject(lambda r: r[2].__setitem__("gold", 0))
-
- def test_row_am_wrong_type_rejected(self):
- self._reject(lambda r: r[2].__setitem__("am", "1"))
-
- def test_row_am_out_of_range_rejected(self):
- self._reject(lambda r: r[2].__setitem__("am", 4))
-
- def test_row_corr_wrong_type_rejected(self):
- self._reject(lambda r: r[2].__setitem__("corr", "1"))
-
- def test_row_corr_out_of_range_rejected(self):
- self._reject(lambda r: r[2].__setitem__("corr", 2))
-
- NUMERIC_FIELDS = ("nll", "glogit", "molo", "mgn", "amlogit", "logZ")
-
- def test_row_numeric_field_wrong_type_rejected(self):
- for field in self.NUMERIC_FIELDS:
- with self.subTest(field=field):
- self._reject(lambda r, field=field: r[2].__setitem__(
- field, "0"))
-
- def test_row_tk_not_a_list_rejected(self):
- self._reject(lambda r: r[2].__setitem__("tk", 5))
-
- def test_row_tk_wrong_length_rejected(self):
- self._reject(lambda r: r[2].__setitem__(
- "tk", r[2]["tk"][:-1]))
-
- def test_row_tk_pair_wrong_shape_rejected(self):
- self._reject(lambda r: r[2]["tk"].__setitem__(0, [0, 0.1, 9]))
-
- def test_row_tk_pair_id_wrong_type_rejected(self):
- self._reject(lambda r: r[2]["tk"].__setitem__(0, ["0", 0.1]))
-
- def test_row_tk_pair_id_out_of_range_rejected(self):
- self._reject(lambda r: r[2]["tk"].__setitem__(0, [4, 0.1]))
-
- def test_row_tk_pair_val_wrong_type_rejected(self):
- self._reject(lambda r: r[2]["tk"].__setitem__(0, [0, "0.1"]))
-
- def test_row_tk_duplicate_ids_rejected(self):
- self._reject(lambda r: r[2]["tk"].__setitem__(0, list(r[2]["tk"][1])))
-
-
-class CrossFieldInvariantTests(_GoldenFixture):
- """Invariants the engine's per-record emitter, `ablate_logit_record`
- (and the row-writer `ablate_logit_line` it calls), guarantees for
- every row it emits.
-
- - `nll >= 0`: `nll` is `-target_lp` (`ablate_logit_record`'s own
- `gnll=-target_lp`), and `target_lp` is `delta - logse` where
- `delta = lo[target] - r.max <= 0` (target's logit minus the row
- max) and `logse = log(sum_i exp(lo[i]-max)) >= log(1) = 0` (the
- max's own term contributes exp(0)=1 to that sum) -- the row-level
- helpers this emitter builds on (`logprob_row_checked`/
- `logprob_from_row_checked`). So `target_lp <= 0` always, hence
- `nll >= 0` always.
- - `corr == (am == gold)`: the emitter passes an `argmax==gold`
- comparison directly as the `corr` argument to `ablate_logit_line`,
- and `am` is that same argmax -- `corr` is never anything but that
- comparison's result.
- - `amlogit >= glogit`: `amlogit` is the row's own maximum logit and
- `glogit` is one particular entry of that same row, so it can
- never exceed the row's own maximum.
-
- Top-k ordering is deliberately NOT enforced: `tk` is unsorted on the
- wire by design (`logit_topk_select`, documented as "deliberately not
- a lowest-token-id tie rule").
- """
-
- def test_nll_negative_rejected(self):
- self._reject(lambda r: r[2].__setitem__("nll", -0.1))
-
- def test_nll_zero_accepted(self):
- with tempfile.TemporaryDirectory() as tmp:
- root = pathlib.Path(tmp)
- records = self.golden_records()
- records[2]["nll"] = 0.0
- manifest, evidence, config = _write(
- root, self.MANIFEST, _serialize(records), self.CONFIG)
- ABLATE.validate_ablate_evidence(manifest, evidence, config)
-
- def test_corr_true_when_am_not_gold_rejected(self):
- # am=1 == gold=1 in the golden row, so corr must be 1; forcing 0
- # while leaving am/gold untouched breaks the agreement.
- self._reject(lambda r: r[2].__setitem__("corr", 0))
-
- def test_corr_false_when_am_equals_gold_rejected(self):
- # am=0 != gold=2 in row2 (index 3), so corr must be 0; forcing 1
- # breaks the agreement the other way.
- self._reject(lambda r: r[3].__setitem__("corr", 1))
-
- def test_amlogit_below_glogit_rejected(self):
- self._reject(lambda r: r[2].__setitem__("amlogit", 0.5))
-
- def test_amlogit_equal_to_glogit_accepted(self):
- # row1 already has amlogit == glogit == 1.0 (am == gold there);
- # confirm the boundary itself -- not just values strictly above
- # it -- is accepted.
- with tempfile.TemporaryDirectory() as tmp:
- root = pathlib.Path(tmp)
- records = self.golden_records()
- self.assertEqual(records[2]["amlogit"], records[2]["glogit"])
- manifest, evidence, config = _write(
- root, self.MANIFEST, _serialize(records), self.CONFIG)
- ABLATE.validate_ablate_evidence(manifest, evidence, config)
-
-
-class TerminalRecordTests(_GoldenFixture):
- def test_missing_terminal_record_rejected(self):
- self._reject(lambda r: r.__delitem__(slice(6, None)))
-
- def test_terminal_not_a_dict_rejected(self):
- self._reject(lambda r: r.__setitem__(6, 5))
-
- def test_terminal_missing_key_rejected(self):
- self._reject(lambda r: r[6].pop("completed_items"))
-
- def test_terminal_extra_key_rejected(self):
- self._reject(lambda r: r[6].__setitem__("extra", 1))
-
- def test_terminal_completed_items_out_of_range_rejected(self):
- self._reject(lambda r: r[6].__setitem__("completed_items", 0))
-
- def test_terminal_completed_targets_out_of_range_rejected(self):
- self._reject(lambda r: r[6].__setitem__("completed_targets", 0))
-
- def test_terminal_wrong_t_rejected(self):
- self._reject(lambda r: r[6].__setitem__("t", "nope"))
-
- def test_terminal_manifest_sha256_mismatch_rejected(self):
- self._reject(lambda r: r[6].__setitem__("manifest_sha256", "1" * 64))
-
- def test_terminal_completed_items_mismatch_rejected(self):
- self._reject(lambda r: r[6].__setitem__("completed_items", 1))
-
- def test_terminal_completed_targets_mismatch_rejected(self):
- self._reject(lambda r: r[6].__setitem__("completed_targets", 1))
-
- def test_trailing_record_after_terminal_rejected(self):
- self._reject(lambda r: r.append(dict(r[6])))
-
-
-class TruncationReplayAndMismatchBiteTests(_GoldenFixture):
- """Bite-style table close to the source's own producer-invariant
- checks, rebuilt on this module's literal golden fixture instead of
- an engine-produced one."""
-
- def test_named_mutations_all_refuse(self):
- cases = (
- ("missing_done", lambda r: r.__delitem__(6)),
- ("truncated_last_row", lambda r: r.__setitem__(
- 5, {"t": "lg", "item": 2})),
- ("replayed_item_header", lambda r: r.insert(2, dict(r[1]))),
- ("duplicate_done", lambda r: r.append(dict(r[6]))),
- ("missing_target_row", lambda r: r.__delitem__(3)),
- ("wrong_gold_downstream", lambda r: r[3].__setitem__(
- "gold", 0)),
- ("header_digest_forged", lambda r: r[0].__setitem__(
- "manifest_sha256", "2" * 64)),
- )
- for name, mutate in cases:
- with self.subTest(name=name):
- self._reject(mutate)
-
-
-class ManifestProofFramingTests(unittest.TestCase):
- """`_manifest_proof`'s non-canonical-text framing checks."""
-
- ARGS = (4, 4, 1, 5) # vocab, n_layers, first_dense, n_experts
-
- def _reject(self, raw):
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE._manifest_proof(raw, *self.ARGS)
-
- def test_empty_manifest_rejected(self):
- self._reject(b"")
-
- def _accept(self, raw):
- return ABLATE._manifest_proof(raw, *self.ARGS)
-
- # The engine accepts a manifest saved with CRLF endings and one whose last
- # line has no terminator, and digests the canonical form of either. A
- # checker that refused them would reject files the producer really ran.
- def test_manifest_missing_trailing_newline_accepted(self):
- self._accept(b"1 3 1 0 0 0 1 2")
-
- def test_manifest_with_crlf_endings_accepted(self):
- self._accept(b"1 3 1 0 0 0 1 2\r\n")
-
- def test_all_three_framings_bind_the_same_digest(self):
- canonical = self._accept(b"1 3 1 0 0 0 1 2\n")["sha256"]
- self.assertEqual(self._accept(b"1 3 1 0 0 0 1 2\r\n")["sha256"], canonical)
- self.assertEqual(self._accept(b"1 3 1 0 0 0 1 2")["sha256"], canonical)
-
- def test_carriage_return_inside_a_record_rejected(self):
- self._reject(b"1 3 1 0\r 0 0 1 2\n")
-
- def test_empty_line_inside_a_manifest_rejected(self):
- self._reject(b"1 3 1 0 0 0 1 2\n\n2 2 1 0 0 0 1\n")
-
- def test_manifest_with_nul_byte_rejected(self):
- self._reject(b"1 3 1 0 0 0 1 2\n\0")
-
- def test_manifest_non_ascii_line_rejected(self):
- self._reject("1 3 1 0 0 0 1 é\n".encode("utf-8"))
-
- def test_manifest_control_byte_breaks_grammar_not_framing(self):
- # A vertical tab embedded mid-line is not a canonical digit/space
- # byte; it must be caught by the integer-grammar check, not
- # silently absorbed as a line boundary bytes.splitlines() would
- # not treat it as one either way (see the dedicated probe below).
- self._reject(b"1 3\x0b1 0 0 0 1 2\n")
-
- def test_duplicate_item_id_across_lines_rejected(self):
- self._reject(b"1 2 1 0 0 0 1\n1 2 1 0 0 0 1\n")
-
- def test_records_are_split_on_newline_only(self):
- # The module now splits the canonical form on b"\n" rather than
- # calling splitlines(), so no other byte can become a record
- # boundary. bytes.splitlines() would additionally break on \r,
- # which the canonical form no longer contains but which a future
- # edit could reintroduce; splitting explicitly removes the
- # question. These bytes must therefore stay inside one record and
- # be caught by the integer-grammar check.
- for value in (0x0B, 0x0C, 0x1C, 0x1D, 0x1E):
- with self.subTest(byte=hex(value)):
- raw = b"1 3 1 0 0 0 1" + bytes([value]) + b"2\n"
- self._reject(raw)
-
-
-class CanonicalManifestDigestTests(unittest.TestCase):
- """The canonical rule, pinned against the engine by a literal digest.
-
- `c/tests/test_ablate_mode.c` asserts the same 64 characters for the same
- manifest content. Two implementations that each only agreed with
- themselves would both pass their own suites while disagreeing in the
- field; a literal known answer on both sides is what rules that out.
- """
-
- RECORD = b"0 3 2 0 0 1 2 3\n"
- KNOWN = "c63a48c375b14ca60f26c7e3c5dd36b5929ffaf669a45511c93deee6e8bbd5ed"
-
- def test_known_answer_matches_the_engine(self):
- digest = hashlib.sha256(
- ABLATE.DOMAIN + engine_evidence.canonical_manifest_bytes(self.RECORD)
- ).hexdigest()
- self.assertEqual(digest, self.KNOWN)
-
- def test_every_accepted_framing_reaches_the_known_answer(self):
- for raw in (self.RECORD, b"0 3 2 0 0 1 2 3\r\n", b"0 3 2 0 0 1 2 3"):
- with self.subTest(raw=raw):
- digest = hashlib.sha256(
- ABLATE.DOMAIN
- + engine_evidence.canonical_manifest_bytes(raw)).hexdigest()
- self.assertEqual(digest, self.KNOWN)
-
- def test_canonicalization_refuses_what_the_engine_refuses(self):
- for raw in (b"", b"\n", b"a\n\nb\n", b"a\rb\n", b"a\0b\n"):
- with self.subTest(raw=raw):
- with self.assertRaises(engine_evidence.ManifestFormError):
- engine_evidence.canonical_manifest_bytes(raw)
-
-
-class ManifestProofFieldBoundaryTests(unittest.TestCase):
- """Per-field/per-cell/per-token bounds `_manifest_proof` enforces.
-
- Table built from `test_manifest_fixed_width_and_topology_c_python_parity`'s
- Python-side expectations (each `expected` value here is the same
- literal that method asserted, not something this module computed):
- that method also cross-checked each case against a C test binary
- this module does not build, so it is not this module's oracle to
- carry (flagged separately, not absorbed here).
- """
-
- def test_boundary_table(self):
- i32 = ABLATE._INT32_MAX
- i64 = ABLATE._INT64_MAX
- sixteen = " ".join(f"{layer} 0 -1" for layer in range(1, 17))
- cases = (
- ("baseline_min", b"0 2 1 0 0 0 1\n", 4, 4, 1, 8, True),
- ("fewer_than_five_fields", b"1 2\n", 4, 4, 1, 8, False),
- ("item_max", f"{i64} 2 1 0 0 0 1\n".encode(),
- 4, 4, 1, 8, True),
- ("item_max_plus_1", f"{i64 + 1} 2 1 0 0 0 1\n".encode(),
- 4, 4, 1, 8, False),
- ("item_min_minus_1", b"-1 2 1 0 0 0 1\n", 4, 4, 1, 8, False),
- ("T_min", b"7 2 1 0 0 0 1\n", 4, 4, 1, 8, True),
- ("T_below_min", b"7 1 1 0 0 0\n", 4, 4, 1, 8, False),
- ("T_max_incomplete", f"7 {i32} 1 0 0\n".encode(),
- 4, 4, 1, 8, False),
- ("T_max_plus_1", f"7 {i32 + 1} 1 0 0\n".encode(),
- 4, 4, 1, 8, False),
- ("prompt_max_incomplete", f"7 {i32} {i32 - 1} 0 0\n".encode(),
- 4, 4, 1, 8, False),
- ("prompt_max_plus_1", f"7 {i32} {i32 + 1} 0 0\n".encode(),
- 4, 4, 1, 8, False),
- ("prompt_min_minus_1", b"7 2 0 0 0 0 1\n",
- 4, 4, 1, 8, False),
- ("mode_max", b"7 2 1 3 1 1 2 3 0 1\n",
- 4, 4, 1, 8, True),
- ("mode_max_plus_1", b"7 2 1 4 1 1 2 -1 0 1\n",
- 4, 4, 1, 8, False),
- ("cells_max", f"7 2 1 1 16 {sixteen} 0 1\n".encode(),
- 4, 17, 1, 8, True),
- ("cells_max_plus_1", b"7 2 1 1 17 0 1\n",
- 4, 17, 1, 8, False),
- ("nonbaseline_zero", b"7 2 1 1 0 0 1\n",
- 4, 4, 1, 8, False),
- ("dense_layer", b"7 2 1 1 1 0 2 -1 0 1\n",
- 4, 4, 1, 8, False),
- ("layer_min", b"7 2 1 1 1 0 2 -1 0 1\n",
- 4, 4, 0, 8, True),
- ("layer_upper", b"7 2 1 1 1 3 2 -1 0 1\n",
- 4, 4, 1, 8, True),
- ("layer_engine_max", b"7 2 1 1 1 127 2 -1 0 1\n",
- 4, 128, 0, 8, True),
- ("layer_engine_max_plus_1", b"7 2 1 1 1 128 2 -1 0 1\n",
- 4, 128, 0, 8, False),
- ("source_upper", b"7 2 1 1 1 1 7 -1 0 1\n",
- 4, 4, 1, 8, True),
- ("source_min", b"7 2 1 1 1 1 0 -1 0 1\n",
- 4, 4, 1, 8, True),
- ("source_engine_max", b"7 2 1 1 1 1 4095 -1 0 1\n",
- 4, 4, 1, 4096, True),
- ("source_engine_max_plus_1", b"7 2 1 1 1 1 4096 -1 0 1\n",
- 4, 4, 1, 4096, False),
- ("target_upper", b"7 2 1 3 1 1 2 7 0 1\n",
- 4, 4, 1, 8, True),
- ("target_self_swap", b"7 2 1 3 1 1 2 2 0 1\n",
- 4, 4, 1, 8, False),
- ("target_signed_min",
- f"7 2 1 3 1 1 2 {ABLATE._INT32_MIN} 0 1\n".encode(),
- 4, 4, 1, 8, False),
- ("target_engine_max", b"7 2 1 3 1 1 0 4095 0 1\n",
- 4, 4, 1, 4096, True),
- ("target_engine_max_plus_1", b"7 2 1 3 1 1 0 4096 0 1\n",
- 4, 4, 1, 4096, False),
- ("target_max_plus_1",
- f"7 2 1 3 1 1 2 {i32 + 1} 0 1\n".encode(),
- 4, 4, 1, 8, False),
- ("duplicate_source", b"7 2 1 1 2 1 2 -1 1 2 -1 0 1\n",
- 4, 4, 1, 8, False),
- ("token_min", b"7 2 1 0 0 0 0\n", 1, 4, 1, 8, True),
- ("token_upper", b"7 2 1 0 0 0 16777215\n",
- 1 << 24, 4, 1, 8, True),
- ("token_max_plus_1", b"7 2 1 0 0 0 16777216\n",
- 1 << 24, 4, 1, 8, False),
- ("vocab_max", b"7 2 1 0 0 0 1\n",
- 1 << 24, 4, 1, 8, True),
- ("vocab_max_plus_1", b"7 2 1 0 0 0 1\n",
- (1 << 24) + 1, 4, 1, 8, False),
- ("leading_zero_rejected", b"07 2 1 0 0 0 1\n",
- 4, 4, 1, 8, False),
- ("plus_sign_rejected", b"+7 2 1 0 0 0 1\n",
- 4, 4, 1, 8, False),
- ("double_space_rejected", b"7 2 1 0 0 0 1\n",
- 4, 4, 1, 8, False),
- ("trailing_space_rejected", b"7 2 1 0 0 0 1 \n",
- 4, 4, 1, 8, False),
- )
- for (name, raw, vocab, layers, first_dense, experts,
- expected) in cases:
- with self.subTest(name=name):
- if expected:
- ABLATE._manifest_proof(
- raw, vocab, layers, first_dense, experts)
- else:
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE._manifest_proof(
- raw, vocab, layers, first_dense, experts)
-
-
-class ConfigIdentityTests(unittest.TestCase):
- CONFIG = (b'{"vocab_size":4,"num_hidden_layers":4,'
- b'"first_k_dense_replace":1,"n_routed_experts":5}\n')
-
- def test_engine_text_size_rejects_non_int_length(self):
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE._checked_engine_text_size(True, "config")
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE._checked_engine_text_size(1.0, "config")
-
- def test_engine_byte_limit_is_inclusive_and_enforced_both_sides(self):
- engine_limit = 256 << 20
- self.assertEqual(ABLATE._ENGINE_TEXT_MAX_BYTES, engine_limit)
- self.assertEqual(
- ABLATE._checked_engine_text_size(engine_limit, "config"),
- engine_limit)
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE._checked_engine_text_size(engine_limit + 1, "config")
-
- with tempfile.TemporaryDirectory() as tmp:
- root = pathlib.Path(tmp)
- ablate_config = root / "ablate-config.json"
- ablate_config.write_bytes(self.CONFIG)
- with mock.patch.object(
- ABLATE, "_ENGINE_TEXT_MAX_BYTES", len(self.CONFIG)):
- identity = ABLATE._config_identity(ablate_config)
- self.assertEqual(identity["vocab"], 4)
- self.assertEqual(
- identity["config_sha256"],
- hashlib.sha256(self.CONFIG).hexdigest())
- ablate_config.write_bytes(self.CONFIG + b" ")
- with self.assertRaisesRegex(
- ABLATE.AblateEvidenceError, "256 MiB"):
- ABLATE._config_identity(ablate_config)
-
- def _reject(self, raw):
- with tempfile.TemporaryDirectory() as tmp:
- path = pathlib.Path(tmp) / "config.json"
- path.write_bytes(raw)
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE._config_identity(path)
-
- def test_empty_config_rejected(self):
- self._reject(b"")
-
- def test_invalid_json_config_rejected(self):
- self._reject(b"{not json}\n")
-
- def test_non_object_root_rejected(self):
- self._reject(b"[1,2,3]\n")
-
- def test_vocab_size_missing_rejected(self):
- self._reject(b'{"num_hidden_layers":4,"first_k_dense_replace":1,'
- b'"n_routed_experts":5}\n')
-
- def test_vocab_size_out_of_range_rejected(self):
- self._reject(b'{"vocab_size":0,"num_hidden_layers":4,'
- b'"first_k_dense_replace":1,"n_routed_experts":5}\n')
-
- def test_num_hidden_layers_out_of_range_rejected(self):
- self._reject(b'{"vocab_size":4,"num_hidden_layers":0,'
- b'"first_k_dense_replace":1,"n_routed_experts":5}\n')
-
- def test_n_routed_experts_out_of_range_rejected(self):
- self._reject(b'{"vocab_size":4,"num_hidden_layers":4,'
- b'"first_k_dense_replace":1,"n_routed_experts":0}\n')
-
- def test_first_k_dense_replace_out_of_range_rejected(self):
- self._reject(b'{"vocab_size":4,"num_hidden_layers":4,'
- b'"first_k_dense_replace":5,"n_routed_experts":5}\n')
-
-
-class JsonHelperTests(unittest.TestCase):
- def test_duplicate_json_key_rejected(self):
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE._json_record(b'{"a":1,"a":2}', "record 1")
-
- def test_nan_constant_rejected(self):
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE._json_record(b'{"nll":NaN}', "record 1")
-
- def test_infinity_constant_rejected(self):
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE._json_record(b'{"nll":Infinity}', "record 1")
-
- def test_negative_infinity_constant_rejected(self):
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE._json_record(b'{"nll":-Infinity}', "record 1")
-
- def test_non_ascii_bytes_rejected(self):
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE._json_record(b"\xff", "record 1")
-
- def test_malformed_json_rejected(self):
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE._json_record(b"{not json}", "record 1")
-
-
-class FixedWidthHelperTests(unittest.TestCase):
- def test_bounded_int_rejects_bool_disguised_as_int(self):
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE._bounded_int(True, "x", 1, 10)
-
- def test_int64_max_is_the_literal_signed_64_bit_bound(self):
- # Pinned by literal, not derived, so a future refactor of the
- # module's own (1 << 63) - 1 expression cannot silently drift.
- self.assertEqual(ABLATE._INT64_MAX, 9223372036854775807)
-
- def test_fixed_width_helpers_and_derived_count_boundaries(self):
- for value in (ABLATE._INT64_MIN, ABLATE._INT64_MAX):
- self.assertEqual(ABLATE._manifest_i64(str(value), 1), value)
- for value in (ABLATE._INT64_MIN - 1, ABLATE._INT64_MAX + 1):
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE._manifest_i64(str(value), 1)
- for value in (ABLATE._INT32_MIN, ABLATE._INT32_MAX):
- self.assertEqual(ABLATE._bounded_int(
- value, "int32", ABLATE._INT32_MIN, ABLATE._INT32_MAX), value)
- for value in (ABLATE._INT32_MIN - 1, ABLATE._INT32_MAX + 1):
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE._bounded_int(
- value, "int32", ABLATE._INT32_MIN, ABLATE._INT32_MAX)
- self.assertEqual(
- ABLATE._count_add(0, ABLATE._INT64_MAX, "count"),
- ABLATE._INT64_MAX)
- self.assertEqual(
- ABLATE._count_add(ABLATE._INT64_MAX, 0, "count"),
- ABLATE._INT64_MAX)
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE._count_add(ABLATE._INT64_MAX, 1, "count")
- for label in ("expected_items", "expected_targets",
- "completed_items", "completed_targets"):
- self.assertEqual(
- ABLATE._bounded_int(1, label, 1, ABLATE._INT64_MAX), 1)
- self.assertEqual(ABLATE._bounded_int(
- ABLATE._INT64_MAX, label, 1, ABLATE._INT64_MAX),
- ABLATE._INT64_MAX)
- for value in (0, ABLATE._INT64_MAX + 1):
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE._bounded_int(value, label, 1, ABLATE._INT64_MAX)
-
- def test_retained_long_max_plus_one_artifact_is_incomplete(self):
- manifest_raw = b"9223372036854775808 2 1 0 0 0 1\n"
- digest = hashlib.sha256(_DOMAIN + manifest_raw).hexdigest()
- self.assertEqual(
- digest,
- "988a1cf2ddc812f38138e51eecfebb2ba0c9980e31b4c7183396716f114d6538")
- with tempfile.TemporaryDirectory() as tmp:
- root = pathlib.Path(tmp)
- config_raw = (b'{"vocab_size":2,"num_hidden_layers":4,'
- b'"first_k_dense_replace":1,"n_routed_experts":8}\n')
- records = (
- {"t": "hdr", "schema": "coli-ablate/2", "vocab": 2,
- "topk": 2, "n_layers": 4, "first_dense": 1,
- "n_experts": 8,
- "config_sha256": hashlib.sha256(config_raw).hexdigest(),
- "manifest_sha256": digest,
- "expected_items": 1, "expected_targets": 1},
- {"t": "ah", "item": 9223372036854775808, "mode": 0,
- "ncells": 0, "T": 2, "n_prompt": 1, "cells": []},
- {"t": "lg", "item": 9223372036854775808, "pos": 0,
- "gold": 1, "nll": 0, "glogit": 0, "molo": 0,
- "mgn": 0, "am": 1, "amlogit": 0, "logZ": 0,
- "corr": 1, "tk": [[1, 0], [0, -1]]},
- {"t": "done", "manifest_sha256": digest,
- "completed_items": 1, "completed_targets": 1},
- )
- manifest, evidence, config = _write(
- root, manifest_raw, _serialize(records), config_raw)
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE.validate_ablate_evidence(manifest, evidence, config)
- cli = _run_cli(manifest, evidence, config)
- self.assertNotEqual(cli.returncode, 0)
- self.assertIn(b"[ablate-evidence] INCOMPLETE:", cli.stderr)
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/c/tests/test_eval_glm.py b/c/tests/test_eval_glm.py
index fd153199..eef6e766 100644
--- a/c/tests/test_eval_glm.py
+++ b/c/tests/test_eval_glm.py
@@ -26,8 +26,7 @@ that performs them:
is exactly the defect dev's plain ``line[0] in "-0123456789"`` filter
does not catch (differential bite, below).
- `score_request_wire`: strict ASCII/LF request grammar, the per-record
- SHA-256 digest, and the inclusive 256 MiB engine text limit shared with
- `check_ablate_evidence.py`.
+ SHA-256 digest, and the inclusive 256 MiB engine text limit.
- `completion_error`: the exact zero-exit/complete-count/positive-token
denominator that alone passes.
- `main`: incremental durability (one written+flushed row per completed
@@ -42,9 +41,6 @@ Deferred (need a live binary this module does not have access to):
- `test_c_emitted_c17g_corpus_is_canonical`, which drives
``test_logprob_wire --score-c17g-fixture`` (a binary produced by a
different part of this project's build, not present here).
-- the ABLATE-block stdout probes (`c_manifest_accepts`/
- `test_complete_production_fixture_is_strict_json`-style checks): out of
- scope for this module (owned by `test_check_ablate_evidence.py`).
No model is run by the committed tests -- every case here drives
`eval_glm.py` against an injected stand-in for the direct engine launch
@@ -75,18 +71,11 @@ _spec = importlib.util.spec_from_file_location(
EVAL = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(EVAL)
-_ablate_spec = importlib.util.spec_from_file_location(
- "check_ablate_evidence_under_test", TOOLS / "check_ablate_evidence.py")
-ABLATE = importlib.util.module_from_spec(_ablate_spec)
-_ablate_spec.loader.exec_module(ABLATE)
-
class EvalGlmEvidenceTests(unittest.TestCase):
BANNER = (
"== GLM C engine (glm_moe_dsa), cache=64 experts/layer | "
"compute experts@4-bit dense@8-bit | idot: neon-i8mm ==")
- CONFIG = (b'{"vocab_size":4,"num_hidden_layers":4,'
- b'"first_k_dense_replace":1,"n_routed_experts":5}\n')
@staticmethod
def loaded(state="ACTIVE", draft=1, layers=78, experts=256,
@@ -759,39 +748,16 @@ class EvalGlmEvidenceTests(unittest.TestCase):
self.assertIn("MEAN acc_norm", self.last_stdout)
def test_python_engine_byte_limits_are_inclusive_and_preallocation(self):
- # The 256 MiB inclusive engine-text limit is shared between
- # this module and check_ablate_evidence.py; checked on both
- # sides of the shared constant/helper.
engine_limit = 256 << 20
- self.assertEqual(ABLATE._ENGINE_TEXT_MAX_BYTES, engine_limit)
self.assertEqual(EVAL._ENGINE_TEXT_MAX_BYTES, engine_limit)
- self.assertEqual(
- ABLATE._checked_engine_text_size(engine_limit, "config"),
- engine_limit)
self.assertEqual(
EVAL._checked_engine_text_size(engine_limit, "SCORE"),
engine_limit)
- with self.assertRaises(ABLATE.AblateEvidenceError):
- ABLATE._checked_engine_text_size(engine_limit + 1, "config")
with self.assertRaises(EVAL.EvidenceError):
EVAL._checked_engine_text_size(engine_limit + 1, "SCORE")
with tempfile.TemporaryDirectory() as tmp:
root = pathlib.Path(tmp)
- ablate_config = root / "ablate-config.json"
- ablate_config.write_bytes(self.CONFIG)
- with mock.patch.object(
- ABLATE, "_ENGINE_TEXT_MAX_BYTES", len(self.CONFIG)):
- identity = ABLATE._config_identity(ablate_config)
- self.assertEqual(identity["vocab"], 4)
- self.assertEqual(
- identity["config_sha256"],
- hashlib.sha256(self.CONFIG).hexdigest())
- ablate_config.write_bytes(self.CONFIG + b" ")
- with self.assertRaisesRegex(
- ABLATE.AblateEvidenceError, "256 MiB"):
- ABLATE._config_identity(ablate_config)
-
score_config = root / "config.json"
score_raw = b'{"vocab_size":4}\n'
score_config.write_bytes(score_raw)
diff --git a/c/tests/test_pack_python.py b/c/tests/test_pack_python.py
index e2f0375a..fee5a4fd 100644
--- a/c/tests/test_pack_python.py
+++ b/c/tests/test_pack_python.py
@@ -259,42 +259,5 @@ class PackPythonDataFileTests(unittest.TestCase):
self.assertIn("1 Python files and 1 data files", buf.getvalue())
-class HumanOnlyToolsShipExplicitly(unittest.TestCase):
- """Tools that only a person invokes are invisible to the import walk.
-
- `pack_python.py` computes the archive from what `coli` reaches, imports and
- subprocesses alike. A script whose caller is a person typing argparse
- arguments is reached by nobody in the code, so static analysis cannot put
- it in the archive -- `k3_tokenizer.py` has always been in that position and
- `release.yml` copies it by hand. `check_ablate_evidence.py` joined it: it
- verifies the ABLATE evidence that the packaged `eval_glm.py` produces, so a
- release that ships the producer without the verifier is half a tool.
-
- This pins both halves of that decision on the real tree: the walk does NOT
- reach the file (if it ever does, the explicit copy became redundant and this
- test should be retired), and the workflow copies AND gates it, the way it
- does for the tokenizer helper. A copy without a gate is how v1.10.0 shipped
- four broken commands green (#1296)."""
-
- HUMAN_ONLY = ("k3_tokenizer.py", "check_ablate_evidence.py")
-
- def test_the_walk_does_not_reach_them(self):
- paths = PACK.needed(HERE.parent)
- for name in self.HUMAN_ONLY:
- self.assertTrue((TOOLS / name).is_file(), name)
- self.assertNotIn(TOOLS / name, paths,
- f"{name} is now reached by the walk; drop its "
- f"explicit copy from release.yml and this pin")
-
- def test_release_copies_and_gates_them(self):
- release = (HERE.parent.parent / ".github" / "workflows"
- / "release.yml").read_text(encoding="utf-8")
- for name in self.HUMAN_ONLY:
- self.assertIn(f"cp c/tools/{name} dist/tools/", release,
- f"release.yml no longer copies {name}")
- self.assertIn(f"test -f tools/{name} ||", release,
- f"release.yml copies {name} but does not gate on it")
-
-
if __name__ == "__main__":
unittest.main()
diff --git a/c/tools/check_ablate_evidence.py b/c/tools/check_ablate_evidence.py
deleted file mode 100644
index 0f8486ed..00000000
--- a/c/tools/check_ablate_evidence.py
+++ /dev/null
@@ -1,403 +0,0 @@
-#!/usr/bin/env python3
-"""Validate one complete ABLATE evidence artifact against a config.json."""
-
-import argparse
-import hashlib
-import json
-import math
-import pathlib
-import re
-import sys
-
-# The canonical-manifest rule is shared with the engine and with the other
-# evidence checkers, so it lives in one module. Imported both ways because this
-# file is run as a script from the engine directory and imported as part of the
-# tools package by the tests.
-try:
- from tools.engine_evidence import (
- ManifestFormError, canonical_manifest_bytes)
-except ImportError: # run directly: tools/ is on the path
- from engine_evidence import ManifestFormError, canonical_manifest_bytes
-
-
-DOMAIN = b"coli-ablate-manifest/2\n"
-_INT = re.compile(r"(?:0|[1-9][0-9]*|-[1-9][0-9]*)")
-_SHA256 = re.compile(r"[0-9a-f]{64}")
-
-# The wire schema (``coli-ablate/2``) is a fixed-width LP64 domain, chosen so
-# a producer and a checker on different host ABIs agree byte-for-byte: every
-# manifest integer and every completion counter is signed 64-bit, while a
-# value the engine narrows to C ``int`` (a layer, an expert, a token id) is
-# signed 32-bit.
-_INT32_MIN = -(1 << 31)
-_INT32_MAX = (1 << 31) - 1
-_INT64_MIN = -(1 << 63)
-_INT64_MAX = (1 << 63) - 1
-_ENGINE_VOCAB_MAX = 1 << 24
-_ENGINE_LAYERS_MAX = 128
-_ENGINE_EXPERTS_MAX = 4096
-_ENGINE_TEXT_MAX_BYTES = 256 << 20
-
-
-class AblateEvidenceError(ValueError):
- """The artifact cannot prove a complete ABLATE denominator."""
-
-
-def _checked_engine_text_size(length, label):
- if type(length) is not int or not 0 <= length <= _ENGINE_TEXT_MAX_BYTES:
- raise AblateEvidenceError(
- f"{label} exceeds the inclusive 256 MiB engine limit")
- return length
-
-
-def _reject_constant(value):
- raise AblateEvidenceError(f"non-JSON constant: {value}")
-
-
-def _reject_duplicate_keys(pairs):
- result = {}
- for key, value in pairs:
- if key in result:
- raise AblateEvidenceError(f"duplicate JSON key: {key}")
- result[key] = value
- return result
-
-
-def _json_record(raw, label):
- try:
- return json.loads(
- raw.decode("ascii"), parse_constant=_reject_constant,
- object_pairs_hook=_reject_duplicate_keys)
- except (UnicodeDecodeError, json.JSONDecodeError,
- AblateEvidenceError) as exc:
- raise AblateEvidenceError(f"invalid {label} JSON: {exc}") from exc
-
-
-def _bounded_config_bytes(config_path):
- path = pathlib.Path(config_path)
- with path.open("rb") as source:
- source.seek(0, 2)
- length = source.tell()
- _checked_engine_text_size(length, "external config.json")
- source.seek(0)
- raw = source.read(_ENGINE_TEXT_MAX_BYTES + 1)
- _checked_engine_text_size(len(raw), "external config.json")
- return raw
-
-
-def _config_identity(config_path):
- raw = _bounded_config_bytes(config_path)
- if not raw:
- raise AblateEvidenceError("external config.json is empty")
- try:
- config = json.loads(
- raw.decode("utf-8"), parse_constant=_reject_constant,
- object_pairs_hook=_reject_duplicate_keys)
- except (UnicodeDecodeError, json.JSONDecodeError,
- AblateEvidenceError) as exc:
- raise AblateEvidenceError(f"external config.json is invalid: {exc}") from exc
- if not isinstance(config, dict):
- raise AblateEvidenceError("external config.json root is not an object")
- try:
- identity = {
- "vocab": _bounded_int(
- config["vocab_size"], "config vocab_size",
- 1, _ENGINE_VOCAB_MAX),
- "n_layers": _bounded_int(
- config["num_hidden_layers"], "config num_hidden_layers",
- 1, _ENGINE_LAYERS_MAX),
- "n_experts": _bounded_int(
- config["n_routed_experts"], "config n_routed_experts",
- 1, _ENGINE_EXPERTS_MAX),
- }
- identity["first_dense"] = _bounded_int(
- config["first_k_dense_replace"],
- "config first_k_dense_replace", 0, identity["n_layers"])
- except (KeyError, AblateEvidenceError) as exc:
- raise AblateEvidenceError(
- "external config.json topology is incomplete or invalid") from exc
- identity["config_sha256"] = hashlib.sha256(raw).hexdigest()
- return identity
-
-
-def _bounded_int(value, label, minimum, maximum):
- if type(value) is not int or not minimum <= value <= maximum:
- raise AblateEvidenceError(
- f"{label} is outside {minimum}..{maximum}")
- return value
-
-
-def _manifest_i64(text, line_number):
- try:
- value = int(text)
- except ValueError as exc:
- raise AblateEvidenceError(
- f"manifest line {line_number} integer is too large") from exc
- return _bounded_int(
- value, f"manifest line {line_number} integer",
- _INT64_MIN, _INT64_MAX)
-
-
-def _count_add(current, increment, label):
- _bounded_int(current, label, 0, _INT64_MAX)
- _bounded_int(increment, f"{label} increment", 0, _INT64_MAX)
- if increment > _INT64_MAX - current:
- raise AblateEvidenceError(f"{label} exceeds signed 64-bit domain")
- return current + increment
-
-
-def _manifest_proof(raw, vocab, n_layers, first_dense, n_experts):
- # The engine accepts a manifest saved with either line ending, and with or
- # without a terminator on its last line, and digests the canonical form
- # rather than the bytes on disk. Reproduce that here from the shared rule,
- # or this checker would reject a file the producer ran and would compute a
- # different digest for one it accepted.
- try:
- raw = canonical_manifest_bytes(raw)
- except ManifestFormError as exc:
- raise AblateEvidenceError(f"manifest is not canonical text: {exc}") from exc
- _bounded_int(vocab, "external vocabulary", 1, _ENGINE_VOCAB_MAX)
- _bounded_int(n_layers, "external n_layers", 1, _ENGINE_LAYERS_MAX)
- _bounded_int(first_dense, "header first_dense", 0, n_layers)
- _bounded_int(n_experts, "external n_experts", 1, _ENGINE_EXPERTS_MAX)
- items = []
- seen = set()
- item_count = targets = 0
- for line_number, raw_line in enumerate(raw[:-1].split(b"\n"), 1):
- try:
- text = raw_line.decode("ascii")
- except UnicodeDecodeError as exc:
- raise AblateEvidenceError(
- f"manifest line {line_number} is not ASCII") from exc
- parts = text.split(" ")
- if (not parts or any(not _INT.fullmatch(part) for part in parts) or
- " ".join(parts) != text):
- raise AblateEvidenceError(
- f"manifest line {line_number} is not canonical integer grammar")
- values = [_manifest_i64(part, line_number) for part in parts]
- if len(values) < 5:
- raise AblateEvidenceError(f"manifest line {line_number} is truncated")
- item, length, prompt, mode, cells = values[:5]
- if (item < 0 or item in seen or
- not 2 <= length <= _INT32_MAX or prompt < 1 or
- prompt >= length or mode not in range(4) or
- cells not in range(17) or
- (mode == 0 and cells != 0) or
- (mode != 0 and cells == 0)):
- raise AblateEvidenceError(
- f"manifest line {line_number} has invalid fields/denominator")
- expected = 5 + 3 * cells + length
- if len(values) != expected:
- raise AblateEvidenceError(
- f"manifest line {line_number} has invalid fields/denominator")
- triples = []
- source_cells = set()
- cursor = 5
- for _ in range(cells):
- layer, expert, applied = values[cursor:cursor + 3]
- cursor += 3
- if (not first_dense <= layer < n_layers or
- not 0 <= expert < n_experts or
- not _INT32_MIN <= applied <= _INT32_MAX or
- (mode == 3 and
- (not 0 <= applied < n_experts or applied == expert)) or
- (mode != 3 and applied != -1) or
- (layer, expert) in source_cells):
- raise AblateEvidenceError(
- f"manifest line {line_number} has invalid cell")
- source_cells.add((layer, expert))
- triples.append([layer, expert, applied])
- tokens = values[cursor:]
- if any(token < 0 or token >= vocab for token in tokens):
- raise AblateEvidenceError(
- f"manifest line {line_number} has out-of-vocabulary token")
- seen.add(item)
- positions = range(prompt - 1, length - 1)
- row_targets = length - prompt
- item_count = _count_add(item_count, 1, "manifest item count")
- targets = _count_add(targets, row_targets, "manifest target count")
- items.append({
- "item": item, "T": length, "n_prompt": prompt,
- "mode": mode, "ncells": cells, "cells": triples,
- "positions": positions, "tokens": tuple(tokens),
- })
- if item_count <= 0 or targets <= 0:
- raise AblateEvidenceError("manifest denominator is not positive")
- return {
- "sha256": hashlib.sha256(DOMAIN + raw).hexdigest(),
- "items": tuple(items), "item_count": item_count, "targets": targets,
- }
-
-
-def validate_ablate_evidence(manifest_path, evidence_path, config_path):
- identity = _config_identity(config_path)
- manifest_raw = pathlib.Path(manifest_path).read_bytes()
- evidence_raw = pathlib.Path(evidence_path).read_bytes()
- if (not evidence_raw or not evidence_raw.endswith(b"\n") or
- b"\r" in evidence_raw or b"\0" in evidence_raw):
- raise AblateEvidenceError("evidence is not nonempty canonical LF JSONL")
- records = [_json_record(line, f"record {index}")
- for index, line in enumerate(evidence_raw.splitlines(), 1)]
- if not records:
- raise AblateEvidenceError("evidence has no header")
- header = records[0]
- if (not isinstance(header, dict) or set(header) != {
- "t", "schema", "vocab", "topk", "manifest_sha256",
- "n_layers", "first_dense", "n_experts",
- "config_sha256",
- "expected_items", "expected_targets"} or
- header.get("t") != "hdr" or header.get("schema") != "coli-ablate/2" or
- type(header.get("topk")) is not int or
- not isinstance(header.get("config_sha256"), str) or
- not _SHA256.fullmatch(header["config_sha256"]) or
- not isinstance(header.get("manifest_sha256"), str) or
- not _SHA256.fullmatch(header["manifest_sha256"])):
- raise AblateEvidenceError("header keys or values are not exact")
- try:
- _bounded_int(header["vocab"], "header vocabulary",
- 1, _ENGINE_VOCAB_MAX)
- if header["topk"] != min(32, header["vocab"]):
- raise AblateEvidenceError("header topk is not producer-exact")
- _bounded_int(header["n_layers"], "header n_layers",
- 1, _ENGINE_LAYERS_MAX)
- _bounded_int(header["first_dense"], "header first_dense",
- 0, header["n_layers"])
- _bounded_int(header["n_experts"], "header n_experts",
- 1, _ENGINE_EXPERTS_MAX)
- _bounded_int(header["expected_items"], "header expected_items",
- 1, _INT64_MAX)
- _bounded_int(header["expected_targets"], "header expected_targets",
- 1, _INT64_MAX)
- except (KeyError, AblateEvidenceError) as exc:
- raise AblateEvidenceError(
- f"header keys or values are not exact: {exc}") from exc
- if any(header[key] != identity[key] for key in (
- "vocab", "n_layers", "first_dense", "n_experts",
- "config_sha256")):
- raise AblateEvidenceError(
- "header does not match the external config identity")
- proof = _manifest_proof(
- manifest_raw, identity["vocab"], identity["n_layers"],
- identity["first_dense"], identity["n_experts"])
- if (header["manifest_sha256"] != proof["sha256"] or
- header["expected_items"] != proof["item_count"] or
- header["expected_targets"] != proof["targets"]):
- raise AblateEvidenceError("header does not bind the source manifest proof")
-
- cursor = 1
- completed_items = completed_targets = 0
- logit_keys = {
- "t", "item", "pos", "gold", "nll", "glogit", "molo", "mgn",
- "am", "amlogit", "logZ", "corr", "tk",
- }
- for expected in proof["items"]:
- if cursor >= len(records):
- raise AblateEvidenceError("missing item header")
- item_header = records[cursor]
- cursor += 1
- if (not isinstance(item_header, dict) or set(item_header) != {
- "t", "item", "mode", "ncells", "T", "n_prompt", "cells"} or
- any(type(item_header.get(key)) is not int for key in (
- "item", "mode", "ncells", "T", "n_prompt")) or
- not isinstance(item_header.get("cells"), list) or
- any(not isinstance(cell, list) or len(cell) != 3 or
- any(type(value) is not int for value in cell)
- for cell in item_header["cells"]) or
- item_header != {key: expected[key] for key in (
- "item", "mode", "ncells", "T", "n_prompt", "cells")} |
- {"t": "ah"}):
- raise AblateEvidenceError("item header does not match manifest order")
- for position in expected["positions"]:
- if cursor >= len(records):
- raise AblateEvidenceError("missing target row")
- row = records[cursor]
- cursor += 1
- if (not isinstance(row, dict) or set(row) != logit_keys or
- row.get("t") != "lg" or type(row.get("item")) is not int or
- row["item"] != expected["item"] or
- type(row.get("pos")) is not int or row["pos"] != position or
- type(row.get("gold")) is not int or
- row["gold"] != expected["tokens"][position + 1] or
- type(row.get("am")) is not int or
- row["am"] not in range(header["vocab"]) or
- type(row.get("corr")) is not int or row["corr"] not in (0, 1)):
- raise AblateEvidenceError("target row identity/fields are invalid")
- for field in ("nll", "glogit", "molo", "mgn", "amlogit", "logZ"):
- if (type(row.get(field)) not in (int, float) or
- not math.isfinite(row[field])):
- raise AblateEvidenceError(f"target row {field} is nonfinite")
- # These three hold for every row the producer can emit: nll is a
- # negated log-probability (always <= 0 before negation); corr is
- # defined as the argmax/gold agreement, not sampled separately;
- # and amlogit is the row's own max logit, so no field can exceed
- # it -- least of all the gold token's own logit.
- if row["nll"] < 0:
- raise AblateEvidenceError("target row nll is negative")
- if row["corr"] != int(row["am"] == row["gold"]):
- raise AblateEvidenceError(
- "target row corr does not match its am/gold agreement")
- if row["amlogit"] < row["glogit"]:
- raise AblateEvidenceError(
- "target row amlogit is below glogit")
- topk = row.get("tk")
- if (not isinstance(topk, list) or len(topk) != header["topk"] or
- any(not isinstance(pair, list) or len(pair) != 2 or
- type(pair[0]) is not int or not 0 <= pair[0] < header["vocab"] or
- type(pair[1]) not in (int, float) or not math.isfinite(pair[1])
- for pair in topk) or
- len({pair[0] for pair in topk}) != len(topk)):
- raise AblateEvidenceError("target row top-k is invalid")
- completed_targets = _count_add(
- completed_targets, 1, "completed target count")
- completed_items = _count_add(
- completed_items, 1, "completed item count")
-
- if cursor >= len(records):
- raise AblateEvidenceError("missing terminal completion record")
- done = records[cursor]
- cursor += 1
- if (not isinstance(done, dict) or set(done) != {
- "t", "manifest_sha256", "completed_items", "completed_targets"}):
- raise AblateEvidenceError("terminal completion proof is invalid")
- try:
- _bounded_int(done["completed_items"], "done completed_items",
- 1, _INT64_MAX)
- _bounded_int(done["completed_targets"], "done completed_targets",
- 1, _INT64_MAX)
- except (KeyError, AblateEvidenceError) as exc:
- raise AblateEvidenceError("terminal completion proof is invalid") from exc
- if done != {"t": "done", "manifest_sha256": proof["sha256"],
- "completed_items": completed_items,
- "completed_targets": completed_targets}:
- raise AblateEvidenceError("terminal completion proof is invalid")
- if cursor != len(records):
- raise AblateEvidenceError("records follow terminal completion proof")
- if (completed_items != proof["item_count"] or
- completed_targets != proof["targets"]):
- raise AblateEvidenceError("completed denominator does not match manifest")
- return {
- "manifest_sha256": proof["sha256"], "items": completed_items,
- "targets": completed_targets,
- }
-
-
-def main(argv=None):
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("manifest")
- parser.add_argument("evidence")
- parser.add_argument("--config", required=True,
- help="independently supplied loaded-model config.json")
- args = parser.parse_args(argv)
- try:
- result = validate_ablate_evidence(
- args.manifest, args.evidence, args.config)
- except (OSError, AblateEvidenceError) as exc:
- print(f"[ablate-evidence] INCOMPLETE: {exc}", file=sys.stderr)
- return 1
- print(f"[ablate-evidence] PASS manifest={result['manifest_sha256']} "
- f"items={result['items']} targets={result['targets']}")
- return 0
-
-
-if __name__ == "__main__":
- sys.exit(main())