Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(tools): bind the eval harness's evidence to engine output
monotophic593e7f6a3a37
diff --git a/c/tests/test_eval_glm.py b/c/tests/test_eval_glm.pynew file mode 100644index 00000000..5295426a--- /dev/null+++ b/c/tests/test_eval_glm.py@@ -0,0 +1,974 @@+"""tools/eval_glm.py must accept only the engine's real SCORE stdout+records and refuse everything else with a named error: an unparsed or+duplicated banner/load preamble, a SCORE line whose numeric token is not+the canonical finite ``%.6f``/``%.17g`` spelling the engine actually+emits, a replayed or out-of-order identity-bound evidence record, and a+foreign stdout line that is neither a preamble nor a SCORE record. It+must also write result rows incrementally (one flush per request, never+buffered until completion) and mark incomplete runs, including a+pre-launch refusal, before the engine is ever started.++Checks enumerated from the source (`tools/eval_glm.py`, read in full+before writing this module) and covered below, grouped by the function+that performs them:++- `parse_c17g`: both exact finite spellings the engine actually emits --+ the shipped ``%.6f`` form and the newer ``%.17g`` evidence form -- and+ nothing else; this module imports nothing from and shares no code with+ `check_data_logprob_gaps.py`, which parses its own independent grammar.+- `parse_score_result` / `_SCORE_RE`: the shipped ``%.6f`` form AND the+ ``%.17g`` evidence form (both numeric forms), non-finite/malformed+ rejection, and the ``<contlen> <greedy>`` metadata bounds.+- `classify_score_stdout` / `ScoreStdoutClassifier`: the banner/load+ preamble lifecycle (missing, duplicated, out-of-order, or a SCORE+ record before the load record all refuse); every line that is not an+ exact banner, an exact load record, or an exact SCORE record refuses+ by name -- a foreign line is never silently treated as a score, which+ is exactly the defect dev's plain ``line[0] in "-0123456789"`` filter+ does not catch (differential bite, below).+- identity-bound evidence mode (``ScoreStdoutClassifier(request_digests)``):+ strict ordinal join, digest binding, replay/duplicate/out-of-order/+ extra-record refusal; a stream that mixes identity-bound and legacy+ records refuses by name; a legacy-only stream still completes, marked+ UNBOUND rather than silently treated as bound.+- `score_request_wire`: strict ASCII/LF request grammar, the per-record+ SHA-256 digest, and the inclusive 256 MiB engine text limit shared with+ `check_ablate_evidence.py`.+- `completion_error`: the exact zero-exit/complete-count/positive-token+ denominator that alone passes.+- `main`: incremental durability (one written+flushed row per completed+ request, never buffered until the run ends) and pre-launch INCOMPLETE+ marking (no benchmark tasks selected; zero SCORE requests produced;+ every choice's context/continuation split is empty) -- the engine is+ never launched for any of these; a mid-run crash or interrupt still+ leaves the INCOMPLETE marker and terminates the child process; a+ partial run still prints the accuracy table over whatever rows landed.++Deferred (need a live binary this module does not have access to):+- `test_c_emitted_c17g_corpus_is_canonical`, which drives+ ``test_logprob_wire --score-c17g-fixture`` (a binary produced by a+ different part of this project's build, not present here).+- the ABLATE-block stdout probes (`c_manifest_accepts`/+ `test_complete_production_fixture_is_strict_json`-style checks): out of+ scope for this module (owned by `test_check_ablate_evidence.py`).++No model is run by the committed tests -- every case here drives+`eval_glm.py` against an injected stand-in for the direct engine launch+(a fake ``subprocess.Popen`` returning canned stdout/stderr), never a+real ``./glm`` process.+"""++import contextlib+import hashlib+import importlib.util+import io+import json+import os+import pathlib+import signal+import sys+import tempfile+import types+import unittest+from unittest import mock+++HERE = pathlib.Path(__file__).resolve().parent+TOOLS = HERE.parent / "tools"++_spec = importlib.util.spec_from_file_location(+ "eval_glm_under_test", TOOLS / "eval_glm.py")+EVAL = importlib.util.module_from_spec(_spec)+_spec.loader.exec_module(EVAL)++_ablate_spec = importlib.util.spec_from_file_location(+ "check_ablate_evidence_under_test", TOOLS / "check_ablate_evidence.py")+ABLATE = importlib.util.module_from_spec(_ablate_spec)+_ablate_spec.loader.exec_module(ABLATE)+++class EvalGlmEvidenceTests(unittest.TestCase):+ BANNER = (+ "== GLM C engine (glm_moe_dsa), cache=64 experts/layer | "+ "compute experts@4-bit dense@8-bit | idot: neon-i8mm ==")+ CONFIG = (b'{"vocab_size":4,"num_hidden_layers":4,'+ b'"first_k_dense_replace":1,"n_routed_experts":5}\n')++ @staticmethod+ def loaded(state="ACTIVE", draft=1, layers=78, experts=256,+ load="1.00", resident="1.00"):+ return (f"loaded in {load}s | resident dense: {resident} MB | "+ f"layers={layers} experts={experts} | MTP {state} "+ f"(draft={draft})")++ def run_eval_main(self, stdout_records, bind_evidence=True):+ class Encoded:+ ids = [1, 2]++ class FakeTokenizer:+ @staticmethod+ def from_file(path):+ return FakeTokenizer()++ @staticmethod+ def encode(text):+ return Encoded()++ request_raw = b"2 2 1 2 1 2\n"+ request_digest = hashlib.sha256(request_raw).hexdigest()+ bound_records = []+ score_index = 0+ for record in stdout_records:+ line = record[:-1] if record.endswith("\n") else record+ if bind_evidence and EVAL._SCORE_RE.fullmatch(line):+ record = f"SCORE {score_index} {request_digest} {line}\n"+ score_index += 1+ bound_records.append(record)+ process = types.SimpleNamespace(+ returncode=0, stderr=(), stdout=tuple(bound_records),+ wait=lambda: 0, poll=lambda: 0, terminate=lambda: None)+ with tempfile.TemporaryDirectory() as tmp:+ output = pathlib.Path(tmp) / "results.csv"+ (pathlib.Path(tmp) / "config.json").write_text(+ '{"vocab_size":3}\n')+ argv = [+ "eval_glm.py", "--snap", tmp, "--tasks", "smoke",+ "--limit", "1", "--glm", "/fake/glm", "--out",+ str(output),+ ]+ tokenizers = types.SimpleNamespace(Tokenizer=FakeTokenizer)+ stderr_buf = io.StringIO()+ stdout_buf = io.StringIO()+ with mock.patch.object(sys, "argv", argv), \+ mock.patch.dict(sys.modules, {"tokenizers": tokenizers}), \+ mock.patch.object(EVAL.subprocess, "Popen",+ return_value=process) as popen, \+ contextlib.redirect_stderr(stderr_buf), \+ contextlib.redirect_stdout(stdout_buf):+ rc = EVAL.main()+ self.last_popen_kwargs = popen.call_args.kwargs if popen.called else {}+ self.last_stderr = stderr_buf.getvalue()+ self.last_stdout = stdout_buf.getvalue()+ return rc, output.read_text(), popen.call_count++ def test_exact_score_text_survives_csv(self):+ text = "-8.5534581234567888"+ exact, value, contlen, greedy = EVAL.parse_score_result(+ f"{text} 4096 1")+ self.assertEqual(exact, text)+ self.assertEqual(contlen, 4096)+ self.assertEqual(greedy, 1)+ self.assertTrue(value < 0)++ out = io.StringIO()+ meta = ("task", 3, 2, 4096, 17, 2)+ EVAL.write_result_row(out, 9, meta, exact, greedy)+ self.assertEqual(+ out.getvalue(),+ "9,task,3,2,4096,17,2,-8.5534581234567888,1\n")++ def test_shipped_dot6f_form_is_still_accepted(self):+ # The tool must accept BOTH the shipped %.6f form the engine+ # actually prints today AND the %.17g evidence form.+ exact, value, contlen, greedy = EVAL.parse_score_result(+ "-8.553458 4096 1")+ self.assertEqual(exact, "-8.553458")+ self.assertEqual(value, -8.553458)+ self.assertEqual((contlen, greedy), (4096, 1))++ def test_score_request_digest_binds_strict_ascii_bytes_including_lf(self):+ requests = ("1 1 1 2", "2 1 0 1 2")+ lines, payload, digests = EVAL.score_request_wire(requests, 3)+ self.assertEqual(lines, (+ b"1 1 1 2\n", b"2 1 0 1 2\n"))+ self.assertEqual(payload, b"".join(lines))+ self.assertEqual(+ digests,+ tuple(hashlib.sha256(line).hexdigest() for line in lines))+ self.assertNotEqual(+ digests[1],+ hashlib.sha256(b"2 1 0 1 2").hexdigest())+ bad_requests = (+ "", "two\nlines", "cr\rline", "1 1", "1 1 0",+ "1 1 0 1 2", "1 1 0 1 junk", "1 1 0 3",+ "1 1 -1 1", "1 1 00 1", "2147483648 1 0 1",+ "1 2147483647 0 1", "evidence-μ",+ )+ for bad in bad_requests:+ with self.subTest(bad=bad):+ with self.assertRaises(EVAL.EvidenceError):+ EVAL.score_request_wire((bad,), 3)+ with self.assertRaises(EVAL.EvidenceError):+ EVAL.score_request_wire((), 3)++ def test_nonfinite_and_malformed_scores_refuse(self):+ bad = [+ "nan 1 1", "inf 1 1", "-inf 1 1", "-1 1", "-1 1 1 extra",+ "not-a-number 1 1", "-1 -1 1", "-1 0 1", "-1 1 2", "1 1 1",+ "-1 1 1", "-1\t1 1", "-1 01 1", "-1_0 1 1",+ "-1 2147483648 1", "+0 1 0", "-01 1 0",+ "-0_125 1 0", "-١ 1 0", "-1e-9999 1 0",+ "-1.00000000000000000 1 0", "-1e-9 1 0",+ "-1e--09 1 0", "-1e+009 1 0", "-1.25 1 0 junk",+ ]+ for line in bad:+ with self.subTest(line=line):+ with self.assertRaises(EVAL.EvidenceError):+ EVAL.parse_score_result(line)++ def test_stdout_grammar_refuses_unknown_records(self):+ self.assertIsNone(EVAL.classify_score_stdout(self.BANNER + "\n"))+ for state, draft in (("ACTIVE", 0), ("ACTIVE", 1),+ ("absent", 0), ("absent", 2),+ ("DISABLED (multiplexed serve)", 0)):+ with self.subTest(state=state, draft=draft):+ self.assertIsNone(EVAL.classify_score_stdout(+ self.loaded(state, draft) + "\n"))+ exact, _, _, _ = EVAL.classify_score_stdout("-1.25 1 0\n")+ self.assertEqual(exact, "-1.25")+ bad = (+ "\n", "unexpected banner\n", "nan 1 0\n", "inf 1 0\n",+ " -1.25 1 0\n", "-1.25 1 0 \n", "-1.25 1 0\t\n",+ "-1.25 1 0", "-1.25 1 0\r\n",+ "PROF 0.001 1 1 0.000 0.000 0.000 0.000 0.000 1\n",+ "DONE 7 STAT 1 1.00 0.0 1.00 1 0\n",+ "== GLM C engine fabricated ==\n",+ "== GLM C engine (glm_moe_dsa), cache=0 experts/layer | "+ "compute experts@4-bit dense@8-bit | idot: neon ==\n",+ "== GLM C engine (glm_moe_dsa), cache=064 experts/layer | "+ "compute experts@4-bit dense@8-bit | idot: neon ==\n",+ "loaded in 1.0s | resident dense: 1.00 MB | layers=78 experts=256 | "+ "MTP ACTIVE (draft=1)\n",+ "loaded in 1.00s | resident dense: 1.00 MB | layers=78 experts=256 | "+ "MTP DISABLED (multiplexed serve) (draft=1)\n",+ "loaded in 1.00s | resident dense: 1.00 MB | layers=78 experts=256 | "+ "MTP unknown (draft=0)\n",+ "loaded in 1.00s | resident dense: 1.00 MB | layers=78 experts=256 | "+ "MTP ACTIVE (draft=64)\n",+ "loaded in 1.00s | resident dense: 1.00 MB | layers=0 experts=256 | "+ "MTP ACTIVE (draft=1)\n",+ "loaded in 1.00s | resident dense: 1.00 MB | layers=129 experts=256 | "+ "MTP ACTIVE (draft=1)\n",+ "loaded in 1.00s | resident dense: 1.00 MB | layers=78 experts=0 | "+ "MTP ACTIVE (draft=1)\n",+ "loaded in 1.00s | resident dense: 1.00 MB | layers=78 experts=4097 | "+ "MTP ACTIVE (draft=1)\n",+ "loaded in -0.01s | resident dense: 1.00 MB | layers=78 experts=256 | "+ "MTP ACTIVE (draft=1)\n",+ "loaded in nan s | resident dense: 1.00 MB | layers=78 experts=256 | "+ "MTP ACTIVE (draft=1)\n",+ "loaded in 1.0s | resident dense: 1.00 MB | layers=78 experts=256 | "+ "MTP ACTIVE (draft=1)\n",+ "loaded in 1.00s | resident dense: 1.00 MB | layers=2147483648 experts=256 | "+ "MTP ACTIVE (draft=1)\n",+ " == GLM C engine (glm_moe_dsa), cache=64 experts/layer | "+ "compute experts@4-bit dense@8-bit | idot: neon ==\n",+ )+ for line in bad:+ with self.subTest(line=line):+ with self.assertRaises(EVAL.EvidenceError):+ EVAL.classify_score_stdout(line)++ def test_banner_kernels_and_load_boundaries_are_exact(self):+ self.assertEqual(+ EVAL.parse_engine_banner(self.BANNER)["kernel"], "neon-i8mm")+ for kernel in ("avx512-vnni", "avx-vnni", "avx2", "neon-i8mm",+ "neon", "vsx", "scalar"):+ line = self.BANNER.replace("neon-i8mm", kernel)+ self.assertEqual(EVAL.parse_engine_banner(line)["kernel"], kernel)+ with self.assertRaises(EVAL.PreambleError):+ EVAL.parse_engine_banner(self.BANNER.replace("neon-i8mm", "fabricated"))++ for layers, experts in ((1, 1), (128, 4096)):+ parsed = EVAL.parse_engine_loaded(self.loaded(+ layers=layers, experts=experts))+ self.assertEqual((parsed["layers"], parsed["experts"]),+ (layers, experts))++ def test_score_stream_owns_banner_load_lifecycle(self):+ parser = EVAL.ScoreStdoutClassifier()+ self.assertIsNone(parser.classify(self.BANNER + "\n"))+ self.assertIsNone(parser.classify(self.loaded("absent", 2) + "\n"))+ exact, _, _, _ = parser.classify("-1.25 1 0\n")+ self.assertEqual(exact, "-1.25")+ parser.finish()++ cases = (+ [self.loaded() + "\n", self.BANNER + "\n"],+ [self.BANNER + "\n", "-1.25 1 0\n"],+ [self.BANNER + "\n", self.BANNER + "\n"],+ [self.BANNER + "\n", self.loaded() + "\n",+ self.loaded() + "\n"],+ [self.BANNER + "\n"],+ )+ for records in cases:+ with self.subTest(records=records):+ parser = EVAL.ScoreStdoutClassifier()+ with self.assertRaises(EVAL.EvidenceError):+ for record in records:+ parser.classify(record)+ parser.finish()++ def test_eval_main_uses_stateful_score_stream(self):+ rc, output, launches = self.run_eval_main((+ self.BANNER + "\n",+ self.loaded("absent", 2) + "\n",+ "-1 2 1\n", "-2 2 0\n", "-3 2 0\n",+ ))+ self.assertEqual(rc, 0)+ self.assertEqual(launches, 1)+ self.assertIn("# finished: 3/3", output)+ self.assertEqual(self.last_popen_kwargs["env"]["SCORE_EVIDENCE"], "1")++ def test_result_rows_are_written_and_flushed_incrementally(self):+ # A run interrupted mid-task must leave a valid partial file --+ # rows are written and flushed per-request, never buffered until+ # the run completes. Simulate a mid-run crash by having the fake+ # engine's stdout iterator raise after the first scored record;+ # the CSV must already contain that row.+ request_raw = b"2 2 1 2 1 2\n"+ request_digest = hashlib.sha256(request_raw).hexdigest()++ class Encoded:+ ids = [1, 2]++ class FakeTokenizer:+ @staticmethod+ def from_file(path):+ return FakeTokenizer()++ @staticmethod+ def encode(text):+ return Encoded()++ class CrashingStdout:+ def __init__(self, lines):+ self._lines = list(lines)++ def __iter__(self):+ for index, line in enumerate(self._lines):+ yield line+ if index == 2: # after banner+load+one SCORE row+ raise OSError("engine died mid-run")++ stdout_lines = [+ self.BANNER + "\n", self.loaded("absent", 2) + "\n",+ f"SCORE 0 {request_digest} -1 2 1\n",+ f"SCORE 1 {request_digest} -2 2 0\n",+ ]+ process = types.SimpleNamespace(+ returncode=1, stderr=(), stdout=CrashingStdout(stdout_lines),+ wait=lambda: 1, poll=lambda: 1, terminate=lambda: None)+ with tempfile.TemporaryDirectory() as tmp:+ output = pathlib.Path(tmp) / "results.csv"+ (pathlib.Path(tmp) / "config.json").write_text(+ '{"vocab_size":3}\n')+ argv = [+ "eval_glm.py", "--snap", tmp, "--tasks", "smoke",+ "--limit", "1", "--glm", "/fake/glm", "--out",+ str(output),+ ]+ tokenizers = types.SimpleNamespace(Tokenizer=FakeTokenizer)+ with mock.patch.object(sys, "argv", argv), \+ mock.patch.dict(sys.modules, {"tokenizers": tokenizers}), \+ mock.patch.object(EVAL.subprocess, "Popen",+ return_value=process), \+ self.assertRaises(OSError):+ EVAL.main()+ text = output.read_text()+ self.assertIn(",-1,1\n", text.replace(".000000", ""))+ self.assertNotIn("# finished:", text)+ # The crash-path marker itself must be present, not+ # just the absence of "# finished:" -- a downstream consumer+ # scans for this exact line to know the run never reached a+ # complete denominator.+ self.assertIn(+ "# INCOMPLETE: evaluator terminated before a complete "+ "denominator", text)++ def test_child_is_terminated_on_mid_run_interrupt_or_exception(self):+ # SIGINT/SIGTERM/any exception mid-run must not leave the+ # engine child running. The fake engine here never exits on its+ # own -- .poll() keeps returning None (as a real child that+ # ignores its stdin being closed would) until .terminate() is+ # actually called -- so a passing test proves main() called+ # terminate() itself rather than relying on the child to die.+ request_raw = b"2 2 1 2 1 2\n"+ request_digest = hashlib.sha256(request_raw).hexdigest()++ class Encoded:+ ids = [1, 2]++ class FakeTokenizer:+ @staticmethod+ def from_file(path):+ return FakeTokenizer()++ @staticmethod+ def encode(text):+ return Encoded()++ banner = self.BANNER + "\n"+ loaded = self.loaded("absent", 2) + "\n"++ def interrupting_stdout():+ yield banner+ yield loaded+ yield f"SCORE 0 {request_digest} -1 2 1\n"+ raise KeyboardInterrupt("operator pressed Ctrl+C")++ class NeverExitingProcess:+ def __init__(self):+ self.returncode = None+ self.stderr = ()+ self.stdout = interrupting_stdout()+ self.terminated = False+ self.terminate_calls = 0+ self.wait_calls = 0++ def poll(self):+ return 0 if self.terminated else None++ def terminate(self):+ self.terminated = True+ self.terminate_calls += 1++ def wait(self):+ self.wait_calls += 1+ return 0++ process = NeverExitingProcess()+ with tempfile.TemporaryDirectory() as tmp:+ output = pathlib.Path(tmp) / "results.csv"+ (pathlib.Path(tmp) / "config.json").write_text(+ '{"vocab_size":3}\n')+ argv = [+ "eval_glm.py", "--snap", tmp, "--tasks", "smoke",+ "--limit", "1", "--glm", "/fake/glm", "--out",+ str(output),+ ]+ tokenizers = types.SimpleNamespace(Tokenizer=FakeTokenizer)+ with mock.patch.object(sys, "argv", argv), \+ mock.patch.dict(sys.modules, {"tokenizers": tokenizers}), \+ mock.patch.object(EVAL.subprocess, "Popen",+ return_value=process), \+ self.assertRaises(KeyboardInterrupt):+ EVAL.main()+ self.assertEqual(process.terminate_calls, 1)+ self.assertGreaterEqual(process.wait_calls, 1)++ def test_sigterm_mid_run_terminates_the_child_and_propagates(self):+ # The SIGTERM half: a real SIGTERM (not just an ordinary+ # Python exception) delivered while the child is running must+ # also be converted into child cleanup, not left to Python's+ # default SIGTERM handling (which does not run this module's+ # `finally` cleanup at all).+ request_raw = b"2 2 1 2 1 2\n"+ request_digest = hashlib.sha256(request_raw).hexdigest()++ class Encoded:+ ids = [1, 2]++ class FakeTokenizer:+ @staticmethod+ def from_file(path):+ return FakeTokenizer()++ @staticmethod+ def encode(text):+ return Encoded()++ banner = self.BANNER + "\n"+ loaded = self.loaded("absent", 2) + "\n"++ def stdout_then_sigterm():+ yield banner+ yield loaded+ yield f"SCORE 0 {request_digest} -1 2 1\n"+ os.kill(os.getpid(), signal.SIGTERM)+ # Not reached if the handler fires promptly, as it must.+ yield f"SCORE 1 {request_digest} -2 2 0\n"++ class NeverExitingProcess:+ def __init__(self):+ self.returncode = None+ self.stderr = ()+ self.stdout = stdout_then_sigterm()+ self.terminated = False+ self.terminate_calls = 0+ self.wait_calls = 0++ def poll(self):+ return 0 if self.terminated else None++ def terminate(self):+ self.terminated = True+ self.terminate_calls += 1++ def wait(self):+ self.wait_calls += 1+ return 0++ process = NeverExitingProcess()+ previous_handler = signal.getsignal(signal.SIGTERM)+ try:+ with tempfile.TemporaryDirectory() as tmp:+ output = pathlib.Path(tmp) / "results.csv"+ (pathlib.Path(tmp) / "config.json").write_text(+ '{"vocab_size":3}\n')+ argv = [+ "eval_glm.py", "--snap", tmp, "--tasks", "smoke",+ "--limit", "1", "--glm", "/fake/glm", "--out",+ str(output),+ ]+ tokenizers = types.SimpleNamespace(Tokenizer=FakeTokenizer)+ with mock.patch.object(sys, "argv", argv), \+ mock.patch.dict(sys.modules, {"tokenizers": tokenizers}), \+ mock.patch.object(EVAL.subprocess, "Popen",+ return_value=process), \+ self.assertRaises(EVAL.ChildTerminateRequested):+ EVAL.main()+ finally:+ # Defensive: main()'s own finally already restores the prior+ # handler, but never trust a test to leave process-global+ # signal state behind if the assertion above ever fails.+ signal.signal(signal.SIGTERM, previous_handler)+ self.assertEqual(process.terminate_calls, 1)+ self.assertGreaterEqual(process.wait_calls, 1)+ self.assertEqual(signal.getsignal(signal.SIGTERM), previous_handler)++ def test_identity_bound_score_join_rejects_replay_and_order_mutations(self):+ requests = (b"1 1 1 2\n", b"1 1 1 3\n", b"1 1 1 4\n")+ digests = tuple(hashlib.sha256(raw).hexdigest()+ for raw in requests)++ def record(ordinal, digest_index, score="-1"):+ return (f"SCORE {ordinal} {digests[digest_index]} "+ f"{score} 1 1\n")++ control = EVAL.ScoreStdoutClassifier(digests)+ self.assertIsNone(control.classify(self.BANNER + "\n"))+ self.assertIsNone(control.classify(self.loaded("absent", 2) + "\n"))+ for index in range(3):+ self.assertEqual(control.classify(record(index, index))[0], "-1")+ control.finish()++ cases = {+ "replay_digest": (record(0, 0), record(1, 0)),+ "duplicate_ordinal": (record(0, 0), record(0, 1)),+ "out_of_order": (record(1, 1),),+ # NOTE: a bare "-1 1 1\n" (no identity prefix) with no prior+ # bound record is no longer an error here -- that is the+ # legitimate legacy/UNBOUND path, covered by+ # test_legacy_engine_completes_unbound below. Mix a legacy+ # line into an ALREADY-bound stream instead, which is still+ # refused (test_mixed_bound_and_legacy_stream_refuses).+ "extra": (record(0, 0), record(1, 1), record(2, 2),+ record(3, 2)),+ }+ for name, records in cases.items():+ with self.subTest(name=name):+ parser = EVAL.ScoreStdoutClassifier(digests)+ parser.classify(self.BANNER + "\n")+ parser.classify(self.loaded("absent", 2) + "\n")+ with self.assertRaises(EVAL.EvidenceError):+ for value in records:+ parser.classify(value)+ parser.finish()++ def test_wrong_digest_at_a_correct_ordinal_is_refused_mid_stream(self):+ # Isolates digest binding from the record-count completeness+ # check above: exactly len(digests) records land at the right+ # ordinals (so a count-only bug would stay quiet), but the+ # second record's digest belongs to a different request.+ requests = (b"1 1 1 2\n", b"1 1 1 3\n", b"1 1 1 4\n")+ digests = tuple(hashlib.sha256(raw).hexdigest()+ for raw in requests)++ def record(ordinal, digest_index, score="-1"):+ return (f"SCORE {ordinal} {digests[digest_index]} "+ f"{score} 1 1\n")++ parser = EVAL.ScoreStdoutClassifier(digests)+ parser.classify(self.BANNER + "\n")+ parser.classify(self.loaded("absent", 2) + "\n")+ parser.classify(record(0, 0))+ with self.assertRaisesRegex(+ EVAL.EvidenceError,+ "digest does not match exact request bytes"):+ parser.classify(record(1, 2)) # ordinal 1, wrong digest (index 2)++ def test_legacy_engine_completes_unbound(self):+ # An engine that never emits the identity-bound+ # "SCORE <ordinal> <digest> ..." prefix -- only+ # the byte-compatible legacy three-field form -- is not a+ # failure. The run completes, every row is written, and the+ # result is marked UNBOUND (never silently treated as bound).+ rc, output, launches = self.run_eval_main((+ self.BANNER + "\n",+ self.loaded("absent", 2) + "\n",+ "-1 2 1\n", "-2 2 0\n", "-3 2 0\n",+ ), bind_evidence=False)+ self.assertEqual(rc, 0)+ self.assertEqual(launches, 1)+ self.assertIn("# finished: 3/3", output)+ self.assertIn("evidence=UNBOUND", output)+ self.assertNotIn("evidence=BOUND", output)+ # SCORE_EVIDENCE is still set for the child -- harmless to an+ # engine that never reads it (confirmed: dev's run_score has no+ # getenv("SCORE_EVIDENCE") call at all).+ self.assertEqual(self.last_popen_kwargs["env"]["SCORE_EVIDENCE"], "1")+ # The UNBOUND stderr announcement must actually be+ # printed, not just the output-file marker -- an operator+ # watching a live run only sees stderr.+ self.assertIn(+ "engine does not emit score evidence lines; results are unbound",+ self.last_stderr)++ def test_evidence_engine_completes_bound(self):+ rc, output, launches = self.run_eval_main((+ self.BANNER + "\n",+ self.loaded("absent", 2) + "\n",+ "-1 2 1\n", "-2 2 0\n", "-3 2 0\n",+ ), bind_evidence=True)+ self.assertEqual(rc, 0)+ self.assertEqual(launches, 1)+ self.assertIn("# finished: 3/3", output)+ self.assertIn("evidence=BOUND", output)+ self.assertNotIn("evidence=UNBOUND", output)+ # A bound run must never print the unbound announcement.+ self.assertNotIn("results are unbound", self.last_stderr)++ def test_mixed_bound_and_legacy_stream_refuses(self):+ requests = (b"1 1 1 2\n", b"1 1 1 3\n")+ digests = tuple(hashlib.sha256(raw).hexdigest()+ for raw in requests)+ parser = EVAL.ScoreStdoutClassifier(digests)+ parser.classify(self.BANNER + "\n")+ parser.classify(self.loaded("absent", 2) + "\n")+ parser.classify(f"SCORE 0 {digests[0]} -1 1 1\n") # bound+ with self.assertRaisesRegex(+ EVAL.EvidenceError,+ "mixes identity-bound and legacy records"):+ parser.classify("-2 1 0\n") # legacy, mid-stream switch++ # Also refused the other way around: legacy first, then bound.+ parser2 = EVAL.ScoreStdoutClassifier(digests)+ parser2.classify(self.BANNER + "\n")+ parser2.classify(self.loaded("absent", 2) + "\n")+ parser2.classify("-2 1 0\n") # legacy+ with self.assertRaisesRegex(+ EVAL.EvidenceError,+ "mixes identity-bound and legacy records"):+ parser2.classify(f"SCORE 1 {digests[1]} -1 1 1\n") # bound++ def test_digest_bound_classifier_refuses_unknown_lines(self):+ # ScoreStdoutClassifier is the class main() actually+ # constructs with request_digests -- classify_score_stdout (the+ # standalone function, covered by test_stdout_grammar_refuses_+ # unknown_records) is a separate code path main() never calls.+ # A foreign line must be refused by the digest-bound classifier+ # itself, not merely by the standalone function.+ requests = (b"1 1 1 2\n",)+ digests = tuple(hashlib.sha256(raw).hexdigest() for raw in requests)+ foreign_lines = (+ "PROF 0.001 1 1 0.000 0.000 0.000 0.000 0.000 1\n",+ "not a score line at all\n",+ "nan 1 0\n",+ "1 1 1\n", # positive logprob, shaped like a legacy record+ )+ for line in foreign_lines:+ with self.subTest(line=line):+ parser = EVAL.ScoreStdoutClassifier(digests)+ parser.classify(self.BANNER + "\n")+ parser.classify(self.loaded("absent", 2) + "\n")+ with self.assertRaises(EVAL.EvidenceError):+ parser.classify(line)++ def test_eval_main_rejects_lifecycle_and_blank_records(self):+ banner = self.BANNER + "\n"+ loaded = self.loaded("absent", 2) + "\n"+ scores = ("-1 2 1\n", "-2 2 0\n", "-3 2 0\n")+ cases = {+ "load_before_banner": (loaded, banner) + scores,+ "score_before_load": (banner, scores[0], loaded) + scores[1:],+ "duplicate_load": (banner, loaded, loaded) + scores,+ "missing_load_at_eof": (banner,),+ "blank_before_banner": ("\n", banner, loaded) + scores,+ "blank_between_preambles": (banner, "\n", loaded) + scores,+ "blank_after_scores": (banner, loaded) + scores + ("\n",),+ }+ for name, records in cases.items():+ with self.subTest(name=name):+ rc, output, launches = self.run_eval_main(records)+ self.assertEqual(rc, 1)+ self.assertEqual(launches, 1)+ self.assertIn("# INCOMPLETE:", output)+ self.assertNotIn("# finished:", output)++ def test_only_complete_zero_exit_denominator_passes(self):+ # NOTE: completion_error's contract changed from the original+ # ported oracle -- it now matches dev's own exit-code contract+ # exactly (a partial or nonzero-exit-but-nonempty run is no+ # longer fatal here; see its docstring), so several of the+ # original oracle's assertions below are inverted rather than+ # reused verbatim.+ self.assertIsNone(EVAL.completion_error(0, 3, 3, 9))+ # A clean exit with zero requests scored is no longer flagged by+ # completion_error itself (dev's own contract only fires on a+ # NONZERO exit with zero scored; `expected`/`continuation_tokens`+ # are not otherwise consulted).+ self.assertIsNone(EVAL.completion_error(0, 0, 0, 0))+ self.assertIsNone(EVAL.completion_error(0, 3, 3, 0))+ # A nonzero exit that still scored at least one request is a+ # partial run, not fatal.+ self.assertIsNone(EVAL.completion_error(2, 7, 7, 7))+ self.assertIsNone(EVAL.completion_error(0, 6, 7, 6))+ # Fatal only when the engine exits nonzero with NOTHING scored...+ self.assertIsNotNone(EVAL.completion_error(2, 0, 7, 0))+ self.assertIn("zero requests scored", EVAL.completion_error(2, 0, 7, 0))+ # ...or a stream_error is present regardless of completion count.+ self.assertIn("broken", EVAL.completion_error(0, 7, 7, 7, "broken"))+ self.assertIn("broken", EVAL.completion_error(0, 0, 7, 0, "broken"))++ def test_empty_selection_refuses_before_engine_launch(self):+ with tempfile.TemporaryDirectory() as tmp:+ out = pathlib.Path(tmp) / "results.csv"+ argv = ["eval_glm.py", "--snap", tmp, "--tasks", "", "--out", str(out)]+ with mock.patch.object(sys, "argv", argv), \+ mock.patch.object(EVAL.subprocess, "Popen") as popen:+ rc = EVAL.main()+ self.assertEqual(rc, 1)+ popen.assert_not_called()+ text = out.read_text()+ self.assertIn("# INCOMPLETE: 0/0; error=no benchmark tasks selected", text)+ self.assertNotIn("finished: 0/0", text)++ def test_zero_request_task_refuses_before_engine_launch(self):+ class FakeTokenizer:+ @staticmethod+ def from_file(path):+ return object()++ with tempfile.TemporaryDirectory() as tmp:+ root = pathlib.Path(tmp)+ (root / "empty.jsonl").write_text("")+ out = root / "results.csv"+ argv = ["eval_glm.py", "--snap", tmp, "--data", tmp,+ "--tasks", "empty", "--out", str(out)]+ fake = types.SimpleNamespace(Tokenizer=FakeTokenizer)+ with mock.patch.object(sys, "argv", argv), \+ mock.patch.dict(sys.modules, {"tokenizers": fake}), \+ mock.patch.object(EVAL.subprocess, "Popen",+ side_effect=AssertionError("engine launched")) as popen:+ rc = EVAL.main()+ self.assertEqual(rc, 1)+ popen.assert_not_called()+ text = out.read_text()+ self.assertIn(+ "# INCOMPLETE: 0/0; error=selected tasks produced zero SCORE requests",+ text)+ self.assertNotIn("finished: 0/0", text)++ def test_zero_continuation_choices_refuse_before_engine_launch(self):+ class Encoded:+ def __init__(self, ids):+ self.ids = ids++ class BoundaryTokenizer:+ @staticmethod+ def from_file(path):+ return BoundaryTokenizer()++ @staticmethod+ def encode(text):+ return Encoded({+ "ctx": [1], "ctxgood": [1, 2], "good": [2],+ "ctxvanish": [1], "vanish": [], "": [],+ }.get(text, [1]))++ cases = {+ "one_empty": [{"ctx": "ctx", "choices": [""], "gold": 0}],+ "all_empty": [{"ctx": "ctx", "choices": ["", ""], "gold": 0}],+ "boundary_still_empty": [+ {"ctx": "ctx", "choices": ["vanish"], "gold": 0}],+ "mixed_positive_zero": [+ {"ctx": "ctx", "choices": ["good", ""], "gold": 0}],+ }+ tokenizer = BoundaryTokenizer()+ for name, docs in cases.items():+ with self.subTest(name=name), \+ self.assertRaisesRegex(EVAL.EvidenceError,+ "no positive context/continuation"):+ EVAL.build_requests(tokenizer, {"task": docs})++ tokenizers = types.SimpleNamespace(Tokenizer=BoundaryTokenizer)+ for name, docs in cases.items():+ with self.subTest(prelaunch=name), \+ tempfile.TemporaryDirectory() as tmp:+ root = pathlib.Path(tmp)+ data = root / f"{name}.jsonl"+ data.write_text(json.dumps(docs[0]) + "\n")+ out = root / "results.csv"+ argv = ["eval_glm.py", "--snap", str(root),+ "--data", str(root), "--tasks", name,+ "--out", str(out)]+ with mock.patch.object(sys, "argv", argv), \+ mock.patch.dict(+ sys.modules, {"tokenizers": tokenizers}), \+ mock.patch.object(+ EVAL.subprocess, "Popen",+ side_effect=AssertionError(+ "engine launched")) as popen:+ rc = EVAL.main()+ self.assertEqual(rc, 1)+ popen.assert_not_called()+ text = out.read_text()+ self.assertIn("# INCOMPLETE: 0/0; error=", text)+ self.assertNotIn("# finished:", text)++ def test_dry_run_does_not_require_a_vocabulary(self):+ # dev's own --dry never looked up config.json's+ # vocab_size at all -- it only builds and tokenizes requests,+ # then stops. This module's vocabulary/digest binding is a+ # per-request-wire step for the real engine launch, not a+ # plumbing check, so --dry must not depend on it.+ class Encoded:+ ids = [1, 2]++ class FakeTokenizer:+ @staticmethod+ def from_file(path):+ return FakeTokenizer()++ @staticmethod+ def encode(text):+ return Encoded()++ with tempfile.TemporaryDirectory() as tmp:+ # No config.json at all in the snapshot directory -- a real+ # vocabulary lookup would raise EvidenceError immediately.+ argv = ["eval_glm.py", "--snap", tmp, "--tasks", "smoke",+ "--limit", "1", "--dry"]+ tokenizers = types.SimpleNamespace(Tokenizer=FakeTokenizer)+ with mock.patch.object(sys, "argv", argv), \+ mock.patch.dict(sys.modules, {"tokenizers": tokenizers}), \+ mock.patch.object(+ EVAL, "score_snapshot_vocab",+ side_effect=AssertionError(+ "vocabulary looked up during --dry")) as vocab, \+ mock.patch.object(+ EVAL.subprocess, "Popen",+ side_effect=AssertionError("engine launched")) as popen:+ rc = EVAL.main()+ self.assertIsNone(rc)+ vocab.assert_not_called()+ popen.assert_not_called()++ def test_partial_run_still_reports_the_table_and_exits_zero(self):+ # dev's own exit-code contract (coli bench does+ # `sys.exit(subprocess.call(cmd, ...))`; diag_harness.py parses+ # this tool's stdout table from a subprocess call) exits nonzero+ # ONLY when the engine produced nothing at all. A partial run --+ # some but not all requests scored, clean stream -- must still+ # print the accuracy table and exit 0; the INCOMPLETE marker is+ # additive (alongside "# finished", not instead of it).+ rc, output, launches = self.run_eval_main((+ self.BANNER + "\n",+ self.loaded("absent", 2) + "\n",+ "-1 2 1\n", # only 1 of 3 requests scored+ ), bind_evidence=False)+ self.assertEqual(rc, 0)+ self.assertEqual(launches, 1)+ self.assertIn("# finished: 1/3", output)+ self.assertIn("# INCOMPLETE: 1/3 requests scored", output)+ self.assertIn(+ "WARNING: only 1/3 requests scored", self.last_stderr)+ self.assertIn("MEAN acc_norm", self.last_stdout)++ def test_python_engine_byte_limits_are_inclusive_and_preallocation(self):+ # The 256 MiB inclusive engine-text limit is shared between+ # this module and check_ablate_evidence.py; checked on both+ # sides of the shared constant/helper.+ engine_limit = 256 << 20+ self.assertEqual(ABLATE._ENGINE_TEXT_MAX_BYTES, engine_limit)+ self.assertEqual(EVAL._ENGINE_TEXT_MAX_BYTES, engine_limit)+ self.assertEqual(+ ABLATE._checked_engine_text_size(engine_limit, "config"),+ engine_limit)+ self.assertEqual(+ EVAL._checked_engine_text_size(engine_limit, "SCORE"),+ engine_limit)+ with self.assertRaises(ABLATE.AblateEvidenceError):+ ABLATE._checked_engine_text_size(engine_limit + 1, "config")+ with self.assertRaises(EVAL.EvidenceError):+ EVAL._checked_engine_text_size(engine_limit + 1, "SCORE")++ with tempfile.TemporaryDirectory() as tmp:+ root = pathlib.Path(tmp)+ ablate_config = root / "ablate-config.json"+ ablate_config.write_bytes(self.CONFIG)+ with mock.patch.object(+ ABLATE, "_ENGINE_TEXT_MAX_BYTES", len(self.CONFIG)):+ identity = ABLATE._config_identity(ablate_config)+ self.assertEqual(identity["vocab"], 4)+ self.assertEqual(+ identity["config_sha256"],+ hashlib.sha256(self.CONFIG).hexdigest())+ ablate_config.write_bytes(self.CONFIG + b" ")+ with self.assertRaisesRegex(+ ABLATE.AblateEvidenceError, "256 MiB"):+ ABLATE._config_identity(ablate_config)++ score_config = root / "config.json"+ score_raw = b'{"vocab_size":4}\n'+ score_config.write_bytes(score_raw)+ with mock.patch.object(+ EVAL, "_ENGINE_TEXT_MAX_BYTES", len(score_raw)):+ self.assertEqual(EVAL.score_snapshot_vocab(root), 4)+ score_config.write_bytes(score_raw + b" ")+ with self.assertRaisesRegex(EVAL.EvidenceError, "256 MiB"):+ EVAL.score_snapshot_vocab(root)++ request = "1 1 0 1"+ request_bytes = len((request + "\n").encode("ascii"))+ with mock.patch.object(+ EVAL, "_ENGINE_TEXT_MAX_BYTES", request_bytes):+ _, payload, _ = EVAL.score_request_wire((request,), 4)+ self.assertEqual(len(payload), request_bytes)+ with mock.patch.object(+ EVAL, "_ENGINE_TEXT_MAX_BYTES", request_bytes - 1):+ with self.assertRaisesRegex(EVAL.EvidenceError, "256 MiB"):+ EVAL.score_request_wire((request,), 4)+++class DifferentialBiteTests(unittest.TestCase):+ """dev's classifier silently accepts a foreign stdout line the new+ copy refuses. dev's side is not itself invoked here (no ported dev+ module exists in this tree); it is asserted against the same fixture+ line via dev's documented filter logic, ported verbatim inline. Only+ the new copy's refusal is exercised by calling real code."""++ FOREIGN_LINE = "1 1 1\n" # shaped like dev's own accepted grammar++ def test_dev_copy_silently_accepts_a_foreign_numeric_line(self):+ # dev's inline stdout filter (ported verbatim as the oracle): any+ # line starting with a digit or '-' is treated as a SCORE record,+ # with no further validation at all.+ line = self.FOREIGN_LINE.strip()+ self.assertTrue(line and line[0] in "-0123456789")+ parts = line.split()+ logprob = float(parts[0]) # dev: "try: logprob = float(parts[0])"+ # dev accepts this as a real SCORE result -- a false positive: a+ # log-likelihood can never be positive, but dev's filter never+ # checks the sign (or finiteness, or field count) at all.+ self.assertEqual(logprob, 1.0)++ def test_new_copy_refuses_the_same_foreign_line_by_name(self):+ with self.assertRaisesRegex(+ EVAL.EvidenceError, "SCORE logprob is not finite/non-positive"):+ EVAL.classify_score_stdout(self.FOREIGN_LINE)+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tools/eval_glm.py b/c/tools/eval_glm.pyindex 76097cd6..4e393108 100644--- a/c/tools/eval_glm.py+++ b/c/tools/eval_glm.py@@ -21,8 +21,40 @@ USO:--tasks hellaswag,arc_challenge,mmlu --limit 40 --ram 15# leve di ricerca: passate al motore via envTOPP=0.9 python3 tools/eval_glm.py --snap /path/to/glm52_i4 --data ./bench --tasks mmlu --ram 15++Evidence binding (current limitation): this harness always sets+SCORE_EVIDENCE=1 in the child environment and always tries to bind each+SCORE result to the exact request bytes that produced it by digest. Only+an engine build that prints the identity-bound wire form -- "SCORE+<ordinal> <sha256-of-request-line> <exact> <contlen> <greedy>" -- can+satisfy that binding; SCORE_EVIDENCE is a plain, unread environment+variable to every other engine build, harmless to set. When the engine+instead prints only the byte-compatible legacy three-field form ("<exact>+<contlen> <greedy>", no identity prefix), the run still completes+normally and every row is still written, but the results are UNBOUND --+marked as such in the output file's summary line and announced once on+stderr -- rather than silently treated as bound. A stream that mixes both+forms in one run is refused with a named error instead of guessed at."""-import os, sys, subprocess, argparse, random, json, tempfile, time, threading+import argparse+import hashlib+import json+import math+import os+import random+import re+import signal+import subprocess+import sys+import tempfile+import threading+import time++_TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))+if _TOOLS_DIR not in sys.path:+ sys.path.insert(0, _TOOLS_DIR)+from engine_evidence import (PreambleError, parse_engine_banner,+ parse_engine_loaded, parse_engine_preamble)# mini-set OFFLINE per testare la meccanica (NON misura qualita': domande banali)SMOKE = [@@ -38,6 +70,277 @@ REFERENCE = {"arc_challenge": {"GLM-5.2 (pubbl.)": None},}++class EvidenceError(ValueError):+ """A SCORE stream cannot support a complete, finite evidence result."""+++class ChildTerminateRequested(BaseException):+ """A termination signal (SIGTERM) arrived while the engine child was+ running. Raised from a signal handler installed only for the+ duration of that child's run, so it unwinds through the same+ ``finally`` cleanup as any other mid-run exception (including+ Python's own SIGINT-to-KeyboardInterrupt) and the child is never+ left running as a zombie/orphan.+ """+++_INT32_MAX = 2**31 - 1+_ENGINE_TEXT_MAX_BYTES = 256 << 20+_UINT_TEXT = r"(?:0|[1-9][0-9]*)"+_C17G_TEXT = (r"-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?"+ r"(?:e[+-](?:0[0-9]|[1-9][0-9]{1,2}))?")+_SCORE_RE = re.compile(+ rf"^({_C17G_TEXT}) ({_UINT_TEXT}) ([01])$")+_SCORE_EVIDENCE_RE = re.compile(+ rf"^SCORE ({_UINT_TEXT}) ([0-9a-f]{{64}}) "+ rf"({_C17G_TEXT}) ({_UINT_TEXT}) ([01])$")+++def _checked_engine_text_size(length, label):+ if type(length) is not int or not 0 <= length <= _ENGINE_TEXT_MAX_BYTES:+ raise EvidenceError(+ f"{label} exceeds the inclusive 256 MiB engine limit")+ return length+++def parse_c17g(text):+ """Parse the canonical C-locale numeric token the engine actually emits.++ The engine has shipped two mutually exclusive SCORE spellings across+ its history -- the byte-compatible ``printf("%.6f")`` form dev still+ emits today, and the newer opt-in ``printf("%.17g")`` evidence form.+ Both are exact, finite, round-trippable spellings of the same+ C-locale numeric domain, so both are accepted here (the function name+ is kept for the newer form this module's identity checks depend on);+ a text that is neither exact spelling -- including any non-canonical+ variant such as a non-canonical ``%.17g`` corpus (nan/inf/-inf never+ survive: neither spelling is finite-preserving for them) -- is+ refused.+ """+ if not isinstance(text, str) or not re.fullmatch(_C17G_TEXT, text):+ raise EvidenceError(f"not a canonical %.6f/%.17g token: {text!r}")+ try:+ value = float(text)+ except ValueError as exc:+ raise EvidenceError(f"malformed %.6f/%.17g token: {text!r}") from exc+ if (not math.isfinite(value) or+ (format(value, ".17g") != text and format(value, ".6f") != text)):+ raise EvidenceError(+ f"not an exact finite %.6f/%.17g spelling: {text!r}")+ return value+++def is_score_preamble(line):+ """Validate one of the two exact stdout records emitted before SCORE."""+ try:+ return parse_engine_preamble(line) is not None+ except PreambleError:+ return False+++def parse_score_result(line):+ """Return (exact_text, value, contlen, greedy) for one complete SCORE line."""+ match = _SCORE_RE.fullmatch(line)+ if not match:+ raise EvidenceError(f"not an exact SCORE record: {line!r}")+ exact, contlen_text, greedy_text = match.groups()+ try:+ value = parse_c17g(exact)+ contlen = int(contlen_text)+ greedy = int(greedy_text)+ except ValueError as exc:+ raise EvidenceError(f"malformed SCORE fields: {line!r}") from exc+ if not math.isfinite(value) or value > 0.0:+ raise EvidenceError(f"SCORE logprob is not finite/non-positive: {exact}")+ if not 1 <= contlen <= _INT32_MAX or greedy not in (0, 1):+ raise EvidenceError(f"invalid SCORE metadata: {line!r}")+ return exact, value, contlen, greedy+++def parse_score_evidence_result(line):+ """Return the strict ordinal/digest identity plus the SCORE payload."""+ match = _SCORE_EVIDENCE_RE.fullmatch(line)+ if not match:+ raise EvidenceError(f"not an exact evidence SCORE record: {line!r}")+ ordinal_text, digest, exact, contlen_text, greedy_text = match.groups()+ ordinal = int(ordinal_text)+ if ordinal > _INT32_MAX:+ raise EvidenceError(f"SCORE request ordinal is outside int32: {line!r}")+ parsed = parse_score_result(f"{exact} {contlen_text} {greedy_text}")+ return ordinal, digest, *parsed+++def classify_score_stdout(raw_line):+ """Accept one exact newline-terminated production stdout record."""+ if (not raw_line.endswith("\n") or raw_line.count("\n") != 1 or+ "\r" in raw_line):+ raise EvidenceError(f"unterminated/non-canonical stdout record: {raw_line!r}")+ line = raw_line[:-1]+ if not line:+ raise EvidenceError("blank SCORE stdout record")+ try:+ preamble = parse_engine_preamble(line)+ except PreambleError as exc:+ raise EvidenceError(str(exc)) from exc+ if preamble is not None:+ return None+ return parse_score_result(line)+++class ScoreStdoutClassifier:+ """Own exactly one banner then one load record before SCORE results.++ Only an engine build that prints the identity-bound wire form+ (``SCORE <ordinal> <sha256-of-request-line> <exact> <contlen>+ <greedy>``) lets results be BOUND to their originating request by+ digest. An engine that prints only the byte-compatible legacy form+ (``<exact> <contlen> <greedy>``, no identity prefix) is still a+ complete, honest run -- this is not a failure -- but nothing ties any+ individual result back to the request that produced it, so the run's+ results are UNBOUND. The two forms are never silently conflated: a+ stream that starts in one form and switches to the other mid-run+ (the classic case would be corrupted/interleaved output) is refused+ with a named error rather than guessed at.++ Beyond the banner/load preamble, no other multiplexed-serve global+ record (``PROF``, ``HITS``, ``EMAP``, ...) or stderr-only banner+ (``[prefill]``, ``[PIN]``, ``[USAGE]``, ...) can ever reach this+ classifier: ``run_score`` never prints them to stdout (verified+ against the engine source), so any such line arriving here is+ refused by name like any other unrecognized record, not specially+ recognized or passed through.+ """++ def __init__(self, request_digests=None):+ self._state = 0+ self._request_digests = (None if request_digests is None+ else tuple(request_digests))+ self._result_index = 0+ self._mode = None # None (no SCORE record yet) | "bound" | "unbound"++ @property+ def binding_mode(self):+ """"bound", "unbound", or None if no SCORE record was classified."""+ return self._mode++ def classify(self, raw_line):+ if (not raw_line.endswith("\n") or raw_line.count("\n") != 1 or+ "\r" in raw_line):+ raise EvidenceError(+ f"unterminated/non-canonical stdout record: {raw_line!r}")+ line = raw_line[:-1]+ if not line:+ raise EvidenceError("blank SCORE stdout record")+ try:+ if self._state == 0:+ parse_engine_banner(line)+ self._state = 1+ return None+ if self._state == 1:+ parse_engine_loaded(line)+ self._state = 2+ return None+ preamble = parse_engine_preamble(line)+ except PreambleError as exc:+ raise EvidenceError(str(exc)) from exc+ if preamble is not None:+ raise EvidenceError(f"duplicate/out-of-order SCORE preamble: {line!r}")+ if self._request_digests is None:+ return parse_score_result(line)+ is_bound_shape = _SCORE_EVIDENCE_RE.fullmatch(line) is not None+ is_unbound_shape = not is_bound_shape and _SCORE_RE.fullmatch(line) is not None+ if not is_bound_shape and not is_unbound_shape:+ raise EvidenceError(f"not an exact SCORE record: {line!r}")+ line_mode = "bound" if is_bound_shape else "unbound"+ if self._mode is None:+ self._mode = line_mode+ elif self._mode != line_mode:+ raise EvidenceError(+ "SCORE stream mixes identity-bound and legacy records: "+ f"{line!r}")+ if line_mode == "unbound":+ self._result_index += 1+ return parse_score_result(line)+ ordinal, digest, exact, value, contlen, greedy = \+ parse_score_evidence_result(line)+ if self._result_index >= len(self._request_digests):+ raise EvidenceError("engine emitted extra evidence SCORE result lines")+ if ordinal != self._result_index:+ raise EvidenceError(+ f"SCORE request ordinal {ordinal} != expected {self._result_index}")+ expected_digest = self._request_digests[self._result_index]+ if digest != expected_digest:+ raise EvidenceError(+ f"SCORE request {ordinal} digest does not match exact request bytes")+ self._result_index += 1+ return exact, value, contlen, greedy++ def finish(self):+ if self._state != 2:+ missing = "engine banner" if self._state == 0 else "engine load record"+ raise EvidenceError(f"missing {missing} before SCORE EOF")+ if (self._mode == "bound" and self._request_digests is not None and+ self._result_index != len(self._request_digests)):+ raise EvidenceError(+ f"only {self._result_index}/{len(self._request_digests)} "+ "identity-bound SCORE records")+++def completion_error(returncode, completed, expected, continuation_tokens,+ stream_error=None):+ """Return the reason this run has NOTHING trustworthy to report, or+ None if it can report a table (even a partial one).++ Matches dev's own exit-code contract exactly, since callers such as+ ``coli bench`` (``sys.exit(subprocess.call(cmd, ...))``) and+ ``diag_harness.py`` (which parses this tool's own accuracy table from+ a subprocess call) depend on it: dev exits nonzero ONLY when the+ engine itself produced nothing at all (nonzero exit AND zero+ requests scored); a partial run -- some but not all requests scored,+ or the engine exiting nonzero after scoring at least one request --+ still prints the accuracy table over whatever landed and exits 0.+ Nothing here checks ``expected``/``continuation_tokens`` against a+ denominator: by the time this runs, ``expected`` (the request count)+ is always positive (an empty selection is refused before the engine+ ever launches), and a positive ``completed`` count always carries a+ positive token count by construction.++ ``stream_error`` is the one condition dev's own contract has no+ analog for: a genuinely corrupted or self-inconsistent SCORE stream+ (mixed identity-bound/legacy records, a replayed digest, an+ out-of-vocabulary token, ...), which this module's evidence layer can+ detect and dev's plain per-line filter cannot. That failure stays+ fatal regardless of how many requests completed, because the parsed+ numbers themselves are not trustworthy.+ """+ if stream_error:+ return str(stream_error)+ if returncode != 0 and completed == 0:+ return f"engine exited {returncode} with zero requests scored"+ return None+++def write_result_row(out_f, req_idx, meta_row, exact_logprob, greedy):+ """Write the exact engine token, never a rounded float reconstruction."""+ task, qi, oi, clen, cchars, gold = meta_row+ out_f.write(f"{req_idx},{task},{qi},{oi},{clen},{cchars},{gold},"+ f"{exact_logprob},{greedy}\n")+++def prelaunch_incomplete(out_path, reason):+ """Refuse vacuous evidence before Popen and durably mark writable output."""+ message = f"EVIDENCE INCOMPLETE before engine launch: {reason}"+ print(message, file=sys.stderr)+ if out_path:+ try:+ with open(out_path, "a") as out_f:+ out_f.write(f"# INCOMPLETE: 0/0; error={reason}\n")+ except OSError as exc:+ print(f"cannot mark output {out_path!r} INCOMPLETE: {exc}",+ file=sys.stderr)+ return 1+def load_docs(task, data_dir, limit, seed):if task == "smoke":return SMOKE[:limit] if limit else SMOKE@@ -72,14 +375,88 @@ def build_requests(tk, docs_by_task, prefix=""):cl = len(ctx_ids)while cl > 0 and (cl > len(full) or full[:cl] != ctx_ids[:cl]): cl -= 1cont_ids = full[cl:]- if not cont_ids: # boundary degenere: forza split esplicito- full = ctx_ids + tk.encode(cont).ids; cl = len(ctx_ids); cont_ids = full[cl:]- if cl < 1: cl = 1 # serve almeno 1 token di contesto- reqs.append(f"{cl} {len(full)-cl} " + " ".join(map(str, full)))- meta.append((t, qi, oi, len(full) - cl, max(1, len(cont)), gold))+ if cl < 1 or not cont_ids: # boundary degenere: forza split esplicito+ choice_ids = tk.encode(cont).ids+ full = ctx_ids + choice_ids+ cl = len(ctx_ids)+ cont_ids = choice_ids+ if cl < 1 or not cont_ids:+ raise EvidenceError(+ f"{t} question {qi} choice {oi} has no positive "+ "context/continuation token denominator")+ reqs.append(f"{cl} {len(cont_ids)} " + " ".join(map(str, full)))+ meta.append((t, qi, oi, len(cont_ids), max(1, len(cont)), gold))perq.setdefault((t, qi), []).append(len(meta) - 1)return reqs, meta, perq++def score_snapshot_vocab(snap):+ """Read the engine's independently loaded vocabulary bound from config."""+ path = os.path.join(snap, "config.json")+ try:+ with open(path, "rb") as source:+ source.seek(0, os.SEEK_END)+ length = source.tell()+ _checked_engine_text_size(length, "SCORE config.json")+ source.seek(0)+ raw = source.read(_ENGINE_TEXT_MAX_BYTES + 1)+ _checked_engine_text_size(len(raw), "SCORE config.json")+ config = json.loads(raw.decode("utf-8"))+ vocab = config["vocab_size"]+ except (OSError, UnicodeDecodeError, json.JSONDecodeError,+ KeyError, TypeError) as exc:+ raise EvidenceError(f"cannot derive SCORE vocabulary from {path}: {exc}") from exc+ if type(vocab) is not int or not 1 <= vocab <= 1 << 24:+ raise EvidenceError(f"invalid SCORE vocabulary in {path}: {vocab!r}")+ return vocab+++def score_request_wire(requests, vocab):+ """Return strict ASCII/LF records, joined bytes, and per-record SHA-256.++ The C SCORE evidence mode hashes the ``getline`` byte span, including LF;+ this helper owns the identical byte domain before the temporary file exists.+ """+ if type(vocab) is not int or not 1 <= vocab <= 1 << 24:+ raise EvidenceError(f"invalid SCORE vocabulary: {vocab!r}")+ lines = []+ continuation_tokens = 0+ image_bytes = 0+ for request in requests:+ if not isinstance(request, str) or not request or "\n" in request or "\r" in request:+ raise EvidenceError(f"request is not one canonical line: {request!r}")+ fields = request.split(" ")+ if (" ".join(fields) != request or len(fields) < 4 or+ any(not re.fullmatch(_UINT_TEXT, field) for field in fields)):+ raise EvidenceError(f"request is not canonical integer grammar: {request!r}")+ values = [int(field) for field in fields]+ ctxlen, contlen = values[:2]+ if (not 1 <= ctxlen <= _INT32_MAX or+ not 1 <= contlen <= _INT32_MAX - ctxlen):+ raise EvidenceError(f"request lengths are invalid: {request!r}")+ total = ctxlen + contlen+ tokens = values[2:]+ if len(tokens) != total:+ raise EvidenceError(f"request token count is invalid: {request!r}")+ if any(token >= vocab for token in tokens):+ raise EvidenceError(f"request token is outside vocabulary: {request!r}")+ if continuation_tokens > (1 << 63) - 1 - contlen:+ raise EvidenceError("SCORE continuation denominator exceeds int64")+ continuation_tokens += contlen+ try:+ line = (request + "\n").encode("ascii")+ except UnicodeEncodeError as exc:+ raise EvidenceError(+ f"request is not canonical ASCII: {request!r}") from exc+ image_bytes = _checked_engine_text_size(+ image_bytes + len(line), "SCORE request image")+ lines.append(line)+ if not lines or continuation_tokens <= 0 or len(lines) > _INT32_MAX:+ raise EvidenceError("SCORE request image has no positive denominator")+ frozen = tuple(lines)+ return (frozen, b"".join(frozen),+ tuple(hashlib.sha256(line).hexdigest() for line in frozen))+def score_accuracy(tasks, meta, perq, lp):print(f"\n{'task':<18} {'n':>4} {'acc':>7} {'acc_norm':>9}")overall = []@@ -123,24 +500,45 @@ def main():score_accuracy(["t"], meta, perq, lp)print("selftest OK" if True else ""); return+ tasks = [t.strip() for t in a.tasks.split(",") if t.strip()]+ if not tasks:+ return prelaunch_incomplete(a.out,"no benchmark tasks selected")+from tokenizers import Tokenizertk = Tokenizer.from_file(os.path.join(a.snap, "tokenizer.json"))- tasks = [t.strip() for t in a.tasks.split(",") if t.strip()]docs_by_task = {t: load_docs(t, a.data, a.limit, a.seed) for t in tasks}for t, d in docs_by_task.items(): print(f"[{t}] {len(d)} questions", file=sys.stderr)- reqs, meta, perq = build_requests(tk, docs_by_task, detect_prefix(a.snap))+ try:+ reqs, meta, perq = build_requests(+ tk, docs_by_task, detect_prefix(a.snap))+ except EvidenceError as exc:+ return prelaunch_incomplete(a.out, str(exc))print(f"total requests: {len(reqs)} (answer options)", file=sys.stderr)+ if not reqs:+ return prelaunch_incomplete(a.out,"selected tasks produced zero SCORE requests")if a.dry:+ # Matches dev exactly: --dry stops right after request+ # construction, before the vocabulary lookup below -- it never+ # needed config.json's vocab_size (a plumbing check has no+ # engine, and therefore no vocabulary, to bind requests against).for r in reqs[:3]: print(" example request:", r[:80], "...", file=sys.stderr)print("DRY: request construction and tokenization passed. Engine was not run.", file=sys.stderr); return+ try:+ score_vocab = score_snapshot_vocab(a.snap)+ _, request_payload, request_digests = score_request_wire(+ reqs, score_vocab)+ except EvidenceError as exc:+ return prelaunch_incomplete(a.out, str(exc))# mkstemp (non mktemp): crea il file atomicamente con permessi 0600, niente# race TOCTOU/symlink su una tmp dir condivisa (CWE-377).fd, req_path = tempfile.mkstemp(suffix=".txt")- with os.fdopen(fd, "w") as f:- f.write("\n".join(reqs) + "\n")- env = dict(os.environ, SNAP=a.snap, SCORE=req_path)+ with os.fdopen(fd, "wb") as f:+ written = f.write(request_payload)+ if written != len(request_payload):+ raise EvidenceError("short write while freezing SCORE requests")+ env = dict(os.environ, SNAP=a.snap, SCORE=req_path, SCORE_EVIDENCE="1")if a.ram: env["RAM_GB"] = str(a.ram)cmd = [a.glm, str(a.cap)] + a.bits.split()print("running:", " ".join(cmd), file=sys.stderr)@@ -155,55 +553,137 @@ def main():out_f.write("req_idx,task,qi,oi,contlen,contchars,gold,logprob,greedy\n")out_f.flush()t0 = time.time()- proc = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE,- text=True, bufsize=1) # line-buffered- lp = [None] * len(reqs)- n_done = 0- # Drain stderr (engine progress lines) to console live on a background thread- # so the [score N req] heartbeat is visible while stdout is consumed below.- def _drain_stderr():- for line in proc.stderr:- print(f" [engine] {line.rstrip()}", file=sys.stderr)- threading.Thread(target=_drain_stderr, daemon=True).start()- for line in proc.stdout:- line = line.strip()- if not line or line[0] not in "-0123456789": continue- parts = line.split()- if n_done >= len(reqs): break- try: logprob = float(parts[0])- except (ValueError, IndexError): continue- lp[n_done] = logprob- greedy = parts[2] if len(parts) > 2 else "?"- t, qi, oi, clen, cchars, gold = meta[n_done]+ proc = None+ previous_sigterm = None++ def _on_sigterm(signum, frame):+ raise ChildTerminateRequested(f"received signal {signum}")++ try:+ # A SIGTERM (or Ctrl+C's SIGINT, which Python already converts to+ # KeyboardInterrupt on its own) must not leave the engine child+ # running as an orphan/zombie -- the handler below converts+ # SIGTERM into the same exception path, so both unwind through+ # the identical `finally` cleanup that terminates the child.+ previous_sigterm = signal.signal(signal.SIGTERM, _on_sigterm)+ proc = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE,+ text=True, bufsize=1) # line-buffered+ lp = [None] * len(reqs)+ n_done = 0+ continuation_tokens = 0+ stream_error = None+ stdout_classifier = ScoreStdoutClassifier(request_digests)+ # Drain stderr (engine progress lines) to console live on a background thread+ # so the [score N req] heartbeat is visible while stdout is consumed below.+ def _drain_stderr():+ for line in proc.stderr:+ print(f" [engine] {line.rstrip()}", file=sys.stderr)+ threading.Thread(target=_drain_stderr, daemon=True).start()+ for raw_line in proc.stdout:+ if stream_error:+ continue # drain fully so the child cannot block+ try:+ result = stdout_classifier.classify(raw_line)+ except EvidenceError as exc:+ stream_error = exc+ continue+ if result is None:+ continue+ if n_done >= len(reqs):+ stream_error = EvidenceError("engine emitted extra SCORE result lines")+ continue+ try:+ exact, logprob, contlen, greedy = result+ if contlen != meta[n_done][3]:+ raise EvidenceError(+ f"request {n_done} contlen {contlen} != expected {meta[n_done][3]}")+ except EvidenceError as exc:+ stream_error = exc+ continue+ lp[n_done] = logprob+ continuation_tokens += contlen+ t, qi, oi, clen, cchars, gold = meta[n_done]+ if out_f:+ write_result_row(out_f,n_done,meta[n_done],exact,greedy)+ out_f.flush()+ n_done += 1+ if n_done % 5 == 0 or n_done == len(reqs):+ elapsed = time.time() - t0+ rate = n_done / elapsed if elapsed > 0 else 0+ eta = (len(reqs) - n_done) / rate if rate > 0 else 0+ print(f"[progress] {n_done}/{len(reqs)} requests scored | {elapsed:.0f}s elapsed | "+ f"{rate:.2f} req/s | ETA {eta:.0f}s | last: {t} q{qi} opt{oi} lp={logprob:.3f}",+ file=sys.stderr)+ if not stream_error:+ try:+ stdout_classifier.finish()+ except EvidenceError as exc:+ stream_error = exc+ binding_mode = stdout_classifier.binding_mode+ proc.wait()+ elapsed = time.time() - t0+ # Fatal only in the two cases dev's own contract (and this+ # module's own evidence layer) recognize -- see completion_error's+ # docstring. Everything else, including a partial request count,+ # still reports the accuracy table and exits 0, matching dev.+ fatal = completion_error(+ proc.returncode,n_done,len(reqs),continuation_tokens,stream_error)+ partial = n_done != len(reqs)+ evidence_status = "BOUND" if binding_mode == "bound" else "UNBOUND"if out_f:- out_f.write(f"{n_done},{t},{qi},{oi},{clen},{cchars},{gold},{logprob:.6f},{greedy}\n")- out_f.flush()- n_done += 1- if n_done % 5 == 0 or n_done == len(reqs):- elapsed = time.time() - t0- rate = n_done / elapsed if elapsed > 0 else 0- eta = (len(reqs) - n_done) / rate if rate > 0 else 0- print(f"[progress] {n_done}/{len(reqs)} requests scored | {elapsed:.0f}s elapsed | "- f"{rate:.2f} req/s | ETA {eta:.0f}s | last: {t} q{qi} opt{oi} lp={logprob:.3f}",- file=sys.stderr)- proc.wait()- elapsed = time.time() - t0- if out_f:- out_f.write(f"# finished: {n_done}/{len(reqs)} in {elapsed:.0f}s, exit={proc.returncode}\n")- out_f.close()- if proc.returncode != 0 and n_done == 0:- print(f"ENGINE ERROR (exit {proc.returncode})", file=sys.stderr); sys.exit(1)- if n_done != len(reqs):- print(f"WARNING: only {n_done}/{len(reqs)} requests scored (engine exited {proc.returncode}); "- f"scoring partial results.", file=sys.stderr)- # Fill any unscored slots with -inf so argmax never picks them- for i in range(len(lp)):- if lp[i] is None: lp[i] = float("-inf")- print(f"(engine: {elapsed:.0f}s, {n_done}/{len(reqs)} scored, exit {proc.returncode})", file=sys.stderr)- score_accuracy(tasks, meta, perq, lp)- print("\nNOTE: compare acc_norm with GLM-5.2's PUBLISHED model-card score. A close result"- "\n indicates that int4 quantization preserved quality. (Fill REFERENCE in tools/eval_glm.py.)")- os.remove(req_path)+ if fatal:+ out_f.write(f"# INCOMPLETE: {n_done}/{len(reqs)} in {elapsed:.0f}s, "+ f"tokens={continuation_tokens}, exit={proc.returncode}; "+ f"error={fatal}\n")+ else:+ out_f.write(f"# finished: {n_done}/{len(reqs)} in {elapsed:.0f}s, "+ f"tokens={continuation_tokens}, exit={proc.returncode}, "+ f"evidence={evidence_status}\n")+ if partial:+ # Additive: the run still finished (exit 0, full+ # table below) -- this line only ANNOUNCES that fewer+ # than the full request count landed, it does not+ # replace the "# finished" line or change the exit+ # code dev's own consumers depend on.+ out_f.write(f"# INCOMPLETE: {n_done}/{len(reqs)} requests "+ f"scored; engine exit={proc.returncode}\n")+ out_f.close(); out_f=None+ if fatal:+ print(f"EVIDENCE INCOMPLETE: {fatal}", file=sys.stderr)+ return 1+ if evidence_status == "UNBOUND":+ print("engine does not emit score evidence lines; "+ "results are unbound", file=sys.stderr)+ if partial:+ # Same wording and the same exit-0 contract dev used: a+ # partial run is a WARNING, not a failure.+ print(f"WARNING: only {n_done}/{len(reqs)} requests scored "+ f"(engine exited {proc.returncode}); scoring partial "+ "results.", file=sys.stderr)+ # Fill any unscored slots with -inf so argmax never picks them+ # (dev's own fallback for a partial run).+ for i in range(len(lp)):+ if lp[i] is None: lp[i] = float("-inf")+ print(f"(engine: {elapsed:.0f}s, {n_done}/{len(reqs)} scored, "+ f"{continuation_tokens} continuation tokens, "+ f"exit {proc.returncode})", file=sys.stderr)+ score_accuracy(tasks, meta, perq, lp)+ print("\nNOTE: compare acc_norm with GLM-5.2's PUBLISHED model-card score. A close result"+ "\n indicates that int4 quantization preserved quality. (Fill REFERENCE in tools/eval_glm.py.)")+ return 0+ finally:+ if previous_sigterm is not None:+ signal.signal(signal.SIGTERM, previous_sigterm)+ if proc is not None and proc.poll() is None:+ # A mid-run exception or termination signal must never leave+ # the engine child running: no zombie, no orphan.+ proc.terminate()+ proc.wait()+ if out_f:+ out_f.write("# INCOMPLETE: evaluator terminated before a complete denominator\n")+ out_f.close()+ try: os.remove(req_path)+ except FileNotFoundError: passif __name__ == "__main__":- main()+ sys.exit(main() or 0)