Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
50.2 KB
Raw Blame
"""tools/eval_glm.py must accept only the engine's real SCORE stdout
records and refuse everything else with a named error: an unparsed or
duplicated banner/load preamble, a SCORE line whose numeric token is not
the canonical finite ``%.6f``/``%.17g`` spelling the engine actually
emits, and a foreign stdout line that is neither a preamble nor a SCORE
record. It must also write result rows incrementally (one flush per
request, never buffered until completion) and mark incomplete runs,
including a pre-launch refusal, before the engine is ever started.

Checks enumerated from the source (`tools/eval_glm.py`, read in full
before writing this module) and covered below, grouped by the function
that performs them:

- `parse_c17g`: both exact finite spellings the engine actually emits --
  the shipped ``%.6f`` form and the newer ``%.17g`` evidence form -- and
  nothing else; this module imports nothing from and shares no code with
  `check_data_logprob_gaps.py`, which parses its own independent grammar.
- `parse_score_result` / `_SCORE_RE`: the shipped ``%.6f`` form AND the
  ``%.17g`` evidence form (both numeric forms), non-finite/malformed
  rejection, and the ``<contlen> <greedy>`` metadata bounds.
- `classify_score_stdout` / `ScoreStdoutClassifier`: the banner/load
  preamble lifecycle (missing, duplicated, out-of-order, or a SCORE
  record before the load record all refuse); every line that is not an
  exact banner, an exact load record, or an exact SCORE record refuses
  by name -- a foreign line is never silently treated as a score, which
  is exactly the defect dev's plain ``line[0] in "-0123456789"`` filter
  does not catch (differential bite, below).
- `score_request_wire`: strict ASCII/LF request grammar, the per-record
  SHA-256 digest, and the inclusive 256 MiB engine text limit.
- `completion_error`: the exact zero-exit/complete-count/positive-token
  denominator that alone passes.
- `main`: incremental durability (one written+flushed row per completed
  request, never buffered until the run ends) and pre-launch INCOMPLETE
  marking (no benchmark tasks selected; zero SCORE requests produced;
  every choice's context/continuation split is empty) -- the engine is
  never launched for any of these; a mid-run crash or interrupt still
  leaves the INCOMPLETE marker and terminates the child process; a
  partial run still prints the accuracy table over whatever rows landed.

Deferred (need a live binary this module does not have access to):
- `test_c_emitted_c17g_corpus_is_canonical`, which drives
  ``test_logprob_wire --score-c17g-fixture`` (a binary produced by a
  different part of this project's build, not present here).

No model is run by the committed tests -- every case here drives
`eval_glm.py` against an injected stand-in for the direct engine launch
(a fake ``subprocess.Popen`` returning canned stdout/stderr), never a
real ``./glm`` process.
"""

import contextlib
import hashlib
import importlib.util
import io
import json
import os
import pathlib
import signal
import sys
import tempfile
import types
import unittest
from unittest import mock


HERE = pathlib.Path(__file__).resolve().parent
TOOLS = HERE.parent / "tools"

_spec = importlib.util.spec_from_file_location(
    "eval_glm_under_test", TOOLS / "eval_glm.py")
EVAL = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(EVAL)


class EvalGlmEvidenceTests(unittest.TestCase):
    BANNER = (
        "== GLM C engine (glm_moe_dsa), cache=64 experts/layer | "
        "compute experts@4-bit dense@8-bit | idot: neon-i8mm ==")

    @staticmethod
    def loaded(state="ACTIVE", draft=1, layers=78, experts=256,
               load="1.00", resident="1.00"):
        return (f"loaded in {load}s | resident dense: {resident} MB | "
                f"layers={layers} experts={experts} | MTP {state} "
                f"(draft={draft})")

    def run_eval_main(self, stdout_records):
        class Encoded:
            ids = [1, 2]

        class FakeTokenizer:
            @staticmethod
            def from_file(path):
                return FakeTokenizer()

            @staticmethod
            def encode(text):
                return Encoded()

        process = types.SimpleNamespace(
            returncode=0, stderr=(), stdout=tuple(stdout_records),
            wait=lambda: 0, poll=lambda: 0, terminate=lambda: None)
        with tempfile.TemporaryDirectory() as tmp:
            output = pathlib.Path(tmp) / "results.csv"
            (pathlib.Path(tmp) / "config.json").write_text(
                '{"vocab_size":3}\n')
            argv = [
                "eval_glm.py", "--snap", tmp, "--tasks", "smoke",
                "--limit", "1", "--glm", "/fake/glm", "--out",
                str(output),
            ]
            tokenizers = types.SimpleNamespace(Tokenizer=FakeTokenizer)
            stderr_buf = io.StringIO()
            stdout_buf = io.StringIO()
            with mock.patch.object(sys, "argv", argv), \
                    mock.patch.dict(sys.modules, {"tokenizers": tokenizers}), \
                    mock.patch.object(EVAL.subprocess, "Popen",
                                      return_value=process) as popen, \
                    contextlib.redirect_stderr(stderr_buf), \
                    contextlib.redirect_stdout(stdout_buf):
                rc = EVAL.main()
            self.last_popen_kwargs = popen.call_args.kwargs if popen.called else {}
            self.last_stderr = stderr_buf.getvalue()
            self.last_stdout = stdout_buf.getvalue()
            return rc, output.read_text(), popen.call_count

    def test_exact_score_text_survives_csv(self):
        text = "-8.5534581234567888"
        exact, value, contlen, greedy = EVAL.parse_score_result(
            f"{text} 4096 1")
        self.assertEqual(exact, text)
        self.assertEqual(contlen, 4096)
        self.assertEqual(greedy, 1)
        self.assertTrue(value < 0)

        out = io.StringIO()
        meta = ("task", 3, 2, 4096, 17, 2)
        EVAL.write_result_row(out, 9, meta, exact, greedy)
        self.assertEqual(
            out.getvalue(),
            "9,task,3,2,4096,17,2,-8.5534581234567888,1\n")

    def test_shipped_dot6f_form_is_still_accepted(self):
        # The tool must accept BOTH the shipped %.6f form the engine
        # actually prints today AND the %.17g evidence form.
        exact, value, contlen, greedy = EVAL.parse_score_result(
            "-8.553458 4096 1")
        self.assertEqual(exact, "-8.553458")
        self.assertEqual(value, -8.553458)
        self.assertEqual((contlen, greedy), (4096, 1))

    def test_direct_call_rejects_trailing_newline_record(self):
        # See parse_score_result's own docstring: _SCORE_RE's "$" anchor
        # matches just before a trailing "\n" as well as at true
        # end-of-string, so fullmatch -> match/search would accept a
        # newline-terminated record fullmatch correctly rejects. The
        # classifier's own callers strip the newline first and are
        # provably out of reach of this (see the docstring), but this
        # function is also called directly -- pin the direct-call
        # contract so a future weakening doesn't slip in unnoticed there.
        with self.assertRaises(EVAL.EvidenceError):
            EVAL.parse_score_result("-8.553458 4096 1\n")

    def test_parse_c17g_rejects_trailing_garbage(self):
        # parse_c17g is only ever reached, elsewhere in this module,
        # through _SCORE_RE's own "^...$"-anchored capture group, which
        # already excludes trailing garbage before parse_c17g ever sees
        # the text -- so nothing else in this module's test suite drives
        # parse_c17g directly. Direct coverage: verified (RAN, against a
        # scratch mutant copy, not committed) that weakening the
        # fullmatch to match here is actually an EQUIVALENT mutant for
        # every input tried -- the round-trip format comparison a few
        # lines below the match call independently rejects any text with
        # a non-canonical tail, since format(value, ...) can never
        # reproduce trailing garbage. This test pins parse_c17g's own
        # contract directly rather than only through callers; it does
        # not by itself prove the fullmatch call is load-bearing.
        with self.assertRaises(EVAL.EvidenceError):
            EVAL.parse_c17g("1.5garbage")
        with self.assertRaises(EVAL.EvidenceError):
            EVAL.parse_c17g("-8.553458 ")  # trailing space after a valid token
        self.assertEqual(EVAL.parse_c17g("1.5"), 1.5)

    def test_parse_c17g_rejects_bare_trailing_decimal_point(self):
        # "1." is valid Python float() syntax (float("1.") == 1.0), so
        # only the round-trip format comparison -- not the decimal
        # group's "one-or-more" digit requirement -- is what actually
        # rejects it: format(1.0, ".17g") == "1", never "1.". Verified
        # (RAN, scratch mutant, not committed) that widening that group
        # to "zero-or-more" does not change parse_c17g's accept/reject
        # outcome for any of these inputs; kept as direct-coverage
        # regression pins, not as proof the digit requirement is
        # independently load-bearing.
        for bad in ("1.", "-0.", "12."):
            with self.subTest(bad=bad):
                with self.assertRaises(EVAL.EvidenceError):
                    EVAL.parse_c17g(bad)

    def test_parse_c17g_rejects_single_digit_exponent(self):
        # C's %e/%g exponent is zero-padded to at least two digits
        # ("e-05", never "e-5"). Verified (RAN, scratch mutant, not
        # committed) that widening the exponent's trailing-digit count
        # from {1,2} to {0,2} does NOT change parse_c17g's outcome here
        # either, for the same reason as the two tests above: no
        # canonical %.6f/%.17g spelling ever produces a single-digit
        # exponent, so the round-trip check rejects it independently of
        # this group's width. Use a magnitude small enough that %.17g
        # actually chooses exponential notation (a mid-size value like
        # 1e5 round-trips through plain "100000" instead and would not
        # even reach the exponent group).
        text = format(-1.0000000000000001e-05, ".17g")
        self.assertEqual(text, "-1.0000000000000001e-05")
        self.assertEqual(EVAL.parse_c17g(text), -1.0000000000000001e-05)
        with self.assertRaises(EVAL.EvidenceError):
            EVAL.parse_c17g(text.replace("e-05", "e-5"))

    def test_parse_engine_loaded_rejects_overflowing_metrics(self):
        # load_s/resident_mb have no magnitude cap in _FIXED2_TEXT (only
        # a fixed 2-decimal-digit suffix), so a long enough digit run
        # converts via float() to +inf; unlike parse_c17g there is no
        # round-trip format comparison to catch that independently here,
        # so the isfinite check is this function's only defense against
        # it -- a removed/weakened isfinite check would silently accept
        # an infinite load time or resident size.
        huge = "1" + "0" * 350 + ".00"
        for field in ("load_s", "resident_mb"):
            with self.subTest(field=field):
                kwargs = {"load": huge} if field == "load_s" else {"resident": huge}
                line = self.loaded(**kwargs)
                with self.assertRaisesRegex(
                        EVAL.PreambleError, "must be finite"):
                    EVAL.parse_engine_loaded(line)

    def test_score_request_digest_binds_strict_ascii_bytes_including_lf(self):
        requests = ("1 1 1 2", "2 1 0 1 2")
        lines, payload, digests = EVAL.score_request_wire(requests, 3)
        self.assertEqual(lines, (
            b"1 1 1 2\n", b"2 1 0 1 2\n"))
        self.assertEqual(payload, b"".join(lines))
        self.assertEqual(
            digests,
            tuple(hashlib.sha256(line).hexdigest() for line in lines))
        self.assertNotEqual(
            digests[1],
            hashlib.sha256(b"2 1 0 1 2").hexdigest())
        bad_requests = (
            "", "two\nlines", "cr\rline", "1 1", "1 1 0",
            "1 1 0 1 2", "1 1 0 1 junk", "1 1 0 3",
            "1 1 -1 1", "1 1 00 1", "2147483648 1 0 1",
            "1 2147483647 0 1", "evidence-μ",
        )
        for bad in bad_requests:
            with self.subTest(bad=bad):
                with self.assertRaises(EVAL.EvidenceError):
                    EVAL.score_request_wire((bad,), 3)
        with self.assertRaises(EVAL.EvidenceError):
            EVAL.score_request_wire((), 3)

    def test_nonfinite_and_malformed_scores_refuse(self):
        bad = [
            "nan 1 1", "inf 1 1", "-inf 1 1", "-1 1", "-1 1 1 extra",
            "not-a-number 1 1", "-1 -1 1", "-1 0 1", "-1 1 2", "1 1 1",
            "-1  1 1", "-1\t1 1", "-1 01 1", "-1_0 1 1",
            "-1 2147483648 1", "+0 1 0", "-01 1 0",
            "-0_125 1 0", "-١ 1 0", "-1e-9999 1 0",
            "-1.00000000000000000 1 0", "-1e-9 1 0",
            "-1e--09 1 0", "-1e+009 1 0", "-1.25 1 0 junk",
        ]
        for line in bad:
            with self.subTest(line=line):
                with self.assertRaises(EVAL.EvidenceError):
                    EVAL.parse_score_result(line)

    def test_stdout_grammar_refuses_unknown_records(self):
        self.assertIsNone(EVAL.classify_score_stdout(self.BANNER + "\n"))
        for state, draft in (("ACTIVE", 0), ("ACTIVE", 1),
                             ("absent", 0), ("absent", 2),
                             ("DISABLED (multiplexed serve)", 0)):
            with self.subTest(state=state, draft=draft):
                self.assertIsNone(EVAL.classify_score_stdout(
                    self.loaded(state, draft) + "\n"))
        exact, _, _, _ = EVAL.classify_score_stdout("-1.25 1 0\n")
        self.assertEqual(exact, "-1.25")
        bad = (
            "\n", "unexpected banner\n", "nan 1 0\n", "inf 1 0\n",
            " -1.25 1 0\n", "-1.25 1 0 \n", "-1.25 1 0\t\n",
            "-1.25 1 0", "-1.25 1 0\r\n",
            "PROF 0.001 1 1 0.000 0.000 0.000 0.000 0.000 1\n",
            "DONE 7 STAT 1 1.00 0.0 1.00 1 0\n",
            "== GLM C engine fabricated ==\n",
            "== GLM C engine (glm_moe_dsa), cache=0 experts/layer | "
            "compute experts@4-bit dense@8-bit | idot: neon ==\n",
            "== GLM C engine (glm_moe_dsa), cache=064 experts/layer | "
            "compute experts@4-bit dense@8-bit | idot: neon ==\n",
            "loaded in 1.0s | resident dense: 1.00 MB | layers=78 experts=256 | "
            "MTP ACTIVE (draft=1)\n",
            "loaded in 1.00s | resident dense: 1.00 MB | layers=78 experts=256 | "
            "MTP DISABLED (multiplexed serve) (draft=1)\n",
            "loaded in 1.00s | resident dense: 1.00 MB | layers=78 experts=256 | "
            "MTP unknown (draft=0)\n",
            "loaded in 1.00s | resident dense: 1.00 MB | layers=78 experts=256 | "
            "MTP ACTIVE (draft=64)\n",
            "loaded in 1.00s | resident dense: 1.00 MB | layers=0 experts=256 | "
            "MTP ACTIVE (draft=1)\n",
            "loaded in 1.00s | resident dense: 1.00 MB | layers=129 experts=256 | "
            "MTP ACTIVE (draft=1)\n",
            "loaded in 1.00s | resident dense: 1.00 MB | layers=78 experts=0 | "
            "MTP ACTIVE (draft=1)\n",
            "loaded in 1.00s | resident dense: 1.00 MB | layers=78 experts=4097 | "
            "MTP ACTIVE (draft=1)\n",
            "loaded in -0.01s | resident dense: 1.00 MB | layers=78 experts=256 | "
            "MTP ACTIVE (draft=1)\n",
            "loaded in nan s | resident dense: 1.00 MB | layers=78 experts=256 | "
            "MTP ACTIVE (draft=1)\n",
            "loaded in 1.0s | resident dense: 1.00 MB | layers=78 experts=256 | "
            "MTP ACTIVE (draft=1)\n",
            "loaded in 1.00s | resident dense: 1.00 MB | layers=2147483648 experts=256 | "
            "MTP ACTIVE (draft=1)\n",
            " == GLM C engine (glm_moe_dsa), cache=64 experts/layer | "
            "compute experts@4-bit dense@8-bit | idot: neon ==\n",
        )
        for line in bad:
            with self.subTest(line=line):
                with self.assertRaises(EVAL.EvidenceError):
                    EVAL.classify_score_stdout(line)

    def test_loaded_prefix_collision_is_refused_not_passed_through(self):
        # engine_evidence.parse_engine_preamble dispatches by literal prefix
        # (line.startswith("loaded in")), not by a semantic match against
        # the load record's shape. "loaded index ..." shares that 9-char
        # prefix purely because "index" itself starts with "in", so it is
        # routed into parse_engine_loaded and refused there, rather than
        # returned as an ordinary unowned log line. Confirmed at primary
        # source (c/colibri.c on origin/dev): no engine anywhere in this
        # tree ever prints a line starting with "loaded index" -- the only
        # stdout "loaded" record any engine emits is the exact "loaded in
        # ...s | resident dense: ..." line this module already pins above.
        # This is deliberately fail-loud dispatch behavior, not a bug --
        # see engine_evidence.parse_engine_preamble's own docstring
        # (fixed in this cycle to document the prefix mechanism and this
        # exact collision explicitly, since no engine anywhere in this
        # tree emits "loaded index" today). This test pins eval_glm's
        # own consumer-visible behavior for the same collision.
        with self.assertRaises(EVAL.EvidenceError):
            EVAL.classify_score_stdout("loaded index 5 whatever\n")

    def test_oversized_numeric_field_refuses_as_evidence_error(self):
        # engine_evidence's own int()/float() calls now catch this at
        # the root (a numeric preamble field beyond the interpreter's
        # 4300-digit int-string conversion limit raises engine_evidence's
        # own PreambleError, not a bare ValueError) -- see
        # test_engine_evidence.py's identical-purpose tests on
        # parse_engine_banner/parse_engine_loaded directly. This
        # module's three call sites still catch ValueError alongside
        # PreambleError too, as defense in depth for any other
        # ValueError the shared module might someday raise; this test
        # pins the end-to-end result through eval_glm's own consumers.
        big = "1" + "0" * 4300
        banner = (
            f"== GLM C engine (glm_moe_dsa), cache={big} experts/layer | "
            "compute experts@4-bit dense@8-bit | idot: avx2 ==\n")
        with self.assertRaises(EVAL.EvidenceError):
            EVAL.classify_score_stdout(banner)
        parser = EVAL.ScoreStdoutClassifier()
        with self.assertRaises(EVAL.EvidenceError):
            parser.classify(banner)
        # is_score_preamble is a query function (bool return); it must
        # swallow the malformed-field case as "not a preamble" rather
        # than raise at all.
        self.assertFalse(EVAL.is_score_preamble(banner.rstrip("\n")))

    def test_oversized_vocab_json_refuses_as_evidence_error(self):
        # Same error-contract class as F3, in a different function:
        # json.loads() raises a bare ValueError (not its
        # json.JSONDecodeError subclass) for an integer literal beyond
        # Python's 4300-digit int-string conversion limit, which
        # score_snapshot_vocab's except clause did not catch -- the
        # bare ValueError escaped main()'s only try/except around this
        # call (which catches EvidenceError to write the INCOMPLETE
        # marker and return before ever launching the engine), so
        # main() crashed outright with NO INCOMPLETE marker written at
        # all, worse than the ordinary prelaunch-refusal contract every
        # other pre-engine-launch failure gets.
        huge_vocab = "1" + "0" * 4300
        with tempfile.TemporaryDirectory() as tmp:
            (pathlib.Path(tmp) / "config.json").write_text(
                '{"vocab_size": ' + huge_vocab + '}\n')
            with self.assertRaises(EVAL.EvidenceError):
                EVAL.score_snapshot_vocab(tmp)

    def test_oversized_vocab_json_marks_incomplete_end_to_end(self):
        # The end-to-end contract test_oversized_vocab_json_refuses_
        # as_evidence_error's unit-level pin implies: main() must reach
        # prelaunch_incomplete() (INCOMPLETE marker written, engine
        # never launched, exit 1) rather than crash uncaught.
        class Encoded:
            ids = [1, 2]

        class FakeTokenizer:
            @staticmethod
            def from_file(path):
                return FakeTokenizer()

            @staticmethod
            def encode(text):
                return Encoded()

        huge_vocab = "1" + "0" * 4300
        with tempfile.TemporaryDirectory() as tmp:
            output = pathlib.Path(tmp) / "results.csv"
            (pathlib.Path(tmp) / "config.json").write_text(
                '{"vocab_size": ' + huge_vocab + '}\n')
            argv = [
                "eval_glm.py", "--snap", tmp, "--tasks", "smoke",
                "--limit", "1", "--glm", "/fake/glm", "--out",
                str(output),
            ]
            tokenizers = types.SimpleNamespace(Tokenizer=FakeTokenizer)
            with mock.patch.object(sys, "argv", argv), \
                    mock.patch.dict(sys.modules, {"tokenizers": tokenizers}), \
                    mock.patch.object(
                        EVAL.subprocess, "Popen",
                        side_effect=AssertionError("engine launched")):
                rc = EVAL.main()
            self.assertEqual(rc, 1)
            self.assertIn("# INCOMPLETE:", output.read_text())

    def test_banner_kernels_and_load_boundaries_are_exact(self):
        self.assertEqual(
            EVAL.parse_engine_banner(self.BANNER)["kernel"], "neon-i8mm")
        for kernel in ("avx512-vnni", "avx-vnni", "avx2", "neon-i8mm",
                       "neon", "vsx", "scalar"):
            line = self.BANNER.replace("neon-i8mm", kernel)
            self.assertEqual(EVAL.parse_engine_banner(line)["kernel"], kernel)
        with self.assertRaises(EVAL.PreambleError):
            EVAL.parse_engine_banner(self.BANNER.replace("neon-i8mm", "fabricated"))

        for layers, experts in ((1, 1), (128, 4096)):
            parsed = EVAL.parse_engine_loaded(self.loaded(
                layers=layers, experts=experts))
            self.assertEqual((parsed["layers"], parsed["experts"]),
                             (layers, experts))

    def test_score_stream_owns_banner_load_lifecycle(self):
        parser = EVAL.ScoreStdoutClassifier()
        self.assertIsNone(parser.classify(self.BANNER + "\n"))
        self.assertIsNone(parser.classify(self.loaded("absent", 2) + "\n"))
        exact, _, _, _ = parser.classify("-1.25 1 0\n")
        self.assertEqual(exact, "-1.25")
        parser.finish()

        cases = (
            [self.loaded() + "\n", self.BANNER + "\n"],
            [self.BANNER + "\n", "-1.25 1 0\n"],
            [self.BANNER + "\n", self.BANNER + "\n"],
            [self.BANNER + "\n", self.loaded() + "\n",
             self.loaded() + "\n"],
            [self.BANNER + "\n"],
        )
        for records in cases:
            with self.subTest(records=records):
                parser = EVAL.ScoreStdoutClassifier()
                with self.assertRaises(EVAL.EvidenceError):
                    for record in records:
                        parser.classify(record)
                    parser.finish()

    def test_eval_main_uses_stateful_score_stream(self):
        rc, output, launches = self.run_eval_main((
            self.BANNER + "\n",
            self.loaded("absent", 2) + "\n",
            "-1 2 1\n", "-2 2 0\n", "-3 2 0\n",
        ))
        self.assertEqual(rc, 0)
        self.assertEqual(launches, 1)
        self.assertIn("# finished: 3/3", output)

    def test_result_rows_are_written_and_flushed_incrementally(self):
        # A run interrupted mid-task must leave a valid partial file --
        # rows are written and flushed per-request, never buffered until
        # the run completes. Simulate a mid-run crash by having the fake
        # engine's stdout iterator raise after the first scored record;
        # the CSV must already contain that row.

        class Encoded:
            ids = [1, 2]

        class FakeTokenizer:
            @staticmethod
            def from_file(path):
                return FakeTokenizer()

            @staticmethod
            def encode(text):
                return Encoded()

        class CrashingStdout:
            def __init__(self, lines):
                self._lines = list(lines)

            def __iter__(self):
                for index, line in enumerate(self._lines):
                    yield line
                    if index == 2:  # after banner+load+one SCORE row
                        raise OSError("engine died mid-run")

        stdout_lines = [
            self.BANNER + "\n", self.loaded("absent", 2) + "\n",
            "-1 2 1\n",
            "-2 2 0\n",
        ]
        process = types.SimpleNamespace(
            returncode=1, stderr=(), stdout=CrashingStdout(stdout_lines),
            wait=lambda: 1, poll=lambda: 1, terminate=lambda: None)
        with tempfile.TemporaryDirectory() as tmp:
            output = pathlib.Path(tmp) / "results.csv"
            (pathlib.Path(tmp) / "config.json").write_text(
                '{"vocab_size":3}\n')
            argv = [
                "eval_glm.py", "--snap", tmp, "--tasks", "smoke",
                "--limit", "1", "--glm", "/fake/glm", "--out",
                str(output),
            ]
            tokenizers = types.SimpleNamespace(Tokenizer=FakeTokenizer)
            with mock.patch.object(sys, "argv", argv), \
                    mock.patch.dict(sys.modules, {"tokenizers": tokenizers}), \
                    mock.patch.object(EVAL.subprocess, "Popen",
                                      return_value=process), \
                    self.assertRaises(OSError):
                EVAL.main()
            text = output.read_text()
            self.assertIn(",-1,1\n", text.replace(".000000", ""))
            self.assertNotIn("# finished:", text)
            # The crash-path marker itself must be present, not
            # just the absence of "# finished:" -- a downstream consumer
            # scans for this exact line to know the run never reached a
            # complete denominator.
            self.assertIn(
                "# INCOMPLETE: evaluator terminated before a complete "
                "denominator", text)

    def test_row_is_durable_on_disk_before_the_run_finishes(self):
        # test_result_rows_are_written_and_flushed_incrementally (above)
        # is named for the per-row flush but cannot actually detect
        # losing it: that test reads the output file only AFTER
        # EVAL.main() has unwound through its `finally` block, which
        # closes out_f (and Python flushes a file on close) regardless
        # of whether any PER-ROW flush ever ran. It defends the ARTIFACT
        # after a clean-ish exit, not the PROPERTY of durability DURING
        # the run.
        #
        # This test instead opens a SECOND, independent file handle on
        # the same path while main() is still running (has not
        # returned) and reads through it -- a second handle can only
        # see bytes actually flushed to the OS by the writer, never
        # bytes still sitting only in the writer's own buffer. The
        # probe runs from inside the fake stdout generator, resumed
        # only once the consumer (main()'s per-line loop body, including
        # write_result_row + out_f.flush() for that line) has fully
        # finished with the PREVIOUS line and is asking for the next one
        # -- so a snapshot taken there reflects exactly what has been
        # made durable so far, mid-run.
        class Encoded:
            ids = [1, 2]

        class FakeTokenizer:
            @staticmethod
            def from_file(path):
                return FakeTokenizer()

            @staticmethod
            def encode(text):
                return Encoded()

        class ProbingStdout:
            def __init__(self, lines, probe_path):
                self._lines = list(lines)
                self._probe_path = probe_path
                self.snapshots = []

            def __iter__(self):
                for line in self._lines:
                    yield line
                    try:
                        with open(self._probe_path) as f:
                            self.snapshots.append(f.read())
                    except FileNotFoundError:
                        self.snapshots.append("")

        with tempfile.TemporaryDirectory() as tmp:
            output = pathlib.Path(tmp) / "results.csv"
            (pathlib.Path(tmp) / "config.json").write_text(
                '{"vocab_size":3}\n')
            stdout = ProbingStdout((
                self.BANNER + "\n",
                self.loaded("absent", 2) + "\n",
                "-1 2 1\n",
                "-2 2 0\n",
            ), str(output))
            process = types.SimpleNamespace(
                returncode=0, stderr=(), stdout=stdout,
                wait=lambda: 0, poll=lambda: 0, terminate=lambda: None)
            argv = [
                "eval_glm.py", "--snap", tmp, "--tasks", "smoke",
                "--limit", "1", "--glm", "/fake/glm", "--out",
                str(output),
            ]
            tokenizers = types.SimpleNamespace(Tokenizer=FakeTokenizer)
            with mock.patch.object(sys, "argv", argv), \
                    mock.patch.dict(sys.modules, {"tokenizers": tokenizers}), \
                    mock.patch.object(EVAL.subprocess, "Popen",
                                      return_value=process):
                rc = EVAL.main()
            self.assertEqual(rc, 0)
        # snapshots[0]/[1]: after the banner/load preamble lines, before
        # any SCORE result -- no row yet.
        self.assertNotIn(",-1,1\n", stdout.snapshots[0].replace(".000000", ""))
        self.assertNotIn(",-1,1\n", stdout.snapshots[1].replace(".000000", ""))
        # snapshots[2]: taken right after the FIRST SCORE line was fully
        # processed but BEFORE the second one is even requested --
        # main() has not returned. The first row must already be
        # readable through the independent handle; the second must not.
        first_row_snapshot = stdout.snapshots[2].replace(".000000", "")
        self.assertIn(",-1,1\n", first_row_snapshot,
                      "first row not durable before the run finished -- "
                      "the per-row flush is not doing its job")
        self.assertNotIn(",-2,0\n", first_row_snapshot)

    def test_child_is_terminated_on_mid_run_interrupt_or_exception(self):
        # SIGINT/SIGTERM/any exception mid-run must not leave the
        # engine child running. The fake engine here never exits on its
        # own -- .poll() keeps returning None (as a real child that
        # ignores its stdin being closed would) until .terminate() is
        # actually called -- so a passing test proves main() called
        # terminate() itself rather than relying on the child to die.
        request_raw = b"2 2 1 2 1 2\n"
        request_digest = hashlib.sha256(request_raw).hexdigest()

        class Encoded:
            ids = [1, 2]

        class FakeTokenizer:
            @staticmethod
            def from_file(path):
                return FakeTokenizer()

            @staticmethod
            def encode(text):
                return Encoded()

        banner = self.BANNER + "\n"
        loaded = self.loaded("absent", 2) + "\n"

        def interrupting_stdout():
            yield banner
            yield loaded
            yield f"SCORE 0 {request_digest} -1 2 1\n"
            raise KeyboardInterrupt("operator pressed Ctrl+C")

        class NeverExitingProcess:
            def __init__(self):
                self.returncode = None
                self.stderr = ()
                self.stdout = interrupting_stdout()
                self.terminated = False
                self.terminate_calls = 0
                self.wait_calls = 0

            def poll(self):
                return 0 if self.terminated else None

            def terminate(self):
                self.terminated = True
                self.terminate_calls += 1

            def wait(self):
                self.wait_calls += 1
                return 0

        process = NeverExitingProcess()
        with tempfile.TemporaryDirectory() as tmp:
            output = pathlib.Path(tmp) / "results.csv"
            (pathlib.Path(tmp) / "config.json").write_text(
                '{"vocab_size":3}\n')
            argv = [
                "eval_glm.py", "--snap", tmp, "--tasks", "smoke",
                "--limit", "1", "--glm", "/fake/glm", "--out",
                str(output),
            ]
            tokenizers = types.SimpleNamespace(Tokenizer=FakeTokenizer)
            with mock.patch.object(sys, "argv", argv), \
                    mock.patch.dict(sys.modules, {"tokenizers": tokenizers}), \
                    mock.patch.object(EVAL.subprocess, "Popen",
                                      return_value=process), \
                    self.assertRaises(KeyboardInterrupt):
                EVAL.main()
        self.assertEqual(process.terminate_calls, 1)
        self.assertGreaterEqual(process.wait_calls, 1)

    @unittest.skipIf(
        sys.platform == "win32",
        "os.kill(pid, SIGTERM) on Windows is TerminateProcess with exit code 15: "
        "no handler runs, so the POSIX mechanism under test does not exist there "
        "(the SIGINT/exception half above covers child cleanup on Windows)")
    def test_sigterm_mid_run_terminates_the_child_and_propagates(self):
        # The SIGTERM half: a real SIGTERM (not just an ordinary
        # Python exception) delivered while the child is running must
        # also be converted into child cleanup, not left to Python's
        # default SIGTERM handling (which does not run this module's
        # `finally` cleanup at all).
        request_raw = b"2 2 1 2 1 2\n"
        request_digest = hashlib.sha256(request_raw).hexdigest()

        class Encoded:
            ids = [1, 2]

        class FakeTokenizer:
            @staticmethod
            def from_file(path):
                return FakeTokenizer()

            @staticmethod
            def encode(text):
                return Encoded()

        banner = self.BANNER + "\n"
        loaded = self.loaded("absent", 2) + "\n"

        def stdout_then_sigterm():
            yield banner
            yield loaded
            yield f"SCORE 0 {request_digest} -1 2 1\n"
            os.kill(os.getpid(), signal.SIGTERM)
            # Not reached if the handler fires promptly, as it must.
            yield f"SCORE 1 {request_digest} -2 2 0\n"

        class NeverExitingProcess:
            def __init__(self):
                self.returncode = None
                self.stderr = ()
                self.stdout = stdout_then_sigterm()
                self.terminated = False
                self.terminate_calls = 0
                self.wait_calls = 0

            def poll(self):
                return 0 if self.terminated else None

            def terminate(self):
                self.terminated = True
                self.terminate_calls += 1

            def wait(self):
                self.wait_calls += 1
                return 0

        process = NeverExitingProcess()
        previous_handler = signal.getsignal(signal.SIGTERM)
        try:
            with tempfile.TemporaryDirectory() as tmp:
                output = pathlib.Path(tmp) / "results.csv"
                (pathlib.Path(tmp) / "config.json").write_text(
                    '{"vocab_size":3}\n')
                argv = [
                    "eval_glm.py", "--snap", tmp, "--tasks", "smoke",
                    "--limit", "1", "--glm", "/fake/glm", "--out",
                    str(output),
                ]
                tokenizers = types.SimpleNamespace(Tokenizer=FakeTokenizer)
                with mock.patch.object(sys, "argv", argv), \
                        mock.patch.dict(sys.modules, {"tokenizers": tokenizers}), \
                        mock.patch.object(EVAL.subprocess, "Popen",
                                          return_value=process), \
                        self.assertRaises(EVAL.ChildTerminateRequested):
                    EVAL.main()
        finally:
            # Defensive: main()'s own finally already restores the prior
            # handler, but never trust a test to leave process-global
            # signal state behind if the assertion above ever fails.
            signal.signal(signal.SIGTERM, previous_handler)
        self.assertEqual(process.terminate_calls, 1)
        self.assertGreaterEqual(process.wait_calls, 1)
        self.assertEqual(signal.getsignal(signal.SIGTERM), previous_handler)

    def test_engine_run_completes_and_reports_finished(self):
        # The engine only ever emits the byte-compatible legacy
        # three-field form ("<exact> <contlen> <greedy>"); a clean run
        # over that form completes and writes every row.
        rc, output, launches = self.run_eval_main((
            self.BANNER + "\n",
            self.loaded("absent", 2) + "\n",
            "-1 2 1\n", "-2 2 0\n", "-3 2 0\n",
        ))
        self.assertEqual(rc, 0)
        self.assertEqual(launches, 1)
        self.assertIn("# finished: 3/3", output)

    def test_classifier_refuses_unknown_lines(self):
        # ScoreStdoutClassifier is the class main() actually constructs
        # -- classify_score_stdout (the standalone function, covered by
        # test_stdout_grammar_refuses_unknown_records) is a separate
        # code path main() never calls. A foreign line must be refused
        # by the classifier itself, not merely by the standalone
        # function.
        foreign_lines = (
            "PROF 0.001 1 1 0.000 0.000 0.000 0.000 0.000 1\n",
            "not a score line at all\n",
            "nan 1 0\n",
            "SCORE 0 " + "a" * 64 + " -1 1 1\n",  # no reader emits this shape
        )
        for line in foreign_lines:
            with self.subTest(line=line):
                parser = EVAL.ScoreStdoutClassifier()
                parser.classify(self.BANNER + "\n")
                parser.classify(self.loaded("absent", 2) + "\n")
                with self.assertRaises(EVAL.EvidenceError):
                    parser.classify(line)

    def test_eval_main_rejects_lifecycle_and_blank_records(self):
        banner = self.BANNER + "\n"
        loaded = self.loaded("absent", 2) + "\n"
        scores = ("-1 2 1\n", "-2 2 0\n", "-3 2 0\n")
        cases = {
            "load_before_banner": (loaded, banner) + scores,
            "score_before_load": (banner, scores[0], loaded) + scores[1:],
            "duplicate_load": (banner, loaded, loaded) + scores,
            "missing_load_at_eof": (banner,),
            "blank_before_banner": ("\n", banner, loaded) + scores,
            "blank_between_preambles": (banner, "\n", loaded) + scores,
            "blank_after_scores": (banner, loaded) + scores + ("\n",),
        }
        for name, records in cases.items():
            with self.subTest(name=name):
                rc, output, launches = self.run_eval_main(records)
                self.assertEqual(rc, 1)
                self.assertEqual(launches, 1)
                self.assertIn("# INCOMPLETE:", output)
                self.assertNotIn("# finished:", output)

    def test_only_complete_zero_exit_denominator_passes(self):
        # NOTE: completion_error's contract changed from the original
        # ported oracle -- it now matches dev's own exit-code contract
        # exactly (a partial or nonzero-exit-but-nonempty run is no
        # longer fatal here; see its docstring), so several of the
        # original oracle's assertions below are inverted rather than
        # reused verbatim.
        self.assertIsNone(EVAL.completion_error(0, 3, 3, 9))
        # A clean exit with zero requests scored is no longer flagged by
        # completion_error itself (dev's own contract only fires on a
        # NONZERO exit with zero scored; `expected`/`continuation_tokens`
        # are not otherwise consulted).
        self.assertIsNone(EVAL.completion_error(0, 0, 0, 0))
        self.assertIsNone(EVAL.completion_error(0, 3, 3, 0))
        # A nonzero exit that still scored at least one request is a
        # partial run, not fatal.
        self.assertIsNone(EVAL.completion_error(2, 7, 7, 7))
        self.assertIsNone(EVAL.completion_error(0, 6, 7, 6))
        # Fatal only when the engine exits nonzero with NOTHING scored...
        self.assertIsNotNone(EVAL.completion_error(2, 0, 7, 0))
        self.assertIn("zero requests scored", EVAL.completion_error(2, 0, 7, 0))
        # ...or a stream_error is present regardless of completion count.
        self.assertIn("broken", EVAL.completion_error(0, 7, 7, 7, "broken"))
        self.assertIn("broken", EVAL.completion_error(0, 0, 7, 0, "broken"))

    def test_empty_selection_refuses_before_engine_launch(self):
        with tempfile.TemporaryDirectory() as tmp:
            out = pathlib.Path(tmp) / "results.csv"
            argv = ["eval_glm.py", "--snap", tmp, "--tasks", "", "--out", str(out)]
            with mock.patch.object(sys, "argv", argv), \
                    mock.patch.object(EVAL.subprocess, "Popen") as popen:
                rc = EVAL.main()
            self.assertEqual(rc, 1)
            popen.assert_not_called()
            text = out.read_text()
            self.assertIn("# INCOMPLETE: 0/0; error=no benchmark tasks selected", text)
            self.assertNotIn("finished: 0/0", text)

    def test_zero_request_task_refuses_before_engine_launch(self):
        class FakeTokenizer:
            @staticmethod
            def from_file(path):
                return object()

        with tempfile.TemporaryDirectory() as tmp:
            root = pathlib.Path(tmp)
            (root / "empty.jsonl").write_text("")
            out = root / "results.csv"
            argv = ["eval_glm.py", "--snap", tmp, "--data", tmp,
                    "--tasks", "empty", "--out", str(out)]
            fake = types.SimpleNamespace(Tokenizer=FakeTokenizer)
            with mock.patch.object(sys, "argv", argv), \
                    mock.patch.dict(sys.modules, {"tokenizers": fake}), \
                    mock.patch.object(EVAL.subprocess, "Popen",
                                      side_effect=AssertionError("engine launched")) as popen:
                rc = EVAL.main()
            self.assertEqual(rc, 1)
            popen.assert_not_called()
            text = out.read_text()
            self.assertIn(
                "# INCOMPLETE: 0/0; error=selected tasks produced zero SCORE requests",
                text)
            self.assertNotIn("finished: 0/0", text)

    def test_zero_continuation_choices_refuse_before_engine_launch(self):
        class Encoded:
            def __init__(self, ids):
                self.ids = ids

        class BoundaryTokenizer:
            @staticmethod
            def from_file(path):
                return BoundaryTokenizer()

            @staticmethod
            def encode(text):
                return Encoded({
                    "ctx": [1], "ctxgood": [1, 2], "good": [2],
                    "ctxvanish": [1], "vanish": [], "": [],
                }.get(text, [1]))

        cases = {
            "one_empty": [{"ctx": "ctx", "choices": [""], "gold": 0}],
            "all_empty": [{"ctx": "ctx", "choices": ["", ""], "gold": 0}],
            "boundary_still_empty": [
                {"ctx": "ctx", "choices": ["vanish"], "gold": 0}],
            "mixed_positive_zero": [
                {"ctx": "ctx", "choices": ["good", ""], "gold": 0}],
        }
        tokenizer = BoundaryTokenizer()
        for name, docs in cases.items():
            with self.subTest(name=name), \
                    self.assertRaisesRegex(EVAL.EvidenceError,
                                           "no positive context/continuation"):
                EVAL.build_requests(tokenizer, {"task": docs})

        tokenizers = types.SimpleNamespace(Tokenizer=BoundaryTokenizer)
        for name, docs in cases.items():
            with self.subTest(prelaunch=name), \
                    tempfile.TemporaryDirectory() as tmp:
                root = pathlib.Path(tmp)
                data = root / f"{name}.jsonl"
                data.write_text(json.dumps(docs[0]) + "\n")
                out = root / "results.csv"
                argv = ["eval_glm.py", "--snap", str(root),
                        "--data", str(root), "--tasks", name,
                        "--out", str(out)]
                with mock.patch.object(sys, "argv", argv), \
                        mock.patch.dict(
                            sys.modules, {"tokenizers": tokenizers}), \
                        mock.patch.object(
                            EVAL.subprocess, "Popen",
                            side_effect=AssertionError(
                                "engine launched")) as popen:
                    rc = EVAL.main()
                self.assertEqual(rc, 1)
                popen.assert_not_called()
                text = out.read_text()
                self.assertIn("# INCOMPLETE: 0/0; error=", text)
                self.assertNotIn("# finished:", text)

    def test_dry_run_does_not_require_a_vocabulary(self):
        # dev's own --dry never looked up config.json's
        # vocab_size at all -- it only builds and tokenizes requests,
        # then stops. This module's vocabulary/digest binding is a
        # per-request-wire step for the real engine launch, not a
        # plumbing check, so --dry must not depend on it.
        class Encoded:
            ids = [1, 2]

        class FakeTokenizer:
            @staticmethod
            def from_file(path):
                return FakeTokenizer()

            @staticmethod
            def encode(text):
                return Encoded()

        with tempfile.TemporaryDirectory() as tmp:
            # No config.json at all in the snapshot directory -- a real
            # vocabulary lookup would raise EvidenceError immediately.
            argv = ["eval_glm.py", "--snap", tmp, "--tasks", "smoke",
                    "--limit", "1", "--dry"]
            tokenizers = types.SimpleNamespace(Tokenizer=FakeTokenizer)
            with mock.patch.object(sys, "argv", argv), \
                    mock.patch.dict(sys.modules, {"tokenizers": tokenizers}), \
                    mock.patch.object(
                        EVAL, "score_snapshot_vocab",
                        side_effect=AssertionError(
                            "vocabulary looked up during --dry")) as vocab, \
                    mock.patch.object(
                        EVAL.subprocess, "Popen",
                        side_effect=AssertionError("engine launched")) as popen:
                rc = EVAL.main()
            self.assertIsNone(rc)
            vocab.assert_not_called()
            popen.assert_not_called()

    def test_partial_run_still_reports_the_table_and_exits_zero(self):
        # dev's own exit-code contract (coli bench does
        # `sys.exit(subprocess.call(cmd, ...))`; diag_harness.py parses
        # this tool's stdout table from a subprocess call) exits nonzero
        # ONLY when the engine produced nothing at all. A partial run --
        # some but not all requests scored, clean stream -- must still
        # print the accuracy table and exit 0; the INCOMPLETE marker is
        # additive (alongside "# finished", not instead of it).
        rc, output, launches = self.run_eval_main((
            self.BANNER + "\n",
            self.loaded("absent", 2) + "\n",
            "-1 2 1\n",  # only 1 of 3 requests scored
        ))
        self.assertEqual(rc, 0)
        self.assertEqual(launches, 1)
        self.assertIn("# finished: 1/3", output)
        self.assertIn("# INCOMPLETE: 1/3 requests scored", output)
        self.assertIn(
            "WARNING: only 1/3 requests scored", self.last_stderr)
        self.assertIn("MEAN acc_norm", self.last_stdout)

    def test_python_engine_byte_limits_are_inclusive_and_preallocation(self):
        engine_limit = 256 << 20
        self.assertEqual(EVAL._ENGINE_TEXT_MAX_BYTES, engine_limit)
        self.assertEqual(
            EVAL._checked_engine_text_size(engine_limit, "SCORE"),
            engine_limit)
        with self.assertRaises(EVAL.EvidenceError):
            EVAL._checked_engine_text_size(engine_limit + 1, "SCORE")

        with tempfile.TemporaryDirectory() as tmp:
            root = pathlib.Path(tmp)
            score_config = root / "config.json"
            score_raw = b'{"vocab_size":4}\n'
            score_config.write_bytes(score_raw)
            with mock.patch.object(
                    EVAL, "_ENGINE_TEXT_MAX_BYTES", len(score_raw)):
                self.assertEqual(EVAL.score_snapshot_vocab(root), 4)
                score_config.write_bytes(score_raw + b" ")
                with self.assertRaisesRegex(EVAL.EvidenceError, "256 MiB"):
                    EVAL.score_snapshot_vocab(root)

        request = "1 1 0 1"
        request_bytes = len((request + "\n").encode("ascii"))
        with mock.patch.object(
                EVAL, "_ENGINE_TEXT_MAX_BYTES", request_bytes):
            _, payload, _ = EVAL.score_request_wire((request,), 4)
            self.assertEqual(len(payload), request_bytes)
        with mock.patch.object(
                EVAL, "_ENGINE_TEXT_MAX_BYTES", request_bytes - 1):
            with self.assertRaisesRegex(EVAL.EvidenceError, "256 MiB"):
                EVAL.score_request_wire((request,), 4)


class DifferentialBiteTests(unittest.TestCase):
    """dev's classifier silently accepts a foreign stdout line the new
    copy refuses. dev's side is not itself invoked here (no ported dev
    module exists in this tree); it is asserted against the same fixture
    line via dev's documented filter logic, ported verbatim inline. Only
    the new copy's refusal is exercised by calling real code."""

    FOREIGN_LINE = "1 1 1\n"  # shaped like dev's own accepted grammar

    def test_dev_copy_silently_accepts_a_foreign_numeric_line(self):
        # dev's inline stdout filter (ported verbatim as the oracle): any
        # line starting with a digit or '-' is treated as a SCORE record,
        # with no further validation at all.
        line = self.FOREIGN_LINE.strip()
        self.assertTrue(line and line[0] in "-0123456789")
        parts = line.split()
        logprob = float(parts[0])  # dev: "try: logprob = float(parts[0])"
        # dev accepts this as a real SCORE result -- a false positive: a
        # log-likelihood can never be positive, but dev's filter never
        # checks the sign (or finiteness, or field count) at all.
        self.assertEqual(logprob, 1.0)

    def test_new_copy_refuses_the_same_foreign_line_by_name(self):
        with self.assertRaisesRegex(
                EVAL.EvidenceError, "SCORE logprob is not finite/non-positive"):
            EVAL.classify_score_stdout(self.FOREIGN_LINE)


if __name__ == "__main__":
    unittest.main()