Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
32.4 KB
Raw Blame
"""
Harness di validazione qualita' per il motore C GLM-5.2 (int4 streaming).
Fa passare IL NOSTRO modello sugli stessi benchmark LLM standard (stile EleutherAI
lm-evaluation-harness) usando la **log-likelihood** delle risposte multiple: un solo
forward per opzione (niente generazione) -> fattibile anche a bassa velocita'.
Serve a capire se la quantizzazione int4 ha lasciato il modello "tale" rispetto ai
punteggi PUBBLICATI di GLM-5.2 (e, per contesto, Claude/GPT).

Dipendenze: solo `tokenizers` + il binario ./glm. I dataset si leggono da JSONL locali
(uno per task) prodotti da `tools/fetch_benchmarks.py`. Formato di ogni riga JSONL:
    {"ctx": "...", "choices": ["...","..."], "gold": 0}
Cosi' la harness e' offline e deterministica.

USO:
  # 1) (una volta, quando hai rete) scarica i benchmark in ./bench/*.jsonl
  python3 tools/fetch_benchmarks.py --out ./bench --tasks hellaswag,arc_challenge,mmlu --limit 200
  # 2) plumbing test della meccanica (senza motore):
  python3 tools/eval_glm.py --snap /path/to/glm52_i4 --data ./bench --tasks smoke --dry
  # 3) validazione vera quando il modello e' pronto:
  python3 tools/eval_glm.py --snap /path/to/glm52_i4 --data ./bench \
                      --tasks hellaswag,arc_challenge,mmlu --limit 40 --ram 15
  # leve di ricerca: passate al motore via env
  TOPP=0.9 python3 tools/eval_glm.py --snap /path/to/glm52_i4 --data ./bench --tasks mmlu --ram 15
"""
import argparse
import hashlib
import json
import math
import os
import random
import re
import signal
import subprocess
import sys
import tempfile
import threading
import time

_TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
if _TOOLS_DIR not in sys.path:
    sys.path.insert(0, _TOOLS_DIR)
from engine_evidence import (PreambleError, parse_engine_banner,
                             parse_engine_loaded, parse_engine_preamble)

# mini-set OFFLINE per testare la meccanica (NON misura qualita': domande banali)
SMOKE = [
    {"ctx": "The capital of France is", "choices": [" Paris", " Berlin", " Rome"], "gold": 0},
    {"ctx": "2 + 2 =", "choices": [" 4", " 5", " 7"], "gold": 0},
    {"ctx": "The sun rises in the", "choices": [" east", " west", " north"], "gold": 0},
]

# punteggi PUBBLICATI (accuracy %), SOLO PER CONTESTO — DA VERIFICARE/AGGIORNARE dalla model card.
REFERENCE = {
    "mmlu":          {"GLM-5.2 (pubbl.)": None, "Claude (rif.)": None, "GPT (rif.)": None},
    "hellaswag":     {"GLM-5.2 (pubbl.)": None},
    "arc_challenge": {"GLM-5.2 (pubbl.)": None},
}


class EvidenceError(ValueError):
    """A SCORE stream cannot support a complete, finite evidence result."""


class ChildTerminateRequested(BaseException):
    """A termination signal (SIGTERM) arrived while the engine child was
    running. Raised from a signal handler installed only for the
    duration of that child's run, so it unwinds through the same
    ``finally`` cleanup as any other mid-run exception (including
    Python's own SIGINT-to-KeyboardInterrupt) and the child is never
    left running as a zombie/orphan.
    """


_INT32_MAX = 2**31 - 1
_ENGINE_TEXT_MAX_BYTES = 256 << 20
_UINT_TEXT = r"(?:0|[1-9][0-9]*)"
_C17G_TEXT = (r"-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?"
              r"(?:e[+-](?:0[0-9]|[1-9][0-9]{1,2}))?")
_SCORE_RE = re.compile(
    rf"^({_C17G_TEXT}) ({_UINT_TEXT}) ([01])$")


def _checked_engine_text_size(length, label):
    if type(length) is not int or not 0 <= length <= _ENGINE_TEXT_MAX_BYTES:
        raise EvidenceError(
            f"{label} exceeds the inclusive 256 MiB engine limit")
    return length


def parse_c17g(text):
    """Parse the canonical C-locale numeric token the engine actually emits.

    The engine has shipped two mutually exclusive SCORE spellings across
    its history -- the byte-compatible ``printf("%.6f")`` form dev still
    emits today, and the newer opt-in ``printf("%.17g")`` evidence form.
    Both are exact, finite, round-trippable spellings of the same
    C-locale numeric domain, so both are accepted here (the function name
    is kept for the newer form this module's identity checks depend on);
    a text that is neither exact spelling -- including any non-canonical
    variant such as a non-canonical ``%.17g`` corpus (nan/inf/-inf never
    survive: neither spelling is finite-preserving for them) -- is
    refused.
    """
    # fullmatch (not match/search) here is UNREACHABLE via the real
    # production path: this function's only caller, parse_score_result,
    # only ever passes the substring _SCORE_RE captured for this exact
    # sub-pattern (_SCORE_RE embeds _C17G_TEXT verbatim as its first
    # group), and a regex capture group's matched text always already
    # satisfies the sub-pattern it was captured with -- fullmatch,
    # match and search are identical for that input by construction,
    # not merely by absence of a counterexample. For a DIRECT caller
    # (this function is called directly elsewhere in this module's own
    # tests) fullmatch vs. match/search is still equivalent in
    # practice: RAN mutation testing (both variants, full suite) found
    # every constructible counterexample (trailing garbage, a bare
    # trailing ".", a single-digit exponent) independently rejected by
    # the round-trip format comparison a few lines below regardless,
    # since format(value, ...) can never reproduce a non-canonical
    # tail. See test_parse_c17g_rejects_* for the direct-caller pins.
    if not isinstance(text, str) or not re.fullmatch(_C17G_TEXT, text):
        raise EvidenceError(f"not a canonical %.6f/%.17g token: {text!r}")
    try:
        value = float(text)
    except ValueError as exc:
        raise EvidenceError(f"malformed %.6f/%.17g token: {text!r}") from exc
    if (not math.isfinite(value) or
            (format(value, ".17g") != text and format(value, ".6f") != text)):
        raise EvidenceError(
            f"not an exact finite %.6f/%.17g spelling: {text!r}")
    return value


def is_score_preamble(line):
    """Validate one of the two exact stdout records emitted before SCORE."""
    try:
        return parse_engine_preamble(line) is not None
    except (PreambleError, ValueError):
        # engine_evidence's own int() conversions can raise a bare
        # ValueError (not its PreambleError) for a numeric field beyond
        # Python's 4300-digit int-string conversion limit; a query
        # function is not the place to let that escape uncaught.
        return False


def parse_score_result(line):
    """Return (exact_text, value, contlen, greedy) for one complete SCORE line.

    fullmatch (not match/search) is UNREACHABLE-via-the-classifier for
    the one input shape that could distinguish them: a `$` anchor (built
    into _SCORE_RE's own pattern text) is zero-width and matches just
    before a trailing "\\n" as well as at true end-of-string, so
    match()/search() would accept a newline-terminated record that
    fullmatch correctly rejects (this is exactly the M11-class gap
    engine_evidence.py's own fullmatch calls had). classify_score_stdout
    and ScoreStdoutClassifier.classify -- the only production callers --
    both strip exactly one trailing newline before calling this function
    and both refuse any OTHER embedded "\\n" first (`raw_line.count("\\n")
    != 1`), so `line` here is guaranteed newline-free through that path:
    the exploit cannot be entered. For a caller reaching this function
    directly (as this module's own tests do), the newline-terminated
    input is still refused today -- test_direct_call_rejects_trailing_
    newline_record pins it so a future match()/search() weakening does
    not go unnoticed on the direct-call path either.
    """
    match = _SCORE_RE.fullmatch(line)
    if not match:
        raise EvidenceError(f"not an exact SCORE record: {line!r}")
    exact, contlen_text, greedy_text = match.groups()
    try:
        value = parse_c17g(exact)
        contlen = int(contlen_text)
        greedy = int(greedy_text)
    except ValueError as exc:
        raise EvidenceError(f"malformed SCORE fields: {line!r}") from exc
    if not math.isfinite(value) or value > 0.0:
        raise EvidenceError(f"SCORE logprob is not finite/non-positive: {exact}")
    if not 1 <= contlen <= _INT32_MAX or greedy not in (0, 1):
        raise EvidenceError(f"invalid SCORE metadata: {line!r}")
    return exact, value, contlen, greedy


def classify_score_stdout(raw_line):
    """Accept one exact newline-terminated production stdout record."""
    if (not raw_line.endswith("\n") or raw_line.count("\n") != 1 or
            "\r" in raw_line):
        raise EvidenceError(f"unterminated/non-canonical stdout record: {raw_line!r}")
    line = raw_line[:-1]
    if not line:
        raise EvidenceError("blank SCORE stdout record")
    try:
        preamble = parse_engine_preamble(line)
    except (PreambleError, ValueError) as exc:
        # See ScoreStdoutClassifier.classify's identical clause: a
        # numeric preamble field beyond Python's int-string conversion
        # limit raises a bare ValueError from engine_evidence, not its
        # own PreambleError -- refuse it with a named error here too
        # rather than let it escape uncaught.
        raise EvidenceError(str(exc)) from exc
    if preamble is not None:
        return None
    return parse_score_result(line)


class ScoreStdoutClassifier:
    """Own exactly one banner then one load record before SCORE results.

    Beyond the banner/load preamble, no other multiplexed-serve global
    record (``PROF``, ``HITS``, ``EMAP``, ...) or stderr-only banner
    (``[prefill]``, ``[PIN]``, ``[USAGE]``, ...) can ever reach this
    classifier: ``run_score`` never prints them to stdout (verified
    against the engine source), so any such line arriving here is
    refused by name like any other unrecognized record, not specially
    recognized or passed through.
    """

    def __init__(self):
        self._state = 0

    def classify(self, raw_line):
        if (not raw_line.endswith("\n") or raw_line.count("\n") != 1 or
                "\r" in raw_line):
            raise EvidenceError(
                f"unterminated/non-canonical stdout record: {raw_line!r}")
        line = raw_line[:-1]
        if not line:
            raise EvidenceError("blank SCORE stdout record")
        try:
            if self._state == 0:
                parse_engine_banner(line)
                self._state = 1
                return None
            if self._state == 1:
                parse_engine_loaded(line)
                self._state = 2
                return None
            preamble = parse_engine_preamble(line)
        except (PreambleError, ValueError) as exc:
            # A numeric preamble field beyond Python's 4300-digit
            # int-string conversion limit raises a bare ValueError from
            # engine_evidence's own int()/float() calls, not its
            # PreambleError -- this module's contract is to refuse every
            # non-canonical record with a named error, never to let one
            # escape uncaught.
            raise EvidenceError(str(exc)) from exc
        if preamble is not None:
            raise EvidenceError(f"duplicate/out-of-order SCORE preamble: {line!r}")
        return parse_score_result(line)

    def finish(self):
        if self._state != 2:
            missing = "engine banner" if self._state == 0 else "engine load record"
            raise EvidenceError(f"missing {missing} before SCORE EOF")


def completion_error(returncode, completed, expected, continuation_tokens,
                     stream_error=None):
    """Return the reason this run has NOTHING trustworthy to report, or
    None if it can report a table (even a partial one).

    Matches dev's own exit-code contract exactly, since callers such as
    ``coli bench`` (``sys.exit(subprocess.call(cmd, ...))``) and
    ``diag_harness.py`` (which parses this tool's own accuracy table from
    a subprocess call) depend on it: dev exits nonzero ONLY when the
    engine itself produced nothing at all (nonzero exit AND zero
    requests scored); a partial run -- some but not all requests scored,
    or the engine exiting nonzero after scoring at least one request --
    still prints the accuracy table over whatever landed and exits 0.
    Nothing here checks ``expected``/``continuation_tokens`` against a
    denominator: by the time this runs, ``expected`` (the request count)
    is always positive (an empty selection is refused before the engine
    ever launches), and a positive ``completed`` count always carries a
    positive token count by construction.

    ``stream_error`` is the one condition dev's own contract has no
    analog for: a genuinely corrupted or self-inconsistent SCORE stream,
    which this module's evidence layer can detect and dev's plain
    per-line filter cannot. Concretely, as of this module's current
    stdout classifier: a foreign, malformed, or duplicated/out-of-order
    banner-or-load preamble line; a SCORE line whose numeric token is
    not an exact finite %.6f/%.17g spelling (a bare "nan"/"inf" included
    -- neither survives either spelling); a completed request's contlen
    not matching what that request actually asked for; more SCORE
    result lines than requests were sent; or the stream ending before
    both preamble records were ever seen. That failure stays fatal
    regardless of how many requests completed, because once any one
    line fails to parse as a complete, well-formed record, nothing
    guarantees the corruption did not already touch earlier-looking-fine
    rows too -- unlike the DATA/logprobs serving channel elsewhere in
    this project, where a non-finite value on a masked row is expected
    and reported, not fatal, this SCORE/benchmark channel has no such
    legitimate non-finite case: a nan/inf SCORE value cannot be compared
    against a benchmark's gold answer, and reporting an accuracy table
    built partly from corrupted numbers is a wrong PASS, which this
    module's whole design treats as worse than refusing to report at
    all. (This docstring previously cited identity-bound-evidence
    conditions -- mixed identity-bound/legacy records, a replayed
    digest, an out-of-vocabulary token -- that no longer exist: the
    identity-binding mode itself was removed as unsatisfiable by any
    current engine build. The reasoning above is unchanged; only the
    concrete list of what can still reach this path is corrected.)
    """
    if stream_error:
        return str(stream_error)
    if returncode != 0 and completed == 0:
        return f"engine exited {returncode} with zero requests scored"
    return None


def write_result_row(out_f, req_idx, meta_row, exact_logprob, greedy):
    """Write the exact engine token, never a rounded float reconstruction."""
    task, qi, oi, clen, cchars, gold = meta_row
    out_f.write(f"{req_idx},{task},{qi},{oi},{clen},{cchars},{gold},"
                f"{exact_logprob},{greedy}\n")


def prelaunch_incomplete(out_path, reason):
    """Refuse vacuous evidence before Popen and durably mark writable output."""
    message = f"EVIDENCE INCOMPLETE before engine launch: {reason}"
    print(message, file=sys.stderr)
    if out_path:
        try:
            with open(out_path, "a") as out_f:
                out_f.write(f"# INCOMPLETE: 0/0; error={reason}\n")
        except OSError as exc:
            print(f"cannot mark output {out_path!r} INCOMPLETE: {exc}",
                  file=sys.stderr)
    return 1

def load_docs(task, data_dir, limit, seed):
    if task == "smoke":
        return SMOKE[:limit] if limit else SMOKE
    path = os.path.join(data_dir, task + ".jsonl")
    if not os.path.exists(path):
        sys.exit(f"missing {path} — generate it with: python3 tools/fetch_benchmarks.py --out {data_dir} --tasks {task}")
    docs = [json.loads(l) for l in open(path) if l.strip()]
    random.Random(seed).shuffle(docs)
    return docs[:limit] if limit else docs

def detect_prefix(snap):
    """GLM sees [gMASK]<sop> at the start of every training sequence; scoring raw text
    without it is out-of-distribution and silently depresses/distorts scores (#108).
    Default the prefix ON for GLM snapshots; EVAL_PREFIX (even empty) overrides."""
    if "EVAL_PREFIX" in os.environ: return os.environ["EVAL_PREFIX"]
    try: mt = json.load(open(os.path.join(snap, "config.json"))).get("model_type", "")
    except Exception: mt = ""
    if "glm" in mt.lower():
        print("[prefix] GLM snapshot: prepending [gMASK]<sop> to every context "
              "(override with EVAL_PREFIX, disable with EVAL_PREFIX=)", file=sys.stderr)
        return "[gMASK]<sop>"
    return ""

def build_requests(tk, docs_by_task, prefix=""):
    reqs, meta, perq = [], [], {}
    for t, docs in docs_by_task.items():
        for qi, d in enumerate(docs):
            ctx, conts, gold = prefix + d["ctx"], d["choices"], int(d["gold"])
            ctx_ids = tk.encode(ctx).ids
            for oi, cont in enumerate(conts):
                full = tk.encode(ctx + cont).ids
                cl = len(ctx_ids)
                while cl > 0 and (cl > len(full) or full[:cl] != ctx_ids[:cl]): cl -= 1
                cont_ids = full[cl:]
                if cl < 1 or not cont_ids:              # boundary degenere: forza split esplicito
                    choice_ids = tk.encode(cont).ids
                    full = ctx_ids + choice_ids
                    cl = len(ctx_ids)
                    cont_ids = choice_ids
                if cl < 1 or not cont_ids:
                    raise EvidenceError(
                        f"{t} question {qi} choice {oi} has no positive "
                        "context/continuation token denominator")
                reqs.append(f"{cl} {len(cont_ids)} " + " ".join(map(str, full)))
                meta.append((t, qi, oi, len(cont_ids), max(1, len(cont)), gold))
                perq.setdefault((t, qi), []).append(len(meta) - 1)
    return reqs, meta, perq


def score_snapshot_vocab(snap):
    """Read the engine's independently loaded vocabulary bound from config."""
    path = os.path.join(snap, "config.json")
    try:
        with open(path, "rb") as source:
            source.seek(0, os.SEEK_END)
            length = source.tell()
            _checked_engine_text_size(length, "SCORE config.json")
            source.seek(0)
            raw = source.read(_ENGINE_TEXT_MAX_BYTES + 1)
        _checked_engine_text_size(len(raw), "SCORE config.json")
        config = json.loads(raw.decode("utf-8"))
        vocab = config["vocab_size"]
    except (OSError, UnicodeDecodeError, ValueError,
            KeyError, TypeError) as exc:
        # ValueError (not just its json.JSONDecodeError subclass): a
        # numeric literal in the JSON beyond Python's 4300-digit
        # int-string conversion limit makes json.loads itself raise a
        # bare ValueError, not JSONDecodeError -- same error-contract
        # class as F3's engine_evidence fix. main()'s only caller
        # catches EvidenceError to mark the run INCOMPLETE before ever
        # launching the engine; a bare ValueError escaping here instead
        # crashes main() outright with no INCOMPLETE marker written.
        raise EvidenceError(f"cannot derive SCORE vocabulary from {path}: {exc}") from exc
    if type(vocab) is not int or not 1 <= vocab <= 1 << 24:
        raise EvidenceError(f"invalid SCORE vocabulary in {path}: {vocab!r}")
    return vocab


def score_request_wire(requests, vocab):
    """Return strict ASCII/LF records, joined bytes, and per-record SHA-256.

    The C SCORE evidence mode hashes the ``getline`` byte span, including LF;
    this helper owns the identical byte domain before the temporary file exists.
    """
    if type(vocab) is not int or not 1 <= vocab <= 1 << 24:
        raise EvidenceError(f"invalid SCORE vocabulary: {vocab!r}")
    lines = []
    continuation_tokens = 0
    image_bytes = 0
    for request in requests:
        if not isinstance(request, str) or not request or "\n" in request or "\r" in request:
            raise EvidenceError(f"request is not one canonical line: {request!r}")
        fields = request.split(" ")
        if (" ".join(fields) != request or len(fields) < 4 or
                any(not re.fullmatch(_UINT_TEXT, field) for field in fields)):
            raise EvidenceError(f"request is not canonical integer grammar: {request!r}")
        values = [int(field) for field in fields]
        ctxlen, contlen = values[:2]
        if (not 1 <= ctxlen <= _INT32_MAX or
                not 1 <= contlen <= _INT32_MAX - ctxlen):
            raise EvidenceError(f"request lengths are invalid: {request!r}")
        total = ctxlen + contlen
        tokens = values[2:]
        if len(tokens) != total:
            raise EvidenceError(f"request token count is invalid: {request!r}")
        if any(token >= vocab for token in tokens):
            raise EvidenceError(f"request token is outside vocabulary: {request!r}")
        if continuation_tokens > (1 << 63) - 1 - contlen:
            raise EvidenceError("SCORE continuation denominator exceeds int64")
        continuation_tokens += contlen
        try:
            line = (request + "\n").encode("ascii")
        except UnicodeEncodeError as exc:
            raise EvidenceError(
                f"request is not canonical ASCII: {request!r}") from exc
        image_bytes = _checked_engine_text_size(
            image_bytes + len(line), "SCORE request image")
        lines.append(line)
    if not lines or continuation_tokens <= 0 or len(lines) > _INT32_MAX:
        raise EvidenceError("SCORE request image has no positive denominator")
    frozen = tuple(lines)
    return (frozen, b"".join(frozen),
            tuple(hashlib.sha256(line).hexdigest() for line in frozen))

def score_accuracy(tasks, meta, perq, lp):
    print(f"\n{'task':<18} {'n':>4} {'acc':>7} {'acc_norm':>9}")
    overall = []
    for t in tasks:
        qs = [k for k in perq if k[0] == t]
        acc = accn = 0
        for k in qs:
            ridx = perq[k]; gold = meta[ridx[0]][5]
            best  = max(ridx, key=lambda r: lp[r])
            bestn = max(ridx, key=lambda r: lp[r] / meta[r][4])    # acc_norm: per carattere
            acc  += (meta[best][2]  == gold)
            accn += (meta[bestn][2] == gold)
        n = len(qs)
        if not n: continue
        print(f"{t:<18} {n:>4} {100*acc/n:>6.1f}% {100*accn/n:>8.1f}%")
        overall.append(100 * accn / n)
        for mdl, sc in REFERENCE.get(t, {}).items():
            if sc is not None: print(f"{'  ref '+mdl:<18} {'':>4} {'':>7} {sc:>8.1f}%")
    if overall:
        print(f"\nMEAN acc_norm: {sum(overall)/len(overall):.1f}% across {len(overall)} tasks")

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--snap", required=True)
    ap.add_argument("--glm", default="./glm")
    ap.add_argument("--data", default="./bench")
    ap.add_argument("--tasks", default="smoke")
    ap.add_argument("--limit", type=int, default=40)
    ap.add_argument("--ram", type=int, default=0)
    ap.add_argument("--cap", type=int, default=64)  # pinned deliberately: benchmarks need a fixed cache size for reproducibility, not the platform-aware auto (#379)
    ap.add_argument("--bits", default="")
    ap.add_argument("--seed", type=int, default=1234)
    ap.add_argument("--dry", action="store_true", help="build requests and stop without running the engine")
    ap.add_argument("--selftest", action="store_true", help="verify the scoring calculations")
    ap.add_argument("--out", default="", help="write incremental results CSV here (one row per request, flushed as it lands)")
    a = ap.parse_args()

    if a.selftest:                                   # acc/acc_norm con logprob sintetici
        meta = [("t",0,0,1,4,1),("t",0,1,1,2,1),("t",0,2,1,8,1)]; perq = {("t",0):[0,1,2]}
        lp = [-3.0, -2.0, -5.0]                       # opt1 ha lp piu' alto -> acc sceglie 1 (=gold) OK
        score_accuracy(["t"], meta, perq, lp)
        print("selftest OK" if True else ""); return

    tasks = [t.strip() for t in a.tasks.split(",") if t.strip()]
    if not tasks:
        return prelaunch_incomplete(a.out,"no benchmark tasks selected")

    from tokenizers import Tokenizer
    tk = Tokenizer.from_file(os.path.join(a.snap, "tokenizer.json"))
    docs_by_task = {t: load_docs(t, a.data, a.limit, a.seed) for t in tasks}
    for t, d in docs_by_task.items(): print(f"[{t}] {len(d)} questions", file=sys.stderr)

    try:
        reqs, meta, perq = build_requests(
            tk, docs_by_task, detect_prefix(a.snap))
    except EvidenceError as exc:
        return prelaunch_incomplete(a.out, str(exc))
    print(f"total requests: {len(reqs)} (answer options)", file=sys.stderr)
    if not reqs:
        return prelaunch_incomplete(a.out,"selected tasks produced zero SCORE requests")
    if a.dry:
        # Matches dev exactly: --dry stops right after request
        # construction, before the vocabulary lookup below -- it never
        # needed config.json's vocab_size (a plumbing check has no
        # engine, and therefore no vocabulary, to bind requests against).
        for r in reqs[:3]: print("  example request:", r[:80], "...", file=sys.stderr)
        print("DRY: request construction and tokenization passed. Engine was not run.", file=sys.stderr); return
    try:
        score_vocab = score_snapshot_vocab(a.snap)
        _, request_payload, _ = score_request_wire(
            reqs, score_vocab)
    except EvidenceError as exc:
        return prelaunch_incomplete(a.out, str(exc))

    # mkstemp (non mktemp): crea il file atomicamente con permessi 0600, niente
    # race TOCTOU/symlink su una tmp dir condivisa (CWE-377).
    fd, req_path = tempfile.mkstemp(suffix=".txt")
    with os.fdopen(fd, "wb") as f:
        written = f.write(request_payload)
        if written != len(request_payload):
            raise EvidenceError("short write while freezing SCORE requests")
    env = dict(os.environ, SNAP=a.snap, SCORE=req_path)
    if a.ram: env["RAM_GB"] = str(a.ram)
    cmd = [a.glm, str(a.cap)] + a.bits.split()
    print("running:", " ".join(cmd), file=sys.stderr)

    # Stream results line-by-line so a crash at request N keeps 1..N-1 and shows
    # exactly where it stopped. The engine prints "<lp> <contlen> <greedy>" per
    # request to stdout and "[score N req | ...]" progress to stderr; buffering
    # both until exit (the old subprocess.run) wastes the whole run on a crash.
    out_f = open(a.out, "a") if a.out else None
    if out_f:
        out_f.write(f"# eval_glm snap={a.snap} tasks={a.tasks} limit={a.limit} seed={a.seed} started={time.strftime('%Y-%m-%dT%H:%M:%S')}\n")
        out_f.write("req_idx,task,qi,oi,contlen,contchars,gold,logprob,greedy\n")
        out_f.flush()
    t0 = time.time()
    proc = None
    previous_sigterm = None

    def _on_sigterm(signum, frame):
        raise ChildTerminateRequested(f"received signal {signum}")

    try:
        # A SIGTERM (or Ctrl+C's SIGINT, which Python already converts to
        # KeyboardInterrupt on its own) must not leave the engine child
        # running as an orphan/zombie -- the handler below converts
        # SIGTERM into the same exception path, so both unwind through
        # the identical `finally` cleanup that terminates the child.
        previous_sigterm = signal.signal(signal.SIGTERM, _on_sigterm)
        proc = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                                text=True, bufsize=1)  # line-buffered
        lp = [None] * len(reqs)
        n_done = 0
        continuation_tokens = 0
        stream_error = None
        stdout_classifier = ScoreStdoutClassifier()
        # Drain stderr (engine progress lines) to console live on a background thread
        # so the [score N req] heartbeat is visible while stdout is consumed below.
        def _drain_stderr():
            for line in proc.stderr:
                print(f"  [engine] {line.rstrip()}", file=sys.stderr)
        threading.Thread(target=_drain_stderr, daemon=True).start()
        for raw_line in proc.stdout:
            if stream_error:
                continue                    # drain fully so the child cannot block
            try:
                result = stdout_classifier.classify(raw_line)
            except EvidenceError as exc:
                stream_error = exc
                continue
            if result is None:
                continue
            if n_done >= len(reqs):
                stream_error = EvidenceError("engine emitted extra SCORE result lines")
                continue
            try:
                exact, logprob, contlen, greedy = result
                if contlen != meta[n_done][3]:
                    raise EvidenceError(
                        f"request {n_done} contlen {contlen} != expected {meta[n_done][3]}")
            except EvidenceError as exc:
                stream_error = exc
                continue
            lp[n_done] = logprob
            continuation_tokens += contlen
            t, qi, oi, clen, cchars, gold = meta[n_done]
            if out_f:
                write_result_row(out_f,n_done,meta[n_done],exact,greedy)
                out_f.flush()
            n_done += 1
            if n_done % 5 == 0 or n_done == len(reqs):
                elapsed = time.time() - t0
                rate = n_done / elapsed if elapsed > 0 else 0
                eta = (len(reqs) - n_done) / rate if rate > 0 else 0
                print(f"[progress] {n_done}/{len(reqs)} requests scored | {elapsed:.0f}s elapsed | "
                      f"{rate:.2f} req/s | ETA {eta:.0f}s | last: {t} q{qi} opt{oi} lp={logprob:.3f}",
                      file=sys.stderr)
        if not stream_error:
            try:
                stdout_classifier.finish()
            except EvidenceError as exc:
                stream_error = exc
        proc.wait()
        elapsed = time.time() - t0
        # Fatal only in the two cases dev's own contract (and this
        # module's own evidence layer) recognize -- see completion_error's
        # docstring. Everything else, including a partial request count,
        # still reports the accuracy table and exits 0, matching dev.
        fatal = completion_error(
            proc.returncode,n_done,len(reqs),continuation_tokens,stream_error)
        partial = n_done != len(reqs)
        if out_f:
            if fatal:
                out_f.write(f"# INCOMPLETE: {n_done}/{len(reqs)} in {elapsed:.0f}s, "
                            f"tokens={continuation_tokens}, exit={proc.returncode}; "
                            f"error={fatal}\n")
            else:
                out_f.write(f"# finished: {n_done}/{len(reqs)} in {elapsed:.0f}s, "
                            f"tokens={continuation_tokens}, exit={proc.returncode}\n")
                if partial:
                    # Additive: the run still finished (exit 0, full
                    # table below) -- this line only ANNOUNCES that fewer
                    # than the full request count landed, it does not
                    # replace the "# finished" line or change the exit
                    # code dev's own consumers depend on.
                    out_f.write(f"# INCOMPLETE: {n_done}/{len(reqs)} requests "
                                f"scored; engine exit={proc.returncode}\n")
            out_f.close(); out_f=None
        if fatal:
            print(f"EVIDENCE INCOMPLETE: {fatal}", file=sys.stderr)
            return 1
        if partial:
            # Same wording and the same exit-0 contract dev used: a
            # partial run is a WARNING, not a failure.
            print(f"WARNING: only {n_done}/{len(reqs)} requests scored "
                  f"(engine exited {proc.returncode}); scoring partial "
                  "results.", file=sys.stderr)
        # Fill any unscored slots with -inf so argmax never picks them
        # (dev's own fallback for a partial run).
        for i in range(len(lp)):
            if lp[i] is None: lp[i] = float("-inf")
        print(f"(engine: {elapsed:.0f}s, {n_done}/{len(reqs)} scored, "
              f"{continuation_tokens} continuation tokens, "
              f"exit {proc.returncode})", file=sys.stderr)
        score_accuracy(tasks, meta, perq, lp)
        print("\nNOTE: compare acc_norm with GLM-5.2's PUBLISHED model-card score. A close result"
              "\n      indicates that int4 quantization preserved quality. (Fill REFERENCE in tools/eval_glm.py.)")
        return 0
    finally:
        if previous_sigterm is not None:
            signal.signal(signal.SIGTERM, previous_sigterm)
        if proc is not None and proc.poll() is None:
            # A mid-run exception or termination signal must never leave
            # the engine child running: no zombie, no orphan.
            proc.terminate()
            proc.wait()
        if out_f:
            out_f.write("# INCOMPLETE: evaluator terminated before a complete denominator\n")
            out_f.close()
        try: os.remove(req_path)
        except FileNotFoundError: pass

if __name__ == "__main__":
    sys.exit(main() or 0)