Skip to content

A computer, compiled into the weights of a language model. An exact CPU inside Qwen3.5-4B, with Gated DeltaNet state as RAM.

27.5 KB
Raw Blame
"""Text benchmarks: is a merged Qwen3.5 + CHIP-8 checkpoint as capable as its base on pure text?

    python -m chipformer.q35.text_eval run MODEL --name NAME [--attn sdpa|eager] [--tokenizer BASE]
    python -m chipformer.q35.text_eval table [dist/evals/text] [--ref base]

``run`` evaluates one checkpoint and writes ``OUT/NAME/TASK.json`` (per-item records; a task whose
file exists is skipped, so a killed run resumes). ``table`` compares every run against ``--ref``.
All runs share one tokenizer (pass the base's) and identical batches, so items line up.

Tasks (all prompts in Qwen's chat template with thinking off, except the ARC/HellaSwag contexts):
  mmlu       cais/mmlu, 1000 seeded test questions, 0-shot; prediction = argmax over A/B/C/D
             logits at the first answer token
  arc        ARC-Challenge test (1172), continuation log-likelihood, acc and acc_norm
  hellaswag  1000 seeded validation items, continuation log-likelihood, acc_norm
  gsm8k      200 seeded test problems, greedy, no tools, final answer in \\boxed{}
  ifeval     google/IFEval (541), greedy, scored by lm-eval's checker (strict and loose)
  niah       needle (a 7-digit "magic number") in a wikitext-103 haystack at 8k/16k/32k tokens,
             chunked prefill; greedy answer and the teacher-forced log-prob of the needle
  ppl        8 disjoint 32k-token wikitext-103 windows: per-token NLL by position and greedy
             next-token agreement (a dense, long-context fidelity measure)

The noise floor is the base model under another attention implementation (eager vs sdpa), which
changes nothing but the rounding inside Qwen's full-attention layers.
"""

import argparse
import itertools
import json
import math
import random
import re
import sys
import time
from pathlib import Path

import numpy as np
import torch

TASKS = ("mmlu", "arc", "hellaswag", "gsm8k", "ifeval", "niah", "ppl")
TEXT_VOCAB = 248320  # Qwen's own vocabulary; the merged checkpoint's chip tokens come after it
TOKEN_BUDGET = 4096  # tokens per log-likelihood batch
GEN_BATCH = 16
CHUNK = 1024  # prefill chunk for the long-context check, same for every model


# --- model ------------------------------------------------------------------------------------


def load(path, attn, mem_gb):
    from transformers import AutoModelForImageTextToText

    if mem_gb:
        total = torch.cuda.get_device_properties(0).total_memory / 2**30
        torch.cuda.set_per_process_memory_fraction(min(1.0, mem_gb / total))
    model = AutoModelForImageTextToText.from_pretrained(
        path, dtype=torch.bfloat16, attn_implementation=attn)
    del model.model.visual  # text only; frees ~0.7 GB
    return model.to("cuda").eval()


def chat(tok, text):
    return tok.apply_chat_template([{"role": "user", "content": text}], add_generation_prompt=True,
                                   return_tensors="pt", enable_thinking=False)["input_ids"][0].tolist()


def batches_by_budget(lengths, budget=TOKEN_BUDGET, max_b=64):
    """Indices sorted by length, grouped so that batch * max_len <= budget."""
    order = sorted(range(len(lengths)), key=lambda i: -lengths[i])
    out, cur = [], []
    for i in order:
        if cur and (len(cur) + 1) * lengths[cur[0]] > budget or len(cur) == max_b:
            out.append(cur)
            cur = []
        cur.append(i)
    return out + [cur] if cur else out


@torch.no_grad()
def target_logprobs(model, seqs, queries, pad):
    """For each sequence, log-probs (full vocabulary) of chosen tokens at chosen positions.

    ``queries[i]`` is a list of (position, [token ids]); returns, per sequence, a list of
    (logprobs of those ids, argmax id) per query. Right padding, so real positions are exact.
    """
    out = [None] * len(seqs)
    for b in batches_by_budget([len(s) for s in seqs]):
        n = max(len(seqs[i]) for i in b)
        ids = torch.full((len(b), n), pad, dtype=torch.long)
        mask = torch.zeros((len(b), n), dtype=torch.long)
        for r, i in enumerate(b):
            ids[r, :len(seqs[i])] = torch.tensor(seqs[i])
            mask[r, :len(seqs[i])] = 1
        h = model.model(input_ids=ids.cuda(), attention_mask=mask.cuda(),
                        use_cache=False).last_hidden_state
        rows = [(r, p, toks) for r, i in enumerate(b) for p, toks in queries[i]]
        sel = h[torch.tensor([r for r, _, _ in rows]), torch.tensor([p for _, p, _ in rows])]
        vals, tops = [], []
        for c in range(0, len(rows), 256):  # bounded memory for the vocabulary-wide softmax
            lp = torch.log_softmax(model.lm_head(sel[c:c + 256]).float(), -1)
            tops += lp.argmax(-1).tolist()
            rr = [k - c for k in range(c, min(c + 256, len(rows))) for _ in rows[k][2]]
            tt = [t for k in range(c, min(c + 256, len(rows))) for t in rows[k][2]]
            vals += lp[rr, tt].tolist()
        k = v = 0
        for i in b:
            res = []
            for _, toks in queries[i]:
                res.append((vals[v:v + len(toks)], tops[k]))
                k, v = k + 1, v + len(toks)
            out[i] = res
    return out


@torch.no_grad()
def generate(model, prompts, max_new, eos, pad, batch=GEN_BATCH):
    """Greedy, batched with left padding; batches of similar prompt length, same for every
    model. Returns the new tokens of each prompt, cut after the first EOS."""
    order = sorted(range(len(prompts)), key=lambda i: -len(prompts[i]))
    out = [None] * len(prompts)
    for s in range(0, len(order), batch):
        b = order[s:s + batch]
        n = max(len(prompts[i]) for i in b)
        ids = torch.full((len(b), n), pad, dtype=torch.long)
        mask = torch.zeros((len(b), n), dtype=torch.long)
        for r, i in enumerate(b):
            ids[r, n - len(prompts[i]):] = torch.tensor(prompts[i])
            mask[r, n - len(prompts[i]):] = 1
        model.model.rope_deltas = None
        g = model.generate(input_ids=ids.cuda(), attention_mask=mask.cuda(), max_new_tokens=max_new,
                           do_sample=False, temperature=None, top_p=None, top_k=None,
                           eos_token_id=eos, pad_token_id=pad)[:, n:].tolist()
        for r, i in enumerate(b):
            toks = g[r]
            cut = next((j for j, t in enumerate(toks) if t in eos), None)
            out[i] = toks if cut is None else toks[:cut + 1]
        print(f"  gen {s + len(b)}/{len(prompts)}", flush=True)
    return out


def gen_record(tok, toks, eos):
    text = tok.decode([t for t in toks if t < TEXT_VOCAB], skip_special_tokens=True)
    return {"text": text, "tokens": toks, "n": len(toks), "finished": bool(toks) and toks[-1] in eos,
            "chip_tokens": sum(t >= TEXT_VOCAB for t in toks)}


# --- tasks ------------------------------------------------------------------------------------


def task_mmlu(model, tok, ctx, n=1000, seed=0):
    from datasets import load_dataset

    ds = load_dataset("cais/mmlu", "all", split="test")
    idx = sorted(random.Random(seed).sample(range(len(ds)), n))
    letters = [tok.encode(c, add_special_tokens=False) for c in "ABCD"]
    assert all(len(x) == 1 for x in letters)
    letters = [x[0] for x in letters]
    seqs, items = [], []
    for i in idx:
        d = ds[i]
        q = (f"The following is a multiple choice question about {d['subject'].replace('_', ' ')}."
             f"\n\n{d['question'].strip()}\n"
             + "".join(f"{c}. {t}\n" for c, t in zip("ABCD", d["choices"]))
             + "\nAnswer with the letter of the correct option only.")
        seqs.append(chat(tok, q))
        items.append({"idx": i, "subject": d["subject"], "gold": d["answer"]})
    res = target_logprobs(model, seqs, [[(len(s) - 1, letters)] for s in seqs], ctx["pad"])
    for it, ((lp, top),) in zip(items, res):
        it.update(lp=lp, pred=int(np.argmax(lp)), top_is_letter=top in letters)
        it["correct"] = it["pred"] == it["gold"]
    return items


def choice_items(model, tok, ctx, docs):
    """docs: (context string, [continuation strings], gold). Continuation log-likelihoods."""
    seqs, queries, meta = [], [], []
    for c, conts, gold in docs:
        cids = tok.encode(c, add_special_tokens=False)
        meta.append((gold, [len(x) for x in conts]))
        for x in conts:
            xids = tok.encode(x, add_special_tokens=False)
            seqs.append(cids + xids)
            queries.append([(len(cids) - 1 + j, [t]) for j, t in enumerate(xids)])
    res = target_logprobs(model, seqs, queries, ctx["pad"])
    items, k = [], 0
    for gold, chars in meta:
        lp, greedy = [], []
        for _ in chars:
            lp.append(sum(r[0][0] for r in res[k]))
            greedy.append(all(r[1] == q[1][0] for r, q in zip(res[k], queries[k])))
            k += 1
        norm = [v / n for v, n in zip(lp, chars)]
        items.append({"gold": gold, "lp": lp, "pred": int(np.argmax(lp)),
                      "pred_norm": int(np.argmax(norm)), "greedy": greedy})
        items[-1]["correct"] = items[-1]["pred"] == gold
        items[-1]["correct_norm"] = items[-1]["pred_norm"] == gold
    return items


def task_arc(model, tok, ctx):
    from datasets import load_dataset

    ds = load_dataset("allenai/ai2_arc", "ARC-Challenge", split="test")
    docs = [(f"Question: {d['question']}\nAnswer:", [" " + t for t in d["choices"]["text"]],
             d["choices"]["label"].index(d["answerKey"])) for d in ds]
    items = choice_items(model, tok, ctx, docs)
    for d, it in zip(ds, items):
        it["id"] = d["id"]
    return items


def _hs(text):
    text = text.strip().replace(" [title]", ". ")
    return re.sub(r"\[.*?\]", "", text).replace("  ", " ")


def task_hellaswag(model, tok, ctx, n=1000, seed=0):
    from datasets import load_dataset

    ds = load_dataset("Rowan/hellaswag", split="validation")
    idx = sorted(random.Random(seed).sample(range(len(ds)), n))
    docs = []
    for i in idx:
        d = ds[i]
        c = _hs(d["activity_label"] + ": " + d["ctx_a"] + " " + d["ctx_b"].capitalize())
        docs.append((c, [" " + _hs(e) for e in d["endings"]], int(d["label"])))
    items = choice_items(model, tok, ctx, docs)
    for i, it in zip(idx, items):
        it["idx"] = i
    return items


def gsm_answer(text):
    boxed = re.findall(r"\\boxed\{((?:[^{}]|\{[^{}]*\})*)\}", text)
    s = boxed[-1] if boxed else text
    nums = re.findall(r"-?\d[\d,]*(?:\.\d+)?", s.replace("$", ""))
    return nums[-1].replace(",", "").rstrip(".") if nums else None


def same_number(a, b):
    try:
        return a is not None and abs(float(a) - float(b)) < 1e-6
    except ValueError:
        return False


def task_gsm8k(model, tok, ctx, n=200, seed=0, max_new=1024):
    from datasets import load_dataset

    ds = load_dataset("openai/gsm8k", "main", split="test")
    idx = sorted(random.Random(seed).sample(range(len(ds)), n))
    prompts = [chat(tok, ds[i]["question"] + "\nPlease reason step by step, and put your final "
                    "answer within \\boxed{}.") for i in idx]
    gens = generate(model, prompts, max_new, ctx["eos"], ctx["pad"])
    items = []
    for i, g in zip(idx, gens):
        gold = ds[i]["answer"].split("####")[-1].strip().replace(",", "")
        r = gen_record(tok, g, ctx["eos"])
        r.update(idx=i, gold=gold, pred=gsm_answer(r["text"]))
        r["correct"] = same_number(r["pred"], gold)
        items.append(r)
    return items


def ifeval_utils():
    """lm-eval's IFEval checker, imported without lm_eval/__init__ (which pulls in the whole
    harness). Needs langdetect, immutabledict, nltk (punkt_tab)."""
    import importlib.util
    import types

    root = Path(importlib.util.find_spec("lm_eval").submodule_search_locations[0])
    for name, p in (("lm_eval", root), ("lm_eval.tasks", root / "tasks")):
        if name not in sys.modules:
            m = types.ModuleType(name)
            m.__path__ = [str(p)]
            sys.modules[name] = m
    from lm_eval.tasks.ifeval import utils

    return utils


def task_ifeval(model, tok, ctx, max_new=1280):
    from datasets import load_dataset

    utils = ifeval_utils()
    ds = load_dataset("google/IFEval", split="train")
    gens = generate(model, [chat(tok, d["prompt"]) for d in ds], max_new, ctx["eos"], ctx["pad"])
    items = []
    for d, g in zip(ds, gens):
        r = gen_record(tok, g, ctx["eos"])
        s = utils.process_results(d, [r["text"]])
        r.update(key=d["key"], strict=s["prompt_level_strict_acc"],
                 loose=s["prompt_level_loose_acc"], inst_strict=s["inst_level_strict_acc"],
                 inst_loose=s["inst_level_loose_acc"])
        r["correct"] = r["strict"]
        items.append(r)
    return items


CITIES = ["Lisbon", "Nairobi", "Osaka", "Quito", "Tbilisi", "Winnipeg", "Hanoi", "Bergen",
          "Cusco", "Dakar", "Perth", "Tallinn"]


def haystack_ids(tok):
    from datasets import load_dataset

    ds = load_dataset("Salesforce/wikitext", "wikitext-103-raw-v1", split="test")
    text = "".join(t for t in ds["text"] if t.strip() and not t.strip().startswith("="))
    return tok.encode(text, add_special_tokens=False)


@torch.no_grad()
def prefill(model, ids, keep=1):
    """Chunked prefill with a fresh cache; logits (text vocab) of the last ``keep`` positions."""
    from transformers import DynamicCache

    model.model.rope_deltas = None  # stale after a batched generate(); text-only deltas are 0
    cache = DynamicCache(config=model.config)
    n = len(ids) - keep
    cuts = list(range(0, n, CHUNK)) + [n, len(ids)]
    for a, b in itertools.pairwise(cuts):
        x = torch.tensor([ids[a:b]], device="cuda")
        out = model(input_ids=x, past_key_values=cache, use_cache=True, logits_to_keep=keep)
    return out.logits[0, :, :TEXT_VOCAB].float(), cache


@torch.no_grad()
def greedy_from(model, logits, cache, n, eos):
    toks = []
    t = int(logits[-1].argmax())
    for _ in range(n):
        toks.append(t)
        if t in eos:
            break
        out = model(input_ids=torch.tensor([[t]], device="cuda"), past_key_values=cache,
                    use_cache=True)
        t = int(out.logits[0, -1].argmax())  # full vocabulary: a chip token would show up
    return toks


def task_niah(model, tok, ctx, lengths=(8192, 16384, 32768), depths=(0.1, 0.3, 0.5, 0.7, 0.9),
              seeds=4):
    hay = haystack_ids(tok)
    items = []
    for L in lengths:
        for depth in depths:
            for s in range(seeds):
                rng = random.Random(L * 100 + int(depth * 10) * 10 + s)
                city, num = rng.choice(CITIES), str(rng.randrange(10**6, 10**7))
                question = (f"What is the special magic number for {city} mentioned in the text? "
                            "Answer with the number only.")
                head = "Read the text below and answer the question that follows.\n\n"
                over = len(chat(tok, head + "\n\n" + question)) + 20
                off = rng.randrange(0, len(hay) - L)
                body = hay[off:off + L - over]
                p = int(depth * len(body))
                left, right = tok.decode(body[:p]), tok.decode(body[p:])
                cut = left.rfind(". ")
                if cut > len(left) - 2000:  # put the needle at a sentence boundary
                    left, right = left[:cut + 2], left[cut + 2:] + right
                needle = f"The special magic number for {city} is {num}. "
                ids = chat(tok, head + left + needle + right + "\n\n" + question)
                ans = tok.encode(num, add_special_tokens=False)
                logits, _ = prefill(model, ids + ans, keep=len(ans) + 1)
                lp = torch.log_softmax(logits, -1)
                gold_lp = float(sum(lp[j, t] for j, t in enumerate(ans)))
                logits, cache = prefill(model, ids)
                g = greedy_from(model, logits, cache, 16, ctx["eos"])
                del cache
                r = gen_record(tok, g, ctx["eos"])
                r.update(length=len(ids), target=L, depth=depth, seed=s, city=city, gold=num,
                         gold_logprob=gold_lp, correct=num in r["text"])
                items.append(r)
                print(f"  niah L={len(ids)} d={depth} s={s} ok={r['correct']} "
                      f"lp={gold_lp:.3f} {r['text']!r}", flush=True)
    return items


@torch.no_grad()
def task_ppl(model, tok, ctx, length=32768, docs=8):
    """Long-context language modelling: per-token NLL and greedy next token over ``docs``
    disjoint ``length``-token windows of wikitext-103 test, chunked prefill."""
    from transformers import DynamicCache

    hay = haystack_ids(tok)
    items = []
    for d in range(docs):
        ids = hay[d * length:(d + 1) * length]
        model.model.rope_deltas = None
        cache = DynamicCache(config=model.config)
        nll, top = [], []
        for a in range(0, length, CHUNK):
            x = torch.tensor([ids[a:a + CHUNK]], device="cuda")
            h = model.model(input_ids=x, past_key_values=cache, use_cache=True).last_hidden_state[0]
            for c in range(0, h.shape[0], 256):
                lp = torch.log_softmax(model.lm_head(h[c:c + 256]).float(), -1)
                p0 = a + c
                tgt = torch.tensor(ids[p0 + 1:p0 + 1 + lp.shape[0]], device="cuda")
                nll += (-lp[torch.arange(len(tgt)), tgt]).tolist()
                top += lp.argmax(-1)[:len(tgt)].tolist()
        del cache
        items.append({"offset": d * length, "nll": [round(v, 5) for v in nll], "top1": top,
                      "target": ids[1:], "correct": float(np.mean(np.array(top) == ids[1:]))})
        print(f"  ppl doc {d}: ppl {math.exp(np.mean(nll)):.3f} top1 {items[-1]['correct']:.4f}",
              flush=True)
    return items


PPL_BUCKETS = ((0, 2048), (2048, 8192), (8192, 16384), (16384, 32768))


def ppl_table(data, names, ref, summary):
    """Rows for the long-context perplexity task: PPL per position bucket, signed mean dNLL vs
    ``ref`` with a bootstrap CI over 1024-token blocks, |dNLL|, greedy next-token agreement."""
    lines = []
    for lo, hi in PPL_BUCKETS:
        cells = []
        for n in names:
            if "ppl" not in data[n]:
                cells.append("-")
                continue
            v = np.concatenate([np.array(it["nll"][lo:hi]) for it in data[n]["ppl"]])
            s = {"ppl": float(np.exp(v.mean())), "tokens": len(v)}
            cell = f"{s['ppl']:.3f}"
            if n != ref:
                r = [np.array(it["nll"][lo:hi]) for it in data[ref]["ppl"]]
                m = [np.array(it["nll"][lo:hi]) for it in data[n]["ppl"]]
                blocks = np.array([(y[k:k + 1024] - x[k:k + 1024]).mean() for x, y in zip(r, m)
                                   for k in range(0, len(x), 1024)])
                boot = np.random.default_rng(0).choice(blocks, (10000, len(blocks))).mean(1)
                dl, dh = np.percentile(boot, [2.5, 97.5])
                d = np.concatenate([y - x for x, y in zip(r, m)])
                agree = np.concatenate([np.array(a["top1"][lo:hi]) == np.array(b["top1"][lo:hi])
                                        for a, b in zip(data[ref]["ppl"], data[n]["ppl"])])
                s.update(dnll=float(d.mean()), dnll_ci95=[float(dl), float(dh)],
                         abs_dnll=float(np.abs(d).mean()), top1_agreement=float(agree.mean()))
                cell += (f"<br>dNLL {d.mean():+.2e} [{dl:+.1e}, {dh:+.1e}], mean abs dNLL "
                         f"{np.abs(d).mean():.2e}, top-1 agree {100 * agree.mean():.2f}%")
            summary.setdefault(n, {}).setdefault("ppl", {})[f"{lo}-{hi}"] = s
            cells.append(cell)
        lines.append(f"| ppl | positions {lo}-{hi} | " + " | ".join(cells) + " |")
    return lines


def stop_tokens(tok):
    """End of turn and end of text stop generation (as in the merged generation_config.json);
    the base snapshot ships no generation config."""
    eos = [tok.convert_tokens_to_ids("<|im_end|>"), tok.convert_tokens_to_ids("<|endoftext|>")]
    return {"eos": eos, "pad": eos[1]}


# --- run --------------------------------------------------------------------------------------


def run(args):
    from transformers import AutoTokenizer

    tok = AutoTokenizer.from_pretrained(args.tokenizer or args.model)
    ctx = stop_tokens(tok)
    out = Path(args.out) / args.name
    out.mkdir(parents=True, exist_ok=True)
    model = None
    for task in args.tasks.split(","):
        f = out / f"{task}.json"
        if f.exists():
            print(f"{args.name}/{task}: done already", flush=True)
            continue
        if model is None:
            model = load(args.model, args.attn, args.mem_gb)
        print(f"{args.name}/{task} ...", flush=True)
        t = time.time()
        torch.cuda.reset_peak_memory_stats()
        items = globals()[f"task_{task}"](model, tok, ctx)
        rec = {"model": args.model, "attn": args.attn, "task": task, "seconds": time.time() - t,
               "peak_gb": torch.cuda.max_memory_allocated() / 2**30, "items": items}
        f.write_text(json.dumps(rec, ensure_ascii=False))
        acc = np.mean([it["correct"] for it in items])
        print(f"{args.name}/{task}: acc {acc:.4f} n={len(items)} {rec['seconds']:.0f}s "
              f"peak {rec['peak_gb']:.1f} GB", flush=True)


# --- statistics -------------------------------------------------------------------------------


def wilson(k, n, z=1.96):
    if n == 0:
        return (float("nan"),) * 2
    p = k / n
    c = (p + z * z / (2 * n)) / (1 + z * z / n)
    h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / (1 + z * z / n)
    return c - h, c + h


def paired_diff(a, b, n_boot=10000, seed=0):
    """Mean of a - b with a paired bootstrap 95% CI, and the exact McNemar p-value."""
    a, b = np.asarray(a, float), np.asarray(b, float)
    rng = np.random.default_rng(seed)
    idx = rng.integers(0, len(a), (n_boot, len(a)))
    d = (a[idx] - b[idx]).mean(1)
    n01, n10 = int(((a == 1) & (b == 0)).sum()), int(((a == 0) & (b == 1)).sum())
    m = n01 + n10
    p = min(1.0, 2 * sum(math.comb(m, k) for k in range(min(n01, n10) + 1)) / 2**m) if m else 1.0
    lo, hi = np.percentile(d, [2.5, 97.5])
    return float((a - b).mean()), float(lo), float(hi), p


def choice_kl(p_lp, q_lp):
    """KL(p || q) between the normalised choice distributions of two runs, per item."""
    p = torch.log_softmax(torch.tensor(p_lp, dtype=torch.float64), -1)
    q = torch.log_softmax(torch.tensor(q_lp, dtype=torch.float64), -1)
    return float((p.exp() * (p - q)).sum())


def first_divergence(x, y):
    return next((i for i, (p, q) in enumerate(zip(x, y)) if p != q),
                None if len(x) == len(y) else min(len(x), len(y)))


METRICS = {  # task -> (metric name, item key) list; the first is the headline
    "mmlu": [("acc", "correct")],
    "arc": [("acc_norm", "correct_norm"), ("acc", "correct")],
    "hellaswag": [("acc_norm", "correct_norm"), ("acc", "correct")],
    "gsm8k": [("acc", "correct")],
    "ifeval": [("prompt_strict", "strict"), ("prompt_loose", "loose")],
    "niah": [("acc", "correct")],
}


def compare_items(task, ref, run):
    """Fine-grained agreement of ``run`` with ``ref`` beyond accuracy."""
    out = {}
    if task in ("mmlu", "arc", "hellaswag"):
        out["same_pred"] = [a["pred"] == b["pred"] for a, b in zip(ref, run)]
        out["mean_choice_kl"] = float(np.mean([choice_kl(a["lp"], b["lp"])
                                               for a, b in zip(ref, run)]))
        out["mean_abs_dlogprob"] = float(np.mean([abs(x - y) for a, b in zip(ref, run)
                                                  for x, y in zip(a["lp"], b["lp"])]))
    elif task == "niah":
        out["same_pred"] = [a["correct"] == b["correct"] for a, b in zip(ref, run)]
        out["mean_abs_dlogprob"] = float(np.mean([abs(a["gold_logprob"] - b["gold_logprob"])
                                                  for a, b in zip(ref, run)]))
        out["identical_text"] = float(np.mean([a["tokens"] == b["tokens"] for a, b in zip(ref, run)]))
    else:
        key = "pred" if task == "gsm8k" else "strict"
        out["same_pred"] = [a[key] == b[key] for a, b in zip(ref, run)]
        out["identical_text"] = float(np.mean([a["tokens"] == b["tokens"] for a, b in zip(ref, run)]))
        fd = [first_divergence(a["tokens"], b["tokens"]) for a, b in zip(ref, run)]
        fd = [x for x in fd if x is not None]
        out["median_first_divergence"] = float(np.median(fd)) if fd else None
        out["unfinished"] = int(sum(not b["finished"] for b in run))
        out["chip_tokens"] = int(sum(b["chip_tokens"] for b in run))
    return out


def table(args):
    root = Path(args.dir)
    names = [args.ref] + [p.name for p in sorted(root.iterdir())
                          if p.is_dir() and p.name != args.ref]
    data = {n: {f.stem: json.loads(f.read_text(encoding="utf-8"))["items"]
                for f in (root / n).glob("*.json")} for n in names}
    summary, lines = {}, []
    lines.append("| task | metric | " + " | ".join(names) + " |")
    lines.append("|---|---|" + "---|" * len(names))
    for task in TASKS:
        if task not in data[args.ref] or task == "ppl":
            continue
        ref = data[args.ref][task]
        for metric, key in METRICS[task]:
            cells = []
            for n in names:
                if task not in data[n]:
                    cells.append("-")
                    continue
                v = [bool(it[key]) for it in data[n][task]]
                lo, hi = wilson(sum(v), len(v))
                s = {"acc": float(np.mean(v)), "ci95": [lo, hi], "n": len(v)}
                cell = f"{100 * s['acc']:.1f} [{100 * lo:.1f}, {100 * hi:.1f}]"
                if n != args.ref:
                    d, dlo, dhi, p = paired_diff(v, [bool(it[key]) for it in ref])
                    s.update(diff=d, diff_ci95=[dlo, dhi], mcnemar_p=p)
                    cell += f"<br>diff {100 * d:+.1f} [{100 * dlo:+.1f}, {100 * dhi:+.1f}] p={p:.2f}"
                summary.setdefault(n, {}).setdefault(task, {})[metric] = s
                cells.append(cell)
            lines.append(f"| {task} | {metric} (n={len(ref)}) | " + " | ".join(cells) + " |")
        cells = []
        for n in names:
            if n == args.ref or task not in data[n]:
                cells.append("ref" if n == args.ref else "-")
                continue
            c = compare_items(task, ref, data[n][task])
            same = c.pop("same_pred")
            lo, hi = wilson(sum(same), len(same))
            c["agreement"] = {"frac": float(np.mean(same)), "ci95": [lo, hi]}
            summary[n][task]["vs_ref"] = c
            extra = ", ".join(f"{k} {v:.3g}" if isinstance(v, float) else f"{k} {v}"
                              for k, v in c.items() if k != "agreement")
            cells.append(f"{100 * np.mean(same):.1f}% [{100 * lo:.1f}, {100 * hi:.1f}]<br>{extra}")
        lines.append(f"| {task} | agreement with {args.ref} | " + " | ".join(cells) + " |")
    if "ppl" in data[args.ref]:
        lines += ppl_table(data, names, args.ref, summary)
    md = "\n".join(lines)
    print(md)
    (root / "summary.json").write_text(json.dumps(summary, indent=2))
    (root / "table.md").write_text(md + "\n", encoding="utf-8")


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
    sub = ap.add_subparsers(dest="cmd", required=True)
    r = sub.add_parser("run")
    r.add_argument("model")
    r.add_argument("--name", required=True)
    r.add_argument("--attn", default="sdpa")
    r.add_argument("--tokenizer", default=None, help="shared tokenizer (the base checkpoint)")
    r.add_argument("--tasks", default=",".join(TASKS))
    r.add_argument("--out", default="dist/evals/text")
    r.add_argument("--mem-gb", type=float, default=20.0, help="per-process GPU memory cap")
    t = sub.add_parser("table")
    t.add_argument("dir", nargs="?", default="dist/evals/text")
    t.add_argument("--ref", default="base")
    args = ap.parse_args()
    run(args) if args.cmd == "run" else table(args)


if __name__ == "__main__":
    main()