Skip to content

A local video studio for your coding agent. Describe a video in one sentence: your agent directs, your machine renders. Claude Code, Codex, Cursor, Devin and more. No API keys, no uploads.

28.3 KB
Raw Blame
#!/usr/bin/env python3
"""Human blind A/B boards (one per task) built with showtime studio, for voting from a phone.

    python benchmarks/scoring/human_board.py build  --run r1 [--task t2] [--focal showtime] [--seed 11] [--rerecord]
    python benchmarks/scoring/human_board.py import --run r1 --task t2 --file feedback.json
    python benchmarks/scoring/human_board.py tally  --run r1

build   Every arm that delivered becomes an unlabeled candidate (letters shuffled per task). Nobody is
        dropped: when the candidates do not fit one page, ALL of them are re-encoded with the same settings
        (same bounding box, bitrate, H.264 + AAC, faststart, no metadata) until the page is <= 15 MB.
        HTML deliverables are screen-recorded to MP4 first, by the same procedure for every page
        (html_record.mjs: Chrome at 1920x1080, play pressed like a viewer would, the video's own length
        if the page exposes it, else 60 s; page audio captured in the page). A candidate that cannot be
        recorded gets a "could not be recorded" card instead of media, and the reason is logged.
        Each candidate carries its video, a poster frame and its length. Up to 5 A/B questions
        ("A or C: which would you ship?"): every pair with the focal arm plus one other pair, so the
        voter cannot tell which letter is the focal one from the questions alone. Voters can also rate
        every candidate 0-5. The board is marked "blind": nothing is recommended.
        Writes <bench home>/human/<run>/<task>.html (studio export --target artifact: a single file to
        publish as a private artifact) and key.json (letter -> arm; never put it on the board), then
        checks that no arm id, arm label or output path appears in the board's text or media metadata.
        One task failing does not stop the others (exit 1 at the end if any failed).
import  Feeds a returned feedback file into the task's studio job (`showtime studio feedback --import`).
tally   Maps answers and ratings back to arms through key.json -> human_summary.json.
"""
import argparse
import base64
import html as htmlmod
import itertools
import json
import os
import random
import re
import shutil
import string
import sys
import tempfile
import traceback
from pathlib import Path
from typing import Dict, List

sys.path.insert(0, str(Path(__file__).resolve().parent))
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "harness"))
import auto_metrics  # noqa: E402
import common  # noqa: E402

PAGE_MAX_BYTES = 15.0e6      # published as an artifact (16 MB hard limit): keep the whole page under this
SHELL_BYTES = 450e3          # page code + board JSON + posters, measured ~0.3 MB; the loop below corrects it
REC_W, REC_H, REC_CAP_S = 1920, 1080, 60
POSTER_BOX = 640


def st(args: List[str], cwd: Path, out_root: Path) -> common.subprocess.CompletedProcess:
    env = dict(os.environ, SHOWTIME_OUT=str(out_root))
    return common.run([common.SHOWTIME] + args, cwd=str(cwd), env=env)


def log(msg: str) -> None:
    common.log("board: " + msg)


# ---------------------------------------------------------------- media

def settings_for(total_s: float, media_bytes: float) -> Dict:
    """One set of encoding settings for every candidate of a board, from the bits each second may use."""
    kbps = media_bytes * 8 / 1000.0 / max(total_s, 1.0)
    audio = (96, 2) if kbps >= 600 else (64, 1)
    video = int(max(120, min(2500, (kbps - audio[0]) * 0.92)))
    box = 1280 if video >= 1600 else 960 if video >= 800 else 854 if video >= 450 else 640
    return {"box": box, "video_kbps": video, "audio_kbps": audio[0], "audio_ch": audio[1]}


def encode(src: Path, dst: Path, s: Dict, audio: bool, tmp: Path) -> None:
    """Same filter, codec, bitrate and container settings for every candidate; never upscales."""
    dst.parent.mkdir(parents=True, exist_ok=True)
    vf = ("scale='min(%d,iw)':'min(%d,ih)':force_original_aspect_ratio=decrease:force_divisible_by=2,format=yuv420p"
          % (s["box"], s["box"]))
    base = [common.ffmpeg(), "-hide_banner", "-loglevel", "error", "-y", "-i", str(src), "-map", "0:v:0", "-vf", vf,
            "-fpsmax", "30", "-c:v", "libx264", "-preset", "slow", "-b:v", "%dk" % s["video_kbps"],
            "-maxrate", "%dk" % int(s["video_kbps"] * 1.5), "-bufsize", "%dk" % (s["video_kbps"] * 2),
            "-passlogfile", str(tmp / "x264")]
    common.run(base + ["-pass", "1", "-an", "-f", "mp4", os.devnull], check=True)
    tail = (["-map", "0:a:0", "-c:a", "aac", "-b:a", "%dk" % s["audio_kbps"], "-ac", str(s["audio_ch"]), "-ar", "48000"]
            if audio else ["-an"])
    common.run(base + ["-pass", "2"] + tail + ["-map_metadata", "-1", "-map_chapters", "-1", "-sn", "-dn",
                                               "-movflags", "+faststart", str(dst)], check=True)


def poster(src: Path, t: float, dst: Path) -> None:
    dst.parent.mkdir(parents=True, exist_ok=True)
    vf = "scale='min(%d,iw)':'min(%d,ih)':force_original_aspect_ratio=decrease" % (POSTER_BOX, POSTER_BOX)
    common.run([common.ffmpeg(), "-hide_banner", "-loglevel", "error", "-y", "-ss", "%.2f" % t, "-i", str(src),
                "-frames:v", "1", "-vf", vf, "-q:v", "5", "-map_metadata", "-1", str(dst)], check=True)


def record_html(src: Path, dst: Path, rerecord: bool) -> Dict:
    """Screen-record one HTML deliverable (html_record.mjs); cached next to the key, off the board."""
    meta = dst.with_suffix(".json")
    sig = {"src": str(src), "bytes": src.stat().st_size, "mtime": src.stat().st_mtime, "w": REC_W, "h": REC_H, "cap": REC_CAP_S}
    old = common.read_json(meta) or {}
    if not rerecord and old.get("sig") == sig and old.get("report", {}).get("ok") and dst.exists():
        return old["report"]
    dst.parent.mkdir(parents=True, exist_ok=True)
    r = common.run([common.node_bin(), str(common.BENCH / "scoring" / "html_record.mjs"), str(src), str(dst),
                    "--width", str(REC_W), "--height", str(REC_H), "--cap", str(REC_CAP_S)], timeout=900)
    try:
        rep = json.loads((r.stdout or "").strip().splitlines()[-1])
    except (ValueError, IndexError):
        rep = {"ok": False, "why": "recorder printed no report (exit %s): %s" % (r.returncode, (r.stderr or "")[-400:])}
    common.write_json(meta, {"sig": sig, "report": rep})
    return rep


# ---------------------------------------------------------------- leak check

def leak_terms() -> List[str]:
    """Arm ids, arm labels, local names of the arms' sources, output-folder names, plus BENCH_LEAK_TERMS."""
    terms = {"showtime-out", "Claude Code", "key.json"}
    try:
        cfg = common.load_arms()
        for a in cfg["arms"]:
            terms.update([a["id"], a.get("label") or ""])
            loc = a.get("local") or {}
            if isinstance(loc, dict) and loc.get("name"):
                terms.add(str(loc["name"]))
    except SystemExit:
        pass
    terms.update(t.strip() for t in os.environ.get("BENCH_LEAK_TERMS", "").split(",") if t.strip())
    return sorted(t for t in terms if len(t) >= 4)


def leak_check(page: Path, terms: List[str]) -> Dict:
    """Search the board's own content (embedded board JSON, <title>, meta tags, and the bytes of every
    embedded media file) for the terms. The page code is the same for every board and candidate; its own
    mentions (the chrome's branding) are counted separately in `chrome`."""
    text = page.read_text(encoding="utf-8", errors="replace")
    m = re.search(r'<script type="application/json" id="board-data">(.*?)</script>', text, re.S)
    data = m.group(1) if m else ""
    media = re.findall(r"data:([a-z]+/[a-z0-9.+-]+);base64,([A-Za-z0-9+/=]+)", data)
    content = re.sub(r"data:[a-z]+/[a-z0-9.+-]+;base64,[A-Za-z0-9+/=]+", "data:", data)
    content = content.replace('"schema":"showtime.studio.board/1"', "")  # the format's fixed id, same on every board
    heads = " ".join(re.findall(r"<title>.*?</title>|<meta name=\"st-job\"[^>]*>", text, re.S))
    board_text = htmlmod.unescape(content + " " + heads)
    rest = text.replace(data, "") if data else text
    rest = re.sub(r"data:[a-z]+/[a-z0-9.+-]+;base64,[A-Za-z0-9+/=]+", "data:", rest)
    hits, chrome = [], {}
    for t in terms:
        rx = re.compile(re.escape(t), re.I)
        for mm in rx.finditer(board_text):
            hits.append({"term": t, "where": "board", "context": board_text[max(0, mm.start() - 40):mm.end() + 40]})
        for mime, b64 in media:
            raw = base64.b64decode(b64)
            if re.search(re.escape(t.encode("utf-8")), raw, re.I):
                hits.append({"term": t, "where": "media " + mime})
        n = len(rx.findall(rest))
        if n:
            chrome[t] = n
    return {"ok": not hits, "hits": hits[:20], "chrome": chrome, "media_files": len(media), "board_json": bool(m)}


# ---------------------------------------------------------------- board

def build(run: str, task_id: str, focal: str, seed: int, rerecord: bool = False, skip=()) -> Dict:
    home = common.bench_home()
    task = common.load_tasks([task_id])[0]
    root = home / "human" / run
    root.mkdir(parents=True, exist_ok=True)
    cands = {}
    for rd in auto_metrics.run_dirs(run, task["id"], None):
        auto = common.read_json(rd / "score" / "auto.json") or {}
        meta = common.read_json(rd / "meta.json") or {}
        if auto.get("produced") and rd.name not in skip:
            cands[rd.name] = {"auto": auto, "path": Path(meta["workspace"]) / meta["deliverable"]["primary"]}
    if len(cands) < 2:
        return {"task": task["id"], "skipped": "fewer than 2 deliverables"}
    rng = random.Random("%s/%s/%d" % (run, task["id"], seed))
    arms = sorted(cands)
    letters = list(string.ascii_uppercase[:len(arms)])
    rng.shuffle(letters)
    key = dict(zip(letters, arms))
    notes = {}  # letter -> why a candidate has no media (logged; kept off the board)

    # 1. a playable source per candidate: the MP4 itself, or a screen recording of the HTML page
    recorded = False
    rec_dir = root / "recordings" / task["id"]
    for letter in sorted(key):
        c = cands[key[letter]]
        c["src"] = c["path"]
        if c["path"].suffix.lower() in common.HTML_EXT:
            recorded = True
            rep = record_html(c["path"], rec_dir / (key[letter] + ".mp4"), rerecord)
            c["rec"] = rep
            if rep.get("ok"):
                c["src"] = rec_dir / (key[letter] + ".mp4")
                log("%s %s: recorded %.1f s (%s, length from %s, played via %s, audio %s)" % (
                    task["id"], letter, rep.get("duration", 0), rep.get("backend"), rep.get("duration_source"),
                    rep.get("played_via"), "captured" if (rep.get("audio") or {}).get("captured") else "none"))
            else:
                c["src"] = None
                notes[letter] = "could not be recorded: " + str(rep.get("why") or "unknown")[:300]
                log("%s %s: %s" % (task["id"], letter, notes[letter]))
        if c["src"] is not None:
            pr = common.probe(c["src"])
            if not pr.get("ok") or not pr.get("video") or not (pr.get("duration") or 0) > 0:
                notes[letter] = "not playable: " + str(pr.get("error") or "no video stream")[:300]
                log("%s %s: %s" % (task["id"], letter, notes[letter]))
                c["src"] = None
            c["probe"] = pr
    # page audio: kept only if it could be captured for every recorded page that had some; else none for all
    strip_audio = any((c.get("rec") or {}).get("ok") and (c["rec"].get("audio") or {}).get("sources")
                      and not c["rec"]["audio"].get("captured") for c in cands.values())
    if strip_audio:
        log("%s: page audio could not be captured for every page: all recordings are shown without sound" % task["id"])

    job = "vote-" + task["id"].split("-")[0]
    r = st(["studio", "init", job, "--title", "Blind vote", "--brief", task["prompt"], "--json"], root, root)
    if r.returncode != 0:
        raise RuntimeError("studio init failed: %s" % r.stderr[-600:])
    info = json.loads(r.stdout)
    jdir = Path(info.get("job_dir") or Path(info["studio_dir"]).parent)
    if not jdir.is_absolute() or root not in jdir.parents:
        raise RuntimeError("studio job landed outside %s: %s" % (root, jdir))
    sdir = jdir / "studio"
    for sub in ("animatic", "thumbs", "frames"):
        shutil.rmtree(sdir / "media" / sub, ignore_errors=True)

    playable = [l for l in sorted(key) if cands[key[l]]["src"] is not None]
    total = sum(cands[key[l]]["probe"]["duration"] for l in playable)
    ref = reference_for(task)  # T10: the style reference, shown for comparison (not a candidate, never tallied)
    if ref:
        total += ref["probe"]["duration"]
    out_html = root / ("%s.html" % task["id"])
    media_budget = (PAGE_MAX_BYTES - SHELL_BYTES) * 3 / 4  # base64 grows 4/3
    tmp = Path(tempfile.mkdtemp(prefix="board-"))
    try:
        alloc, grown = media_budget, False  # alloc: what the settings plan for; media_budget: the hard limit
        for attempt in range(6):
            s = settings_for(total, alloc)
            sizes = {}
            for letter in playable:
                c = cands[key[letter]]
                has_audio = bool(c["probe"].get("audio")) and not (strip_audio and c.get("rec"))
                c["has_audio"] = has_audio
                dst = sdir / "media" / "animatic" / (letter + ".mp4")
                encode(c["src"], dst, s, has_audio, tmp)
                sizes[letter] = dst.stat().st_size
                poster(c["src"], max(0.0, c["probe"]["duration"] * 0.6), sdir / "media" / "thumbs" / (letter + ".jpg"))
            if ref:
                dst = sdir / "media" / "animatic" / (REF_TAG + ".mp4")
                encode(ref["src"], dst, s, bool(ref["probe"].get("audio")), tmp)
                sizes[REF_TAG] = dst.stat().st_size
                poster(ref["src"], ref["probe"]["duration"] * 0.6, sdir / "media" / "thumbs" / (REF_TAG + ".jpg"))
            enc_total = sum(sizes.values())
            if enc_total > media_budget and attempt < 5:
                log("%s: media %.1f MB over the %.1f MB budget at %s; re-encoding all candidates smaller" % (
                    task["id"], enc_total / 1e6, media_budget / 1e6, s))
                alloc *= media_budget / enc_total * 0.96
                continue
            if not grown and enc_total < 0.85 * media_budget and s["video_kbps"] < 2500 and attempt < 5:
                # simple or silent sources left budget unused: give every candidate the same higher settings
                grown = True
                alloc *= min(1.6, media_budget / max(enc_total, 1) * 0.95)
                continue
            board = board_json(task, jdir, key, cands, playable, notes, s, recorded, strip_audio, focal, arms, rng_seed=(run, seed),
                               ref=ref)
            common.write_json(sdir / "board.json", board)
            r = st(["studio", "board", str(jdir)], root, root)
            if r.returncode != 0:
                raise RuntimeError("studio board rejected the board:\n%s%s" % (r.stdout[-1500:], r.stderr[-800:]))
            warns = [ln for ln in (r.stdout + r.stderr).splitlines() if "warning" in ln]
            r = st(["studio", "export", str(jdir), "--target", "artifact", "-o", str(out_html), "--json"], root, root)
            if r.returncode != 0:
                raise RuntimeError("studio export failed: %s" % r.stderr[-800:])
            exp = json.loads(r.stdout or "{}")
            size = out_html.stat().st_size
            if (exp.get("skipped") or size > PAGE_MAX_BYTES) and attempt < 5:
                log("%s: page %.2f MB, skipped %s; re-encoding all candidates smaller" % (task["id"], size / 1e6, exp.get("skipped")))
                shrink = min(0.93, PAGE_MAX_BYTES / max(size, 1) * 0.97)
                media_budget *= shrink
                alloc *= shrink
                grown = True
                continue
            if exp.get("skipped") or size > PAGE_MAX_BYTES:
                raise RuntimeError("board still over budget after 6 tries (%.2f MB, skipped %s)" % (size / 1e6, exp.get("skipped")))
            break
    finally:
        shutil.rmtree(tmp, ignore_errors=True)

    questions = board["questions"]
    keys = common.read_json(root / "key.json", {}) or {}
    keys[task["id"]] = {"letters": key, "questions": [q["id"] for q in questions], "job": str(jdir),  # absolute path
                        "notes": notes, "encode": s, "audio_stripped": strip_audio}
    common.write_json(root / "key.json", keys)
    leak = leak_check(out_html, leak_terms())
    if not leak["ok"]:
        raise RuntimeError("the board names a tool or an output path: %s" % json.dumps(leak["hits"])[:1500])
    return {"task": task["id"], "html": str(out_html), "bytes": size, "candidates": len(key), "reference": bool(ref),
            "playable": len(playable), "no_media": sorted(notes), "encode": s, "audio_stripped": strip_audio,
            "export_skipped": exp.get("skipped"), "warnings": warns, "leak_check": {"ok": leak["ok"], "chrome": leak["chrome"]}}


REF_TAG = "REF"


def reference_for(task: Dict):
    """The task's style reference video (spec "judge_reference"), probed, or None."""
    jr = task.get("judge_reference") or {}
    if not jr:
        return None
    src = (common.BENCH / "fixtures" / "media" / (jr["fetch"] + ".mp4")) if jr.get("fetch") else common.BENCH / jr["from"]
    if not src.exists():
        raise RuntimeError("reference %s missing. FIX: python benchmarks/fixtures/fetch_fixtures.py" % src)
    pr = common.probe(src)
    return {"src": src, "probe": pr} if pr.get("ok") and pr.get("video") else None


def board_json(task: Dict, jdir: Path, key: Dict, cands: Dict, playable: List[str], notes: Dict, s: Dict,
               recorded: bool, strip_audio: bool, focal: str, arms: List[str], rng_seed, ref=None) -> Dict:
    concepts = []
    for letter in sorted(key):
        c = cands[key[letter]]
        entry = {"id": "cand-" + letter, "tag": letter, "title": "Version " + letter}
        if letter not in playable:
            entry.update({"logline": "Could not be recorded or played: this version is shown without media.",
                          "frames": [{"id": "cand-%s-f1" % letter, "src": "media/thumbs/none.svg", "placeholder": True,
                                      "caption": "Could not be recorded"}]})
            concepts.append(entry)
            continue
        pr = c["probe"]
        v = pr.get("video") or {}
        dur = round(pr.get("duration") or 0, 1)
        bits = ["%.1f s" % dur, common.aspect_label(v.get("width") or 0, v.get("height") or 0)]
        if c.get("rec"):
            bits.append("screen-recorded HTML page")
        if not c.get("has_audio"):
            bits.append("no sound" if not (strip_audio and c.get("rec")) else "sound not captured")
        entry.update({"duration": dur, "logline": " · ".join(bits),
                      "frames": [{"id": "cand-%s-f1" % letter, "src": "media/thumbs/%s.jpg" % letter,
                                  "thumb": "media/thumbs/%s.jpg" % letter, "caption": "Frame at 60%"}],
                      "animatic": {"src": "media/animatic/%s.mp4" % letter, "poster": "media/thumbs/%s.jpg" % letter,
                                   "duration": dur}})
        concepts.append(entry)
    if ref:
        pr = ref["probe"]
        v = pr.get("video") or {}
        dur = round(pr.get("duration") or 0, 1)
        concepts.append({"id": "cand-" + REF_TAG, "tag": REF_TAG, "title": "Reference (not a candidate)",
                         "duration": dur, "logline": "The style to match: the video the request asked to imitate. %.1f s · %s. Do not rate it."
                         % (dur, common.aspect_label(v.get("width") or 0, v.get("height") or 0)),
                         "frames": [{"id": "cand-%s-f1" % REF_TAG, "src": "media/thumbs/%s.jpg" % REF_TAG,
                                     "thumb": "media/thumbs/%s.jpg" % REF_TAG, "caption": "Frame at 60%"}],
                         "animatic": {"src": "media/animatic/%s.mp4" % REF_TAG, "poster": "media/thumbs/%s.jpg" % REF_TAG,
                                      "duration": dur}})
    if len(playable) < len(key):
        none_svg = jdir / "studio" / "media" / "thumbs" / "none.svg"
        none_svg.parent.mkdir(parents=True, exist_ok=True)
        none_svg.write_text('<svg xmlns="http://www.w3.org/2000/svg" width="640" height="360" viewBox="0 0 640 360">'
                            '<rect width="640" height="360" fill="#2a2a2a"/><text x="320" y="186" font-family="sans-serif" '
                            'font-size="28" fill="#dddddd" text-anchor="middle">Could not be recorded</text></svg>', encoding="utf-8")
    rng = random.Random("%s/%s/%d/q" % (rng_seed[0], task["id"], rng_seed[1]))
    inv = {a: l for l, a in key.items()}
    pairs = []
    if focal in inv:
        pairs = [tuple(sorted((inv[focal], inv[o]))) for o in arms if o != focal]
    others = [p for p in itertools.combinations(sorted(key), 2) if p not in pairs]
    rng.shuffle(others)
    pairs = (pairs + others)[:5]
    rng.shuffle(pairs)
    questions = [{"id": "q-%s%s" % p, "text": "%s or %s: which would you ship?" % p,
                  "options": [{"id": "q-%s%s-%s" % (p[0], p[1], x), "label": x} for x in (p[0], p[1])] +
                             [{"id": "q-%s%s-tie" % p, "label": "Can't choose"}], "allowText": True} for p in pairs]
    small = ["Small print: every version is re-encoded with the same settings so all fit on one page "
             "(H.264 within %dx%d px, %d kbps video, AAC %d kbps; nothing upscaled)." % (s["box"], s["box"], s["video_kbps"], s["audio_kbps"])]
    if recorded:
        small.append("HTML deliverables were screen-recorded the same way for every version: Chrome at %dx%d, "
                     "play pressed as a viewer would, for the video's own length when the page exposes it (else %d s), "
                     "page sound captured in the page%s." % (REC_W, REC_H, REC_CAP_S,
                                                           "; sound could not be captured for every page, so none is shown" if strip_audio else ""))
    return {"schema": "showtime.studio.board/1", "job": jdir.name, "title": "Blind vote: " + task["title"],
            "brief": task["prompt"], "rev": 1, "phase": "review", "blind": True,
            "round": {"n": 1, "label": "Blind A/B",
                      "prompt": "Play each version, rate each 0-5, then answer the A/B questions. Nothing says which tool made which.",
                      "note": " ".join(small)},
            "history": [{"rev": 1, "note": "%d blind candidates" % len(key) + (" and the reference" if ref else "")}],
            "concepts": concepts, "questions": questions}


DIGEST_PAIR = re.compile(r"^\s*-\s*([A-Z]) or ([A-Z]):.*?->\s*([A-Z]|tie)\s*$", re.I)
DIGEST_RATING = re.compile(r'^\s*-\s*([A-Z])\s+"[^"]*":\s*(\d)/5\s*$')
DIGEST_PICK = re.compile(r'^\s*-\s*concept:\s*([A-Z])\b')


def parse_digest(text: str) -> Dict:
    """Votes from the board's "Copy for your agent" text ("Copy for Claude" before 0.2.0; the digest is the same):
    the only way back from a board opened as a claude.ai artifact, where downloads are blocked.
    Returns {answers: [(x, y, winner)], ratings: {letter: n}, pick}."""
    out = {"answers": [], "ratings": {}, "pick": None}
    section = ""
    for line in text.splitlines():
        head = line.strip().rstrip(":").lower()
        if head in ("picks", "answers", "reactions", "comments") or head.startswith("comments ("):
            section = head.split(" ")[0]
            continue
        m = DIGEST_PAIR.match(line)
        if m and section == "answers":
            out["answers"].append((m.group(1).upper(), m.group(2).upper(), m.group(3)))
            continue
        m = DIGEST_RATING.match(line)
        if m and section == "reactions":
            out["ratings"][m.group(1)] = int(m.group(2))
            continue
        m = DIGEST_PICK.match(line)
        if m and section == "picks":
            out["pick"] = m.group(1)
    return out


def tally(run: str) -> Dict:
    root = common.bench_home() / "human" / run
    keys = common.read_json(root / "key.json", {}) or {}
    out = {"tasks": {}, "pairs": [], "ratings": {}, "picks": {}}
    for task_id, k in keys.items():
        dg = root / "votes" / (task_id + ".txt")
        if dg.exists():  # "Copy for your agent" digest (artifact-hosted boards; older boards said "Copy for Claude")
            d = parse_digest(dg.read_text(encoding="utf-8"))
            letters = k["letters"]
            for x, y, w in d["answers"]:
                winner = "tie" if w.lower() == "tie" else letters.get(w.upper())
                out["pairs"].append({"task": task_id, "a": letters.get(x), "b": letters.get(y), "winner": winner})
            for L, n in d["ratings"].items():
                if letters.get(L):
                    out["ratings"].setdefault(letters[L], []).append(n)
            if d["pick"] and letters.get(d["pick"]):
                out["picks"][task_id] = letters[d["pick"]]
            out["tasks"][task_id] = {"answers": len(d["answers"]), "ratings": len(d["ratings"]), "source": "digest"}
            continue
        r = st(["studio", "feedback", k["job"], "--json"], root, root)
        try:
            state = json.loads(r.stdout).get("state", {})
        except ValueError:
            continue
        letters = k["letters"]
        for qid, ans in (state.get("answers") or {}).items():
            val = (ans or {}).get("value") if isinstance(ans, dict) else ans
            pair = qid[2:]
            a, b = letters.get(pair[0]), letters.get(pair[1])
            pick = str(val or "").rsplit("-", 1)[-1]
            winner = "tie" if pick == "tie" else letters.get(pick)
            out["pairs"].append({"task": task_id, "a": a, "b": b, "winner": winner})
        for target, n in (state.get("ratings") or {}).items():
            arm = letters.get(target.replace("cand-", ""))
            if arm:
                out["ratings"].setdefault(arm, []).append(n)
        out["tasks"][task_id] = {"answers": len(state.get("answers") or {}), "ratings": len(state.get("ratings") or {})}
    common.write_json(common.bench_home() / "runs" / run / "human_summary.json", out)
    return out


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("command", choices=["build", "import", "tally"])
    ap.add_argument("--run", required=True)
    ap.add_argument("--task")
    ap.add_argument("--focal", default="showtime")
    ap.add_argument("--seed", type=int, default=11)
    ap.add_argument("--file")
    ap.add_argument("--rerecord", action="store_true", help="screen-record HTML deliverables again (default: reuse)")
    ap.add_argument("--skip-cells", default="", help="build: comma list of cell folders to leave off the boards (e.g. a second run)")
    a = ap.parse_args()
    if a.command == "build":
        failed = 0
        for t in common.load_tasks([a.task] if a.task else None):
            try:
                print(json.dumps(build(a.run, t["id"], a.focal, a.seed, a.rerecord,
                                       {x for x in a.skip_cells.split(",") if x})), flush=True)
            except Exception as e:  # one task's failure never stops the others
                failed += 1
                log("%s: FAILED: %s" % (t["id"], e))
                traceback.print_exc(file=sys.stderr)
                print(json.dumps({"task": t["id"], "error": str(e)[:1500]}), flush=True)
        return 1 if failed else 0
    elif a.command == "import":
        keys = common.read_json(common.bench_home() / "human" / a.run / "key.json", {}) or {}
        k = keys[common.load_tasks([a.task])[0]["id"]]
        root = common.bench_home() / "human" / a.run
        text = Path(a.file).read_text(encoding="utf-8", errors="replace")
        if text.lstrip().startswith("STUDIO FEEDBACK"):  # the board's "Copy for your agent" text (or "Copy for Claude" from older boards)
            d = parse_digest(text)
            if not d["answers"] and not d["ratings"] and not d["pick"]:
                print("no votes found in %s (expected the 'Copy for your agent' text of a blind board)" % a.file)
                return 1
            (root / "votes").mkdir(parents=True, exist_ok=True)
            (root / "votes" / (k_task := common.load_tasks([a.task])[0]["id"])).with_suffix(".txt").write_text(text, encoding="utf-8")
            print("imported %s: %d answers, %d ratings, pick %s" % (k_task, len(d["answers"]), len(d["ratings"]), d["pick"]))
            return 0
        r = st(["studio", "feedback", k["job"], "--import", str(Path(a.file).resolve())], root, root)
        print(r.stdout or r.stderr)
    else:
        print(json.dumps(tally(a.run), indent=2))
    return 0


if __name__ == "__main__":
    sys.exit(main())