Skip to content

A local video studio for your coding agent. Describe a video in one sentence: your agent directs, your machine renders. Claude Code, Codex, Cursor, Devin and more. No API keys, no uploads.

8.1 KB
Raw Blame
#!/usr/bin/env python3
"""Aggregate one benchmark round into results.json and a Markdown report (from report/TEMPLATE.md).

    python benchmarks/scoring/aggregate.py --run r1 [--out report.md]

Headline numbers, per arm (all tasks pooled, then per task):
  delivered        share of tasks with a playable deliverable of the right kind
  spec             mean share of the task's format checks met (duration, aspect, audio, voice, captions...)
  qa_fail / warn   mean `showtime qa` FAIL / WARN findings per deliverable
  invented         mean invented claims (unsupported + contradicted) per deliverable
  judge_win        blind pairwise win rate (ties = 0.5) and Bradley-Terry log strength
  judge_rank       blind ranking judge (rank.py): mean rank score, 1 = ranked first, 0 = ranked last
  human_win        blind human A/B win rate (if votes were imported)
  ttfo / wall      median time to first output / wall time, seconds
  cost / turns     median USD and turns; questions asked in total
Nothing is dropped: failed and capped runs count as "not delivered" and appear in the per-run table.
"""
import argparse
import json
import statistics
import string
import sys
from pathlib import Path
from typing import Dict, List

sys.path.insert(0, str(Path(__file__).resolve().parent))
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "harness"))
import auto_metrics  # noqa: E402
import common  # noqa: E402


def med(xs: List) -> object:
    xs = [x for x in xs if isinstance(x, (int, float))]
    return round(statistics.median(xs), 1) if xs else None


def mean(xs: List) -> object:
    xs = [x for x in xs if isinstance(x, (int, float))]
    return round(sum(xs) / len(xs), 3) if xs else None


def collect(run: str) -> Dict:
    rows = []
    for rd in auto_metrics.run_dirs(run, None, None):
        meta = common.read_json(rd / "meta.json") or {}
        auto = common.read_json(rd / "score" / "auto.json") or {}
        fc = common.read_json(rd / "score" / "factcheck.json") or {}
        qa = auto.get("qa") or {}
        rows.append({
            "task": rd.parent.name, "arm": rd.name, "delivered": bool(auto.get("produced")),
            "spec": auto.get("spec_score"), "spec_detail": auto.get("spec"),
            "qa_verdict": qa.get("verdict"), "qa_fail": (qa.get("summary") or {}).get("fail"),
            "qa_warn": (qa.get("summary") or {}).get("warn"), "lufs": (qa.get("loudness") or {}).get("integrated_lufs"),
            "invented": fc.get("invented"), "footage": auto.get("footage"), "html": (auto.get("html") or {}).get("offline"),
            "ttfo_s": meta.get("ttfo_s"), "wall_s": meta.get("wall_s"), "cost_usd": meta.get("cost_usd"),
            "turns": meta.get("num_turns"), "questions": meta.get("questions"), "capped": meta.get("capped"),
            "is_error": meta.get("is_error"), "tokens_out": (meta.get("tokens") or {}).get("output_tokens"),
            "skills_invoked": meta.get("skills_invoked"), "integrity_ok": (meta.get("integrity") or {}).get("operator_state_unchanged"),
        })
    return {"rows": rows}


def per_arm(rows: List[Dict], pw: Dict, human: Dict, rk: Dict = None) -> Dict:
    arms = sorted({r["arm"] for r in rows})
    hp = {}
    for p in (human or {}).get("pairs", []):
        if not p.get("a") or not p.get("b"):
            continue
        s = 0.5 if p["winner"] == "tie" else (1.0 if p["winner"] == p["a"] else 0.0)
        for arm, pts in ((p["a"], s), (p["b"], 1 - s)):
            hp.setdefault(arm, []).append(pts)
    out = {}
    for a in arms:
        rs = [r for r in rows if r["arm"] == a]
        dl = [r for r in rs if r["delivered"]]
        out[a] = {
            "runs": len(rs), "delivered": round(len(dl) / len(rs), 3) if rs else 0,
            "spec": mean([r["spec"] for r in rs]), "qa_fail": mean([r["qa_fail"] for r in dl]),
            "qa_warn": mean([r["qa_warn"] for r in dl]), "invented": mean([r["invented"] for r in dl]),
            "judge_win": (pw.get("win_rate") or {}).get(a), "judge_rank": ((rk or {}).get("rank_score") or {}).get(a),
            "judge_bt": (pw.get("bradley_terry_log_strength") or {}).get(a),
            "human_win": mean(hp.get(a, [])), "human_rating": mean((human or {}).get("ratings", {}).get(a, [])),
            "ttfo_med_s": med([r["ttfo_s"] for r in rs]), "wall_med_s": med([r["wall_s"] for r in rs]),
            "cost_med_usd": med([r["cost_usd"] for r in rs]), "cost_total_usd": round(sum(r["cost_usd"] or 0 for r in rs), 2),
            "turns_med": med([r["turns"] for r in rs]), "questions_total": sum(r["questions"] or 0 for r in rs),
            "capped": sum(1 for r in rs if r["capped"]), "integrity_ok": all(r["integrity_ok"] is not False for r in rs),
        }
    return out


def table(headers: List[str], rows: List[List]) -> str:
    fmt = lambda v: "-" if v is None else ("%.2f" % v if isinstance(v, float) else str(v))
    lines = ["| " + " | ".join(headers) + " |", "|" + "---|" * len(headers)]
    lines += ["| " + " | ".join(fmt(c) for c in r) + " |" for r in rows]
    return "\n".join(lines)


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--run", required=True)
    ap.add_argument("--out", help="report path (default <bench home>/runs/<run>/report.md)")
    a = ap.parse_args()
    root = common.bench_home() / "runs" / a.run
    data = collect(a.run)
    pw = common.read_json(root / "pairwise_summary.json", {}) or {}
    human = common.read_json(root / "human_summary.json", {}) or {}
    rk = common.read_json(root / "rank_summary.json", {}) or {}
    arms = per_arm(data["rows"], pw, human, rk)
    arm_labels = {x["id"]: x["label"] for x in common.load_arms()["arms"]}
    res = {"run": a.run, "created": common.now_iso(), "per_arm": arms, "pairwise": pw, "ranking": rk, "human": human,
           "rows": data["rows"]}
    common.write_json(root / "results.json", res)
    headline = table(["arm", "delivered", "spec", "qa FAIL", "qa WARN", "invented", "judge win", "BT", "judge rank", "human win",
                      "TTFO s", "wall s", "cost $ (med)", "turns", "questions", "capped"],
                     [[arm_labels.get(k, k), v["delivered"], v["spec"], v["qa_fail"], v["qa_warn"], v["invented"],
                       v["judge_win"], v["judge_bt"], v["judge_rank"], v["human_win"], v["ttfo_med_s"], v["wall_med_s"], v["cost_med_usd"],
                       v["turns_med"], v["questions_total"], v["capped"]] for k, v in arms.items()])
    runs = table(["task", "arm", "delivered", "spec", "qa", "LUFS", "invented", "TTFO s", "wall s", "$", "turns", "questions", "capped"],
                 [[r["task"], r["arm"], r["delivered"], r["spec"], r["qa_verdict"], r["lufs"], r["invented"], r["ttfo_s"],
                   r["wall_s"], r["cost_usd"], r["turns"], r["questions"], r["capped"]] for r in data["rows"]])
    pt = pw.get("per_task") or {}
    per_task = table(["task"] + sorted(arms), [[t] + [d.get(x) for x in sorted(arms)] for t, d in sorted(pt.items())])
    rt = rk.get("per_task") or {}
    ranking = table(["task"] + sorted(arms), [[t] + [(d.get(x) or {}).get("mean_rank") for x in sorted(arms)]
                                              for t, d in sorted(rt.items())]) if rt else "(no ranking judge in this round)"
    tmpl = string.Template((common.BENCH / "report" / "TEMPLATE.md").read_text(encoding="utf-8"))
    md = tmpl.safe_substitute(run=a.run, created=res["created"], headline=headline, per_task=per_task, runs=runs,
                              judgments=pw.get("judgments", 0), first_pos=pw.get("first_position_win_rate"),
                              ranking=ranking, rank_judgments=rk.get("judgments", 0),
                              rank_first=rk.get("first_position_top_rate"), rank_expected=rk.get("first_position_expected"),
                              integrity="all runs left the operator's Claude Code settings and skills untouched"
                              if all(v["integrity_ok"] for v in arms.values()) else "INTEGRITY PROBLEM: see results.json")
    out = Path(a.out) if a.out else root / "report.md"
    out.write_text(md, encoding="utf-8")
    print(headline)
    print("\nreport: %s" % out)
    return 0


if __name__ == "__main__":
    sys.exit(main())