A local video studio for your coding agent. Describe a video in one sentence: your agent directs, your machine renders. Claude Code, Codex, Cursor, Devin and more. No API keys, no uploads.
Python agent-skillsai-agentscaptionsclaude-codeclaude-code-pluginclaude-skillscodexcursorexplainer-videoffmpegheadless-chromekokoro-ttslocal-firstmanimmcp-servermotion-graphicstext-to-speechvideovideo-editingvideo-generation
#!/usr/bin/env python3
"""Per-cell stream metrics and the release gates for a round described by a manifest (benchmarks/rounds/*.json).
python benchmarks/scoring/round_report.py --manifest benchmarks/rounds/r4.json [--json]
Reads what the other steps wrote (meta.json, stream.jsonl, score/auto.json, rank_summary.json,
human_summary.json) and writes, next to the round's report.md:
stream_metrics.json per cell: cost, tokens, model calls, images read into the main context (and by
sub-agents), largest main context, tool text, full renders, receipt, foreign paths
gates.json / gates.md every gate of the manifest as PASS / FAIL / PENDING with the numbers behind it
and appends both tables to report.md (between markers, so running it again replaces them).
Full renders per job come from the job's receipt.json when showtime wrote one (renders.full), else from
job.json, else they are estimated from the stream (render commands without --from/--to), and the table
says which. "Foreign paths" lists showtime folders a run touched outside its own arm (a contaminated run,
as two round-3 cells were): a plugin cell may use only its snapshot and its own runtime home; a cell of any
other arm may use none.
"""
import argparse
import json
import re
import statistics
import sys
from pathlib import Path
from typing import Dict, List, Optional
sys.path.insert(0, str(Path(__file__).resolve().parent))
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "harness"))
import common # noqa: E402
import stream_cost # noqa: E402
BEGIN, END = "<!-- round_report: begin -->", "<!-- round_report: end -->"
RENDER_RE = re.compile(r"""(?:^|[\s;&|(`])(?:"?\$\{?\w+\}?"?|\S*showtime(?:\.cmd)?)\s+(?:edit\s+)?render\b([^;&|\n]*)""")
PARTIAL_RE = re.compile(r"--(?:from|to|span)\b")
SHOWTIME_PATH_RE = re.compile(r"(/[^\s'\"`;|&]*?(?:/skills/showtime|/\.showtime))(?=[/\s'\"`;|&]|$)([^\s'\"`;|&]*)")
# ---------------------------------------------------------------- per cell
def tool_calls(stream: Path) -> List[Dict]:
"""Main-agent tool calls (name, input) in order."""
out = []
for line in stream.read_text(encoding="utf-8", errors="replace").splitlines():
try:
m = json.loads(line)
except ValueError:
continue
if m.get("type") != "assistant" or m.get("parent_tool_use_id"):
continue
for b in (m.get("message") or {}).get("content") or []:
if isinstance(b, dict) and b.get("type") == "tool_use":
out.append({"name": b.get("name", ""), "input": b.get("input") or {}})
return out
def renders_from_stream(calls: List[Dict]) -> Dict:
"""Render commands the agent ran: full vs partial (--from/--to/--span). An estimate: renders started by a
script the agent wrote are not seen."""
full = partial = 0
for c in calls:
if c["name"] == "Bash":
for m in RENDER_RE.finditer(str(c["input"].get("command") or "")):
if PARTIAL_RE.search(m.group(1)):
partial += 1
else:
full += 1
elif c["name"].endswith("__render"):
if any(k in c["input"] for k in ("from", "to", "span")):
partial += 1
else:
full += 1
return {"full": full, "partial": partial, "source": "stream (estimate)"}
def job_dirs(ws: Path) -> List[Path]:
root = ws / "showtime-out"
return sorted((d for d in root.iterdir() if d.is_dir()), key=lambda d: d.stat().st_mtime) if root.is_dir() else []
def receipt_for(ws: Path, primary: Optional[str]) -> Optional[Dict]:
"""The receipt of the job that made the deliverable (else the newest job with one)."""
jobs = job_dirs(ws)
if primary:
owner = [j for j in jobs if (ws / primary).resolve().as_posix().startswith(j.resolve().as_posix() + "/")]
jobs = owner + [j for j in jobs if j not in owner][::-1]
else:
jobs = jobs[::-1]
for j in jobs:
rec = common.read_json(j / "receipt.json")
if rec:
return rec
return None
def renders_from_job(ws: Path, primary: Optional[str]) -> Optional[Dict]:
rec = receipt_for(ws, primary)
if rec and isinstance(rec.get("renders"), dict):
r = rec["renders"]
return {"full": r.get("full"), "partial": r.get("partial"), "preview": r.get("preview"), "source": "receipt.json"}
for j in job_dirs(ws)[::-1]:
data = common.read_json(j / "job.json") or {}
rl = [x for x in data.get("renders") or [] if isinstance(x, dict)]
if rl:
return {"full": sum(1 for x in rl if x.get("kind") == "full"),
"partial": sum(1 for x in rl if x.get("kind") == "partial"),
"preview": sum(1 for x in rl if x.get("kind") == "preview"), "source": "job.json"}
return None
def _strings(v) -> List[str]:
"""Every string in a tool input, decoded (a JSON dump would glue an escaped newline onto the next path)."""
if isinstance(v, str):
return [v]
if isinstance(v, dict):
return [s for x in v.values() for s in _strings(x)]
if isinstance(v, list):
return [s for x in v for s in _strings(x)]
return []
def foreign_paths(calls: List[Dict], allowed: List[str]) -> List[str]:
"""showtime folders named in the run's commands and file reads that are not the arm's own. A path is the
arm's own when it lies under an allowed root (or is a parent of one, e.g. `ls` of the arm's folder)."""
roots = [a.rstrip("/") for a in allowed]
seen = []
for c in calls:
for text in _strings(c["input"]):
for m in SHOWTIME_PATH_RE.finditer(text):
p, full = m.group(1), (m.group(1) + m.group(2)).rstrip("/")
if any(full == a or full.startswith(a + "/") or a.startswith(full + "/") for a in roots):
continue
if p not in seen:
seen.append(p)
return seen
def arm_roots(arm_id: str) -> List[str]:
"""Where a plugin cell may legitimately touch showtime: its snapshot and its runtime-home overlay."""
import arms as armlib
d = armlib.arm_dir(arm_id)
man = common.read_json(d / "manifest.json") or {}
if man.get("kind") != "plugin":
return []
out = [str(d / "home" / ".showtime")]
if man.get("plugin_dir"):
out.append(str(Path(man["plugin_dir"])))
# the overlay links the operator's tools, models and caches in by design; only bin/ and the skill path are the
# arm's own, so reaching those in the shared home is still foreign (the round 3 contamination)
shared = common.showtime_home()
if shared.is_dir():
out += [str(e) for e in sorted(shared.iterdir()) if e.name not in armlib.OWN_ENTRIES]
return out
def cell_metrics(rdir: Path) -> Dict:
meta = common.read_json(rdir / "meta.json") or {}
stream = rdir / "stream.jsonl"
row = {"task": rdir.parent.name, "cell": rdir.name, "arm": meta.get("arm"),
"reused": bool(meta.get("reused")), "cost_usd": meta.get("cost_usd"), "wall_s": meta.get("wall_s"),
"turns": meta.get("num_turns"), "tokens": meta.get("tokens") or {}}
if not stream.exists():
return dict(row, error="no stream.jsonl")
sc = stream_cost.analyse(stream, top=3)
tok = row["tokens"]
row.update({"calls_main": sc["calls_main"], "images_main": sc["images_main"], "images_sub": sc["images_sub"],
"ctx_max": sc["ctx_max"], "ctx_median": sc["ctx_median"], "tool_text_chars": sc["tool_output_chars"],
"tokens_in": (tok.get("input_tokens") or 0) + (tok.get("cache_read_input_tokens") or 0)
+ (tok.get("cache_creation_input_tokens") or 0),
"tokens_out": tok.get("output_tokens") or 0,
"image_sources": sc["image_sources"]})
if row["cost_usd"] is None:
row["cost_usd"] = sc["cost_usd"]
calls = tool_calls(stream)
ws = Path(meta.get("workspace") or "/nonexistent")
prim = (meta.get("deliverable") or {}).get("primary")
rj = renders_from_job(ws, prim) if ws.exists() else None
row["renders"] = rj or renders_from_stream(calls)
rec = receipt_for(ws, prim) if ws.exists() else None
row["receipt"] = bool(rec)
row["receipt_cost"] = None
if rec:
cost = (rec.get("usage") or {}).get("cost_usd") # receipt.json schema 1 (references/receipt.md)
row["receipt_cost"] = cost if isinstance(cost, (int, float)) else None
allowed = arm_roots(meta.get("arm") or "") if not meta.get("reused") else []
row["foreign_paths"] = foreign_paths(calls, allowed) if not meta.get("reused") else []
return row
# ---------------------------------------------------------------- gates
def med(xs: List) -> Optional[float]:
xs = [x for x in xs if isinstance(x, (int, float))]
return round(statistics.median(xs), 2) if xs else None
def gate(gid: str, name: str, status: str, detail: str, **numbers) -> Dict:
return dict({"id": gid, "name": name, "status": status, "detail": detail}, **numbers)
def vote_between(human: Dict, task: str, a: str, b: str) -> Optional[str]:
"""The blind vote between two cells on a task: "a", "b", "tie" or None (no answer, no ratings)."""
for p in human.get("pairs") or []:
if p.get("task") == task and {p.get("a"), p.get("b")} == {a, b}:
w = p.get("winner")
return "tie" if w == "tie" else ("a" if w == a else "b" if w == b else None)
ra = (human.get("ratings_by_task") or {}).get(task, {}).get(a)
rb = (human.get("ratings_by_task") or {}).get(task, {}).get(b)
if isinstance(ra, (int, float)) and isinstance(rb, (int, float)):
return "a" if ra > rb else "b" if rb > ra else "tie"
return None
def ratings_by_task(root: Path) -> Dict[str, Dict[str, float]]:
"""{task: {cell: rating}} from the imported vote digests (tally keeps ratings pooled per arm only)."""
import human_board
keys = common.read_json(common.bench_home() / "human" / root.name / "key.json", {}) or {}
out = {}
for task, k in keys.items():
dg = common.bench_home() / "human" / root.name / "votes" / (task + ".txt")
if not dg.exists():
continue
d = human_board.parse_digest(dg.read_text(encoding="utf-8"))
out[task] = {k["letters"][L]: n for L, n in d["ratings"].items() if L in k["letters"]}
return out
def gates(m: Dict, rows: List[Dict], root: Path) -> List[Dict]:
g = m.get("gates") or {}
focal, prev, base = m.get("focal", "showtime"), m.get("previous", ""), m.get("baseline", "baseline")
rank = (common.read_json(root / "rank_summary.json", {}) or {}).get("per_task") or {}
human = common.read_json(root / "human_summary.json", {}) or {}
human["ratings_by_task"] = ratings_by_task(root)
auto = {(r["task"], r["cell"]): (common.read_json(root / r["task"] / r["cell"] / "score" / "auto.json") or {}) for r in rows}
focal_rows = [r for r in rows if r["cell"] == focal]
focal_runs = [r for r in rows if r["arm"] == focal and not r["reused"]] # every new run of the arm (reps too)
out = []
# 1. cost per video vs the baseline on the same tasks
tasks = sorted({r["task"] for r in focal_rows} & {r["task"] for r in rows if r["cell"] == base})
fc = med([r["cost_usd"] for r in focal_rows if r["task"] in tasks])
bc = med([r["cost_usd"] for r in rows if r["cell"] == base and r["task"] in tasks])
out.append(gate("cost", "Median cost per video <= plain Opus on the same tasks",
"PENDING" if fc is None or bc is None else ("PASS" if fc <= bc else "FAIL"),
"%s $%s vs %s $%s on %d tasks (API-equivalent, from the stream)" % (focal, fc, base, bc, len(tasks)),
focal_median=fc, baseline_median=bc))
# 2. images read into the main context
im = med([r.get("images_main") for r in focal_rows])
cap = g.get("images_main_median_max", 12)
out.append(gate("images", "Images read into the main context: median <= %d per job" % cap,
"PENDING" if im is None else ("PASS" if im <= cap else "FAIL"),
"median %s (per task: %s)" % (im, ", ".join("%s %s" % (r["task"].split("-")[0], r.get("images_main")) for r in focal_rows)),
median=im))
# 3. full renders per job
rn = med([(r.get("renders") or {}).get("full") for r in focal_rows])
rcap = g.get("full_renders_median_max", 1)
srcs = sorted({(r.get("renders") or {}).get("source") for r in focal_rows if r.get("renders")})
out.append(gate("renders", "Full renders per job: median %s" % rcap,
"PENDING" if rn is None else ("PASS" if rn <= rcap else "FAIL"),
"median %s (per task: %s; from %s)" % (rn, ", ".join("%s %s" % (r["task"].split("-")[0], (r.get("renders") or {}).get("full"))
for r in focal_rows), " / ".join(srcs) or "-"),
median=rn))
# 4. quality not worse than the previous version: judge AND vote, per task
qt = g.get("quality_tasks") or []
need = g.get("quality_min_tasks", max(0, len(qt) - 1))
per, passed, pending = [], 0, 0
for t in qt:
tid = common.load_tasks([t])[0]["id"]
jr = rank.get(tid) or {}
jf, jp = (jr.get(focal) or {}).get("mean_rank"), (jr.get(prev) or {}).get("mean_rank")
judge_ok = None if jf is None or jp is None else jf <= jp
if jf is None and jp is not None and any(r["task"] == tid and r["cell"] == focal for r in rows):
judge_ok = False # the new run delivered nothing the judges could rank
v = vote_between(human, tid, focal, prev)
vote_ok = None if v is None else v in ("a", "tie")
ok = False if (judge_ok is False or vote_ok is False) else (None if judge_ok is None or vote_ok is None else True)
passed += ok is True
pending += ok is None
per.append("%s judge %s vs %s%s, vote %s" % (t, jf, jp, "" if judge_ok is None else (" ok" if judge_ok else " worse"),
{"a": "won", "b": "lost", "tie": "tie", None: "pending"}[v]))
status = "PASS" if passed >= need else ("PENDING" if passed + pending >= need else "FAIL")
out.append(gate("quality", "Quality: judge and blind vote >= %s on at least %d of %d tasks" % (prev, need, len(qt)), status,
"%d of %d pass, %d pending: %s" % (passed, len(qt), pending, "; ".join(per)), passed=passed, pending=pending))
# 4b. qa failures on every new run of the focal arm
fails = [(r["task"], r["cell"], ((auto[(r["task"], r["cell"])].get("qa") or {}).get("summary") or {}).get("fail"))
for r in focal_runs]
known = [f for f in fails if isinstance(f[2], int)]
bad = [f for f in known if f[2] > g.get("qa_fail_max", 0)]
out.append(gate("qa", "qa failures: %d" % g.get("qa_fail_max", 0),
"PENDING" if not known else ("FAIL" if bad else "PASS"),
("failures in " + ", ".join("%s/%s %d" % f for f in bad)) if bad else
"%d runs checked, none failed%s" % (len(known), "" if len(known) == len(fails) else " (%d without a qa result)" % (len(fails) - len(known)))))
# 4c. the launch task is not last (judge and vote), among every cell but the focal arm's other new runs
lt = g.get("launch_task")
if lt:
tid = common.load_tasks([lt])[0]["id"]
field_cells = {r["cell"] for r in rows if r["task"] == tid and not (r["arm"] == focal and not r["reused"] and r["cell"] != focal)}
field = {c: v.get("mean_rank") for c, v in (rank.get(tid) or {}).items() if c in field_cells}
judge_last = None
if focal in field and len(field) >= 2:
others = [v for c, v in field.items() if c != focal]
judge_last = field[focal] > max(others)
rt = {c: n for c, n in (human["ratings_by_task"].get(tid) or {}).items() if c in field_cells}
vote_last, vote_note = None, "vote rating %s (field %s)" % (rt.get(focal), json.dumps(rt, sort_keys=True))
if focal in rt and len(rt) >= 2:
vote_last = rt[focal] < min(n for c, n in rt.items() if c != focal)
else: # a board voted as pairs only (no ratings): last = lost every pair it was in
mine = [pr for pr in human.get("pairs") or [] if pr.get("task") == tid and focal in (pr.get("a"), pr.get("b"))
and {pr.get("a"), pr.get("b")} <= field_cells and pr.get("winner") in (pr.get("a"), pr.get("b"))]
if mine:
won = sum(1 for pr in mine if pr["winner"] == focal)
vote_last = won == 0
vote_note = "blind pairs won %d of %d" % (won, len(mine))
st = "FAIL" if (judge_last or vote_last) else ("PENDING" if judge_last is None or vote_last is None else "PASS")
out.append(gate("launch", "Launch task (%s) not last" % lt, st,
"judge mean rank %s (field %s); %s" % (field.get(focal), json.dumps(field, sort_keys=True), vote_note)))
# 5. receipts
have = [r for r in focal_runs if r.get("receipt")]
costed = [r for r in have if r.get("receipt_cost") is not None]
out.append(gate("receipts", "Every job writes a receipt (cost filled where the host reports it)",
"PENDING" if not focal_runs else ("PASS" if len(have) == len(focal_runs) else "FAIL"),
"%d of %d runs have receipt.json; %d with a cost" % (len(have), len(focal_runs), len(costed))))
# isolation
dirty = [r for r in rows if r.get("foreign_paths")]
out.append(gate("isolation", "No run touched another showtime folder (round 3 contamination check)",
"FAIL" if dirty else "PASS",
"; ".join("%s/%s: %s" % (r["task"], r["cell"], ", ".join(r["foreign_paths"][:3])) for r in dirty) or
"%d new runs checked" % sum(1 for r in rows if not r["reused"])))
return out
# ---------------------------------------------------------------- output
def fmt(v) -> str:
if v is None:
return "-"
if isinstance(v, float):
return "%.2f" % v
return str(v)
def metrics_table(rows: List[Dict]) -> str:
head = ["task", "cell", "reused", "$", "wall min", "turns", "calls", "tokens in (k)", "tokens out (k)", "images main",
"images sub", "max ctx (k)", "tool text (k chars)", "full renders", "receipt"]
lines = ["| " + " | ".join(head) + " |", "|" + "---|" * len(head)]
for r in rows:
rn = r.get("renders") or {}
lines.append("| " + " | ".join(fmt(x) for x in [
r["task"], r["cell"], "yes" if r["reused"] else "", r.get("cost_usd"),
round((r.get("wall_s") or 0) / 60.0, 1), r.get("turns"), r.get("calls_main"),
(r.get("tokens_in") or 0) // 1000, (r.get("tokens_out") or 0) // 1000, r.get("images_main"), r.get("images_sub"),
(r.get("ctx_max") or 0) // 1000, (r.get("tool_text_chars") or 0) // 1000,
"%s%s" % (fmt(rn.get("full")), "" if rn.get("source") in ("receipt.json", "job.json") else "*"),
"yes" if r.get("receipt") else ""]) + " |")
return "\n".join(lines)
def spread_table(rows: List[Dict], focal: str) -> str:
"""The focal arm's repeated runs side by side (cost, images, renders, context)."""
reps = [r for r in rows if r["arm"] == focal and not r["reused"]]
tasks = sorted({r["task"] for r in reps if r["cell"] != focal})
if not tasks:
return ""
lines = ["| task | cell | $ | images main | full renders | max ctx (k) | wall min |", "|---|---|---|---|---|---|---|"]
for t in tasks:
for r in sorted((x for x in reps if x["task"] == t), key=lambda x: x["cell"]):
lines.append("| %s | %s | %s | %s | %s | %s | %s |" % (t, r["cell"], fmt(r.get("cost_usd")), fmt(r.get("images_main")),
fmt((r.get("renders") or {}).get("full")), (r.get("ctx_max") or 0) // 1000,
round((r.get("wall_s") or 0) / 60.0, 1)))
return "\n".join(lines)
def gates_md(gs: List[Dict]) -> str:
return "\n".join("- **%s** %s: %s" % (x["status"], x["name"], x["detail"]) for x in gs)
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--manifest", required=True)
ap.add_argument("--json", action="store_true")
a = ap.parse_args()
m = json.loads(Path(a.manifest).read_text(encoding="utf-8"))
root = common.bench_home() / "runs" / m["run"]
rows = [cell_metrics(mj.parent) for mj in sorted(root.glob("*/*/meta.json"))]
common.write_json(root / "stream_metrics.json", rows)
gs = gates(m, rows, root)
common.write_json(root / "gates.json", gs)
md = ("## Release gates\n\n" + gates_md(gs) + "\n\n## Process metrics from the stream (every cell)\n\n" + metrics_table(rows)
+ "\n\n`*` = full renders estimated from the stream's commands (no receipt.json or job.json record). "
"Images main / sub: images that entered the main agent's context / a sub-agent's. Max ctx: the largest "
"input (fresh + cached) of one main-agent model call. Tool text: characters of non-image tool output the "
"main agent read. Reused cells keep the numbers of the round they were made in.\n")
sp = spread_table(rows, m.get("focal", "showtime"))
if sp:
md += "\n## Spread: repeated runs of %s\n\n%s\n" % (m.get("focal", "showtime"), sp)
(root / "gates.md").write_text(md, encoding="utf-8")
rep = root / "report.md"
if rep.exists():
text = rep.read_text(encoding="utf-8")
text = re.sub(re.escape(BEGIN) + ".*?" + re.escape(END) + "\n?", "", text, flags=re.S)
rep.write_text(text.rstrip() + "\n\n" + BEGIN + "\n" + md + END + "\n", encoding="utf-8")
if a.json:
print(json.dumps({"gates": gs, "cells": rows}, indent=1))
else:
print(gates_md(gs))
print("\nwrote %s" % (root / "gates.md"))
return 0
if __name__ == "__main__":
sys.exit(main())