A local video studio for your coding agent. Describe a video in one sentence: your agent directs, your machine renders. Claude Code, Codex, Cursor, Devin and more. No API keys, no uploads.
Python agent-skillsai-agentscaptionsclaude-codeclaude-code-pluginclaude-skillscodexcursorexplainer-videoffmpegheadless-chromekokoro-ttslocal-firstmanimmcp-servermotion-graphicstext-to-speechvideovideo-editingvideo-generation
#!/usr/bin/env python3
"""Automatic metrics for every run of a benchmark round (same tools, same settings, for every arm).
python benchmarks/scoring/auto_metrics.py --run smoke-01 [--task t7] [--arm showtime] [--no-asr]
Per run folder (runs/<run>/<task>/<arm>/) it writes score/auto.json with:
produced a deliverable of the right kind exists and decodes (full decode, errors counted)
spec duration / aspect / audio / captions / voice as the task asked (each true/false/None)
qa `showtime qa` on a COPY of the video in a neutral folder, with the task's expect block
(loudness, true peak, clipping, silence, black and frozen stretches, frame 0, captions);
the same command and thresholds for every arm, so no arm gets its own project context
asr local word-level transcript of the deliverable (voice tasks and the footage task)
footage (t4) fillers left, content words kept in order, long silences left
captions_sync (sidecar captions) median |cue start - spoken word start|
html (t6) headless-Chrome probe: network requests, single file, plays, phone overflow
frames contact sheet + 6 key frames for the judges (from `showtime snap`)
process wall time, time to first output, tokens, cost, turns, questions (copied from meta.json)
"""
import argparse
import json
import re
import shutil
import sys
from pathlib import Path
from typing import Dict, List, Optional
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "harness"))
import common # noqa: E402
FILLERS = {"um", "uh", "er", "ah", "erm", "hmm", "mm", "umm", "uhh", "uhm"}
def norm_words(words: List[Dict]) -> List[str]:
out = []
for w in words:
t = re.sub(r"[^a-z0-9']", "", (w.get("word") or w.get("text") or "").lower())
if t:
out.append(t)
return out
def lcs_len(a: List[str], b: List[str]) -> int:
prev = [0] * (len(b) + 1)
for x in a:
cur = [0]
for j, y in enumerate(b):
cur.append(prev[j] + 1 if x == y else max(prev[j + 1], cur[j]))
prev = cur
return prev[-1]
def transcribe(video: Path, out_dir: Path, model: Optional[str] = None) -> Optional[Dict]:
"""Local transcription with showtime's default model, or `model` (e.g. turbo) when given."""
out_dir.mkdir(parents=True, exist_ok=True)
cmd = [common.SHOWTIME, "transcribe", str(video), "--edit-dir", str(out_dir), "--json"]
if model:
cmd += ["--model", model]
r = common.run(cmd, cwd=str(out_dir))
js = sorted((out_dir / "transcripts").glob("*.json"))
if r.returncode != 0 or not js:
return {"error": (r.stderr or r.stdout)[-600:]}
d = json.loads(js[0].read_text(encoding="utf-8"))
return {"text": d.get("text", ""), "words": d.get("words", []), "duration": d.get("duration")}
def parse_srt(path: Path) -> List[Dict]:
cues = []
text = path.read_text(encoding="utf-8", errors="replace").replace("\r", "")
for block in re.split(r"\n\s*\n", text):
m = re.search(r"(\d+):(\d+):(\d+)[,.](\d+)\s*-->\s*(\d+):(\d+):(\d+)[,.](\d+)", block)
if not m:
continue
g = [int(x) for x in m.groups()]
start = g[0] * 3600 + g[1] * 60 + g[2] + g[3] / 1000.0
end = g[4] * 3600 + g[5] * 60 + g[6] + g[7] / 1000.0
body = block[m.end():].strip()
cues.append({"start": start, "end": end, "text": re.sub(r"<[^>]+>", "", body)})
return cues
def caption_sync(cues: List[Dict], words: List[Dict]) -> Dict:
"""Median offset between each cue's first word and the same word spoken (searched nearby)."""
offs = []
for c in cues:
first = norm_words([{"word": c["text"].split()[0]}]) if c["text"].split() else []
if not first:
continue
near = [w for w in words if abs(float(w.get("start", 0)) - c["start"]) < 3.0
and norm_words([w]) == first]
if near:
offs.append(min(abs(float(w["start"]) - c["start"]) for w in near))
offs.sort()
return {"cues": len(cues), "matched": len(offs),
"median_offset_s": round(offs[len(offs) // 2], 3) if offs else None,
"p90_offset_s": round(offs[int(len(offs) * 0.9)], 3) if offs else None}
def decode_errors(video: Path) -> int:
r = common.run([common.ffmpeg(), "-v", "error", "-i", str(video), "-f", "null", "-"])
return len([ln for ln in r.stderr.splitlines() if ln.strip()])
def expect_block(task: Dict, pr: Dict) -> Dict:
ex = task.get("expect", {})
e = {}
if ex.get("duration"):
lo, hi = ex["duration"]
e["duration"], e["tolerance"] = (lo + hi) / 2.0, (hi - lo) / 2.0
if ex.get("audio") == "required":
e["audio"] = True
if ex.get("lufs") is not None:
e["lufs"] = ex["lufs"]
if ex.get("platform"):
e["platform"] = ex["platform"]
return e
def score_video(task: Dict, meta: Dict, rdir: Path, video: Path, asr_on: bool) -> Dict:
ex = task.get("expect", {})
sd = rdir / "score"
pr = common.probe(video)
out: Dict = {"probe": pr, "decode_errors": decode_errors(video)}
out["produced"] = bool(pr.get("ok") and pr.get("video") and pr.get("duration", 0) > 0.5)
v = pr.get("video") or {}
aspect = common.aspect_label(v.get("width") or 0, v.get("height") or 0)
dur = pr.get("duration") or 0
has_audio = bool(pr.get("audio"))
spec = {"aspect": aspect in ex.get("aspect", [aspect]), "aspect_is": aspect}
if ex.get("duration"):
spec["duration"] = ex["duration"][0] <= dur <= ex["duration"][1]
if ex.get("audio") in ("required", "preferred"):
spec["audio"] = has_audio
out["spec"] = spec
# qa on a neutral copy so no arm's project/job metadata changes what is checked
qin = sd / "qa-input"
if qin.exists():
shutil.rmtree(qin)
qin.mkdir(parents=True)
copy = qin / ("deliverable" + video.suffix.lower())
shutil.copy2(video, copy)
sidecars = []
for sc in sorted(video.parent.glob("*")):
if sc.suffix.lower() in (".srt", ".vtt") and sc.stem.split(".")[0] == video.stem.split(".")[0]:
dst = qin / ("deliverable" + sc.suffix.lower())
shutil.copy2(sc, dst)
sidecars.append(dst)
if not sidecars: # any caption file the agent wrote in the workspace
ws = Path(meta["workspace"])
found = [p for p in ws.rglob("*") if p.suffix.lower() in (".srt", ".vtt") and "node_modules" not in p.parts]
for sc in found[:1]:
dst = qin / ("deliverable" + sc.suffix.lower())
shutil.copy2(sc, dst)
sidecars.append(dst)
common.write_json(qin / "expect.json", {"expect": expect_block(task, pr)})
cmd = [common.SHOWTIME, "qa", str(copy), "--expect", str(qin / "expect.json"), "--json", "-o", str(sd / "qa")]
if ex.get("platform"):
cmd += ["--platform", ex["platform"]]
for sc in sidecars:
cmd += ["--captions", str(sc)]
r = common.run(cmd, cwd=str(qin))
try:
qa = json.loads(r.stdout)
out["qa"] = {"verdict": qa.get("verdict"), "summary": qa.get("summary"),
"findings": [{"rule": f.get("rule"), "severity": f.get("severity"), "message": f.get("message")}
for f in qa.get("findings", [])],
"loudness": {k: (qa.get("loudness") or {}).get(k) for k in
("integrated_lufs", "true_peak_dbtp", "lra", "silent_gaps")},
"clipping": ((qa.get("loudness") or {}).get("clipping") or {}).get("runs"),
"detect": qa.get("detect"), "sheet": qa.get("sheet")}
except ValueError:
out["qa"] = {"error": (r.stderr or r.stdout)[-800:]}
out["captions_sidecar"] = [p.name for p in sidecars]
# frames for the judges
snap = sd / "frames"
r = common.run([common.SHOWTIME, "snap", str(copy), "--count", "12", "-o", str(snap), "--json"], cwd=str(qin))
ts = [round(dur * f, 2) for f in (0.02, 0.2, 0.4, 0.6, 0.8, 0.98)]
r2 = common.run([common.SHOWTIME, "snap", str(copy), "--at", ",".join(str(t) for t in ts), "--width", "960",
"--format", "jpg", "-o", str(snap / "key"), "--json"], cwd=str(qin))
out["frames"] = {"sheet": next((str(p) for p in snap.glob("*.jpg") if "sheet" in p.name), None),
"key": sorted(str(p) for p in (snap / "key").glob("*.jpg")),
"error": None if r.returncode == 0 and r2.returncode == 0 else (r.stderr + r2.stderr)[-400:]}
# speech
if asr_on and has_audio and (ex.get("voice") or ex.get("footage") or ex.get("captions")):
asr = transcribe(copy, sd / "asr")
out["asr"] = {"words": len((asr or {}).get("words") or []), "text": ((asr or {}).get("text") or "")[:3000],
"error": (asr or {}).get("error")}
if ex.get("voice"):
spec["voice"] = out["asr"]["words"] >= 20
words = (asr or {}).get("words") or []
if ex.get("footage"):
out["footage"] = footage_metrics(task, words, sd)
if sidecars and sidecars[0].suffix == ".srt":
out["captions_sync"] = caption_sync(parse_srt(sidecars[0]), words)
if ex.get("captions") == "required":
spec["captions"] = bool(sidecars) or None # None = burned-in or absent: the judges decide from frames
return out
def lcs_align(a: List[str], b: List[str]) -> Dict[int, int]:
"""Longest-common-subsequence alignment: {index in a: index in b}."""
n, m = len(a), len(b)
t = [[0] * (m + 1) for _ in range(n + 1)]
for i in range(n - 1, -1, -1):
for j in range(m - 1, -1, -1):
t[i][j] = t[i + 1][j + 1] + 1 if a[i] == b[j] else max(t[i + 1][j], t[i][j + 1])
out, i, j = {}, 0, 0
while i < n and j < m:
if a[i] == b[j]:
out[i] = j
i, j = i + 1, j + 1
elif t[i + 1][j] >= t[i][j + 1]:
i += 1
else:
j += 1
return out
def content(words: List[Dict]) -> List[Dict]:
"""Words with a normalised key, fillers dropped."""
out = []
for w in words:
k = norm_words([w])
if k and k[0] not in FILLERS:
out.append({"k": k[0], "start": float(w.get("start", 0)), "end": float(w.get("end", 0))})
return out
def footage_metrics(task: Dict, words: List[Dict], sd: Path, model: Optional[str] = None) -> Dict:
"""Footage edit (t4): known fillers removed, content kept in order, long pauses left.
Transcribed fillers are not trusted (ASR often folds an 'uh' into the next word), so each known filler
is checked by timing: the span of content words around it must shrink by >= 60% of the filler length.
Source and output are transcribed by the same local model, so any folding happens the same way."""
spec = common.read_json(common.BENCH / "fixtures" / "media" / "interview.source.json") or {}
gt = spec.get("ground_truth", {})
src = common.BENCH / "fixtures" / "media" / "interview.mp4"
cache = common.bench_home() / "cache" / ("asr-source-" + model if model else "asr-source")
js = sorted((cache / "transcripts").glob("*.json")) if (cache / "transcripts").exists() else []
ref = json.loads(js[0].read_text(encoding="utf-8")) if js else transcribe(src, cache, model)
rw, ow = content((ref or {}).get("words") or []), content(words)
rk, ok = [w["k"] for w in rw], [w["k"] for w in ow]
align = lcs_align(rk, ok)
spans = []
for fs in gt.get("filler_spans", []):
pat = fs["words"]
idx = next((i for i in range(len(rk) - len(pat) + 1) if rk[i:i + len(pat)] == pat), None)
if idx is None:
spans.append({"words": pat, "result": "not found in source transcript"})
continue
src_span = rw[idx + len(pat) - 1]["end"] - rw[idx]["start"]
mapped = [align.get(idx + k) for k in range(len(pat))]
if any(m is None for m in mapped):
spans.append({"words": pat, "result": "words cut"})
continue
out_span = ow[mapped[-1]]["end"] - ow[mapped[0]]["start"]
removed = (src_span - out_span) >= 0.6 * fs["filler_s"]
spans.append({"words": pat, "src_span": round(src_span, 2), "out_span": round(out_span, 2),
"result": "removed" if removed else "kept"})
kept = len(align)
gaps = [b["start"] - a["end"] for a, b in zip(ow, ow[1:])]
n_fill = sum(2 if "two" in fs.get("note", "") else 1
for fs, r in zip(gt.get("filler_spans", []), spans) if r["result"] == "removed")
return {"fillers_known": gt.get("fillers"), "fillers_removed": n_fill, "filler_spans": spans,
"fillers_transcribed_in_output": sum(1 for w in norm_words(words) if w in FILLERS),
"content_words_source": len(rk), "content_words_kept_in_order": kept,
"content_kept_ratio": round(kept / len(rk), 3) if rk else None,
"extra_words": max(0, len(ok) - kept),
"pauses_over_1s": sum(1 for g in gaps if g > 1.0),
"longest_pause_s": round(max(gaps), 2) if gaps else None}
def score_html(meta: Dict, rdir: Path, page: Path) -> Dict:
sd = rdir / "score" / "html"
r = common.run([common.node_bin(), str(common.BENCH / "scoring" / "html_probe.mjs"), str(page), str(sd)])
try:
rep = json.loads(r.stdout.strip().splitlines()[-1])
except (ValueError, IndexError):
return {"produced": False, "error": (r.stderr or r.stdout)[-800:]}
mb = page.stat().st_size / 1e6
spec = {"single_file": rep.get("single_file"), "offline": rep.get("offline"), "plays": rep.get("animates"),
"size_ok": mb <= 16, "phone_ok": rep.get("phone_overflow") is False}
return {"produced": True, "html": rep, "spec": spec,
"frames": {"key": [str(sd / s) for s in rep.get("shots", [])], "sheet": None}}
def score_run(rdir: Path, asr_on: bool = True) -> Dict:
meta = common.read_json(rdir / "meta.json")
task = common.load_tasks([meta["task"]])[0]
prim = (meta.get("deliverable") or {}).get("primary")
res: Dict = {"task": meta["task"], "arm": meta["arm"], "produced": False}
if prim:
p = Path(meta["workspace"]) / prim
if task.get("deliverable") == "html":
res.update(score_html(meta, rdir, p))
else:
res.update(score_video(task, meta, rdir, p, asr_on))
res["process"] = {k: meta.get(k) for k in ("wall_s", "ttfo_s", "first_tool_s", "cost_usd", "num_turns", "questions",
"auto_replies", "capped", "is_error", "tokens", "subagents",
"skills_invoked")}
spec = res.get("spec") or {}
checks = [v for k, v in spec.items() if isinstance(v, bool)]
res["spec_score"] = round(sum(checks) / len(checks), 3) if checks else 0.0
common.write_json(rdir / "score" / "auto.json", res)
return res
def rescore_footage(rdir: Path, model: str) -> Optional[Dict]:
"""Footage metrics again with another transcription model, kept next to the first as auto.json
"footage_<model>" (rounds 1 and 2 scored the filler cut with Whisper turbo: this makes a later round
comparable with them while the default model, which writes fillers verbatim, does the rest)."""
auto = common.read_json(rdir / "score" / "auto.json") or {}
src = rdir / "score" / "qa-input"
vids = sorted(src.glob("deliverable.*")) if src.exists() else []
task = common.load_tasks([auto.get("task")])[0] if auto.get("task") else None
if not (task and task.get("expect", {}).get("footage") and vids and auto.get("produced")):
return None
sd = rdir / "score"
asr = transcribe(vids[0], sd / ("asr-" + model), model)
words = (asr or {}).get("words") or []
res = footage_metrics(task, words, sd, model)
res["asr_model"] = model
auto["footage_" + model] = res
common.write_json(sd / "auto.json", auto)
return res
def run_dirs(run: str, task: Optional[str], arm: Optional[str]) -> List[Path]:
root = common.bench_home() / "runs" / run
out = []
for m in sorted(root.glob("*/*/meta.json")):
t, a = m.parent.parent.name, m.parent.name
# a task id or its prefix ("t1" is t1-launch, never t10-like-reference)
if (task and not (t == task or t.split("-")[0] == task.split("-")[0] and task in t)) or (arm and a != arm):
continue
out.append(m.parent)
return out
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--run", required=True)
ap.add_argument("--task")
ap.add_argument("--arm")
ap.add_argument("--no-asr", action="store_true", help="skip local transcription (faster)")
ap.add_argument("--footage-model", metavar="MODEL",
help="only re-score the footage task with this transcription model (e.g. turbo, what rounds 1 and 2 used) "
"and keep it as auto.json footage_<MODEL>; nothing else is recomputed")
a = ap.parse_args()
common.wait_for_round(a.run)
if a.footage_model:
for rd in run_dirs(a.run, a.task, a.arm):
r = rescore_footage(rd, a.footage_model)
if r:
print("%-18s %-14s fillers removed %s/%s, words kept %s, longest pause %s (%s)" % (
rd.parent.name, rd.name, r["fillers_removed"], r["fillers_known"], r["content_kept_ratio"],
r["longest_pause_s"], a.footage_model))
return 0
for rd in run_dirs(a.run, a.task, a.arm):
r = score_run(rd, not a.no_asr)
qa = r.get("qa") or {}
print("%-18s %-13s produced=%-5s spec=%.2f qa=%s %s" % (r["task"], rd.name, r["produced"], r["spec_score"],
qa.get("verdict"), qa.get("summary")))
return 0
if __name__ == "__main__":
sys.exit(main())