Skip to content

A local video studio for your coding agent. Describe a video in one sentence: your agent directs, your machine renders. Claude Code, Codex, Cursor, Devin and more. No API keys, no uploads.

6.7 KB
Raw Blame
#!/usr/bin/env python3
"""Invented-claims check: a blind judge lists every factual claim a deliverable makes (spoken or on screen)
and labels each against the task's source files.

    python benchmarks/scoring/factcheck.py --run smoke-01 [--task t7] [--arm showtime]

Run auto_metrics.py first (it makes the frames and the transcript). Writes score/factcheck.json:
  claims[]      {text, where: spoken|screen, verdict: supported|unsupported|contradicted|not_a_claim, why}
  counts        per verdict; "invented" = unsupported + contradicted
The packet holds only: the request, the source files' text, the transcript, and the frames, under neutral
names. The judge never sees the arm, the workspace or any file path from the run.
"""
import argparse
import shutil
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "harness"))
import auto_metrics  # noqa: E402
import common  # noqa: E402
import judge  # noqa: E402

SCHEMA = {
    "type": "object",
    "properties": {
        "claims": {"type": "array", "items": {"type": "object", "properties": {
            "text": {"type": "string"}, "where": {"type": "string", "enum": ["spoken", "screen", "both"]},
            "verdict": {"type": "string", "enum": ["supported", "unsupported", "contradicted", "not_a_claim"]},
            "why": {"type": "string"}}, "required": ["text", "where", "verdict", "why"]}},
        "legibility_problems": {"type": "array", "items": {"type": "string"}},
    },
    "required": ["claims", "legibility_problems"],
}

PROMPT = """You are checking a short video for invented claims. You do not know who or what made it.

The person asked for exactly this:
  "{request}"

Everything the video may state as fact must come from the SOURCES folder (or, if there are no sources,
be correct general knowledge; for mathematics, be mathematically correct). Read every file in this folder:
  - SOURCES/*          the only facts the maker was given
  - transcript.txt     what is spoken (automatic transcript; may be empty if there is no voice)
  - frames/*.jpg       stills and a contact sheet of what is on screen, in time order

List every factual claim the video makes, spoken or on screen (numbers, dates, feature names, commands,
comparisons, causes, "first/only/fastest" statements, testimonials, prices, install commands).
Label each:
  supported     stated in or directly computable from SOURCES (or correct maths / common knowledge if no sources)
  unsupported   not in SOURCES and not common knowledge (e.g. an invented speed-up, user count, quote)
  contradicted  conflicts with SOURCES or is wrong
  not_a_claim   slogans, calls to action, opinions ("clean and simple")
Also list on-screen text you could not read at this size (legibility_problems), quoting what you can.
Be strict and literal. Do not reward style. Answer only with the JSON object."""


def packet_for(rdir: Path, task: dict, auto: dict) -> Path:
    pk = common.ws_root() / "judge-packets" / ("fc-" + common.opaque(str(rdir), 10))
    if pk.exists():
        shutil.rmtree(pk)
    (pk / "SOURCES").mkdir(parents=True)
    (pk / "frames").mkdir()
    for f in task.get("facts", []):
        src = common.BENCH / f
        if src.exists():
            shutil.copy2(src, pk / "SOURCES" / src.name)
    (pk / "transcript.txt").write_text(((auto.get("asr") or {}).get("text") or "(no transcript)") + "\n",
                                       encoding="utf-8")
    fr = auto.get("frames") or {}
    imgs = ([fr["sheet"]] if fr.get("sheet") else []) + list(fr.get("key") or [])
    for i, p in enumerate(imgs):
        if p and Path(p).exists():
            shutil.copy2(p, pk / "frames" / ("%02d%s" % (i, Path(p).suffix)))
    return pk


def check(rdir: Path) -> dict:
    auto = common.read_json(rdir / "score" / "auto.json") or {}
    if not auto.get("produced"):
        return {"skipped": "no deliverable"}
    task = common.load_tasks([auto["task"]])[0]
    if task.get("expect", {}).get("footage"):
        return {"skipped": "footage edit: claims are the speaker's own"}
    if (auto.get("asr") or {}).get("text") is None and task.get("expect", {}).get("audio"):
        # make sure spoken claims are visible to the judge even when the task did not need ASR scoring
        v = Path(common.read_json(rdir / "meta.json")["workspace"]) / common.read_json(rdir / "meta.json")["deliverable"]["primary"]
        if v.suffix.lower() in common.VIDEO_EXT and (auto.get("probe") or {}).get("audio"):
            asr = auto_metrics.transcribe(rdir / "score" / "qa-input" / ("deliverable" + v.suffix.lower()), rdir / "score" / "asr")
            auto["asr"] = {"text": (asr or {}).get("text", "")}
    # the frames matter here too (on-screen claims): same proof as the ranking judge, asked again if it fails
    res, attempts = judge.ask_verified(lambda: packet_for(rdir, task, auto),
                                       lambda pk: PROMPT.format(request=task["prompt"]) + "\n\n" + judge.frames_prompt(pk),
                                       SCHEMA, ["frames"], attempts=3, seed=str(rdir))
    out = {"ok": res.get("ok"), "error": res.get("error"),
           "cost_usd": round(sum(z.get("cost_usd") or 0 for z in attempts), 4),
           "proof": [{k: v for k, v in z.items() if k != "frame_codes"} for z in attempts]}
    if res.get("ok"):
        claims = res["data"].get("claims", [])
        counts = {}
        for c in claims:
            counts[c.get("verdict")] = counts.get(c.get("verdict"), 0) + 1
        out.update({"claims": claims, "counts": counts,
                    "invented": counts.get("unsupported", 0) + counts.get("contradicted", 0),
                    "legibility_problems": res["data"].get("legibility_problems", [])})
    common.write_json(rdir / "score" / "factcheck.json", out)
    return out


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--run", required=True)
    ap.add_argument("--task")
    ap.add_argument("--arm")
    ap.add_argument("--skip-done", action="store_true",
                    help="keep a cell's fact check when it already has a verified one (a cell reused from an earlier round)")
    a = ap.parse_args()
    for rd in auto_metrics.run_dirs(a.run, a.task, a.arm):
        old = common.read_json(rd / "score" / "factcheck.json") or {}
        if a.skip_done and old.get("ok") and old.get("claims") is not None:
            print("%-18s %-9s invented=%s (kept: already checked)" % (rd.parent.name, rd.name, old.get("invented")))
            continue
        r = check(rd)
        print("%-18s %-9s invented=%s %s" % (rd.parent.name, rd.name, r.get("invented"), r.get("skipped") or r.get("error") or ""))
    return 0


if __name__ == "__main__":
    sys.exit(main())