Skip to content

Official Codebase of 3DCodeVerse

Python

bench: refs_ab_readout — judge both arms of a reference-photo A/B with one LikenessJudge

gaoypeng1da8df54be99
diff --git a/harness/bench/refs_ab_readout.py b/harness/bench/refs_ab_readout.py
new file mode 100644
index 00000000..b2449b2b
--- /dev/null
+++ b/harness/bench/refs_ab_readout.py
@@ -0,0 +1,107 @@
+"""Read out a reference-photo A/B: judge BOTH arms offline with the same LikenessJudge.
+
+ python bench/refs_ab_readout.py bench/out/refs_v1_graphics --refs bench/refs/tsr_gfx_aurora_ridge \
+ --rubric shader_v2 --model gemini:gemini-3.1-pro-preview --n 2
+
+The in-loop judge differs by arm on purpose (the arm with photos is judged by LikenessJudge,
+the arm without by the plain rubric judge), so loop-time scores cannot be paired. This script
+takes each run's best round (sheet, frames, gates, plan digest, frame metrics — what the loop
+hands the judge), attaches the SAME reference photos to every run's spec, and judges all of
+them with one LikenessJudge. Arms are read from the prompt id (``<name>_ref_<seed>`` /
+``<name>_noref_<seed>``); the readout pairs seeds and prints per-run scores, defects, and the
+paired mean delta. Verdicts are cached under ``<out>/refs_ab_judge/<id>.json``.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import statistics as st
+from pathlib import Path
+from typing import Any
+
+from codeverse.contracts.artifacts import GateReport, RenderSet
+from codeverse.contracts.spec import ReferenceImage, Spec
+from codeverse.judges.base import JudgeInput
+from codeverse.judges.reference import LikenessJudge
+from codeverse.judges.replay_input import plan_digest
+from codeverse.tracks.graphics_steps import frame_stats_text
+from codeverse.workspace import Workspace
+
+SUFFIXES = (".png", ".jpg", ".jpeg", ".webp")
+
+
+def best_round(run: Path) -> dict[str, Any] | None:
+ best: dict[str, Any] | None = None
+ for f in sorted(run.glob("rounds/r*.json")):
+ r = json.loads(f.read_text())
+ j = r.get("judgment") or {}
+ if j.get("overall") is not None and (best is None or j["overall"] > (best.get("judgment") or {}).get("overall", -1)):
+ best = r
+ return best
+
+
+def judge_run(run: Path, refs: list[Path], judge: LikenessJudge, cache: Path) -> dict[str, Any]:
+ cached = cache / f"{run.name}.json"
+ if cached.is_file():
+ return json.loads(cached.read_text())
+ rnd = best_round(run)
+ if rnd is None:
+ return {"id": run.name, "error": "no judged round"}
+ spec = Spec.model_validate(json.loads((run / "spec.json").read_text()))
+ spec = spec.model_copy(update={"references": [ReferenceImage(path=str(p), role="likeness", note=p.stem.replace("_", " ")) for p in refs]})
+ ws = Workspace(run)
+ plan_path = run / "plan.json"
+ last_index = max((int(f.stem[1:]) for f in run.glob("rounds/r*.json")), default=0)
+ extra = ("FRAME METRICS (harness-measured):\n" + frame_stats_text(ws)) if int(rnd.get("index", 0)) == last_index else \
+ "FRAME METRICS: not available for this round; judge motion from the frames."
+ inp = JudgeInput(spec=spec, renders=RenderSet.model_validate(rnd["renders"]),
+ gates=[GateReport.model_validate(g) for g in rnd.get("gates") or []],
+ round_index=int(rnd.get("index", 0)),
+ plan_summary=plan_digest(json.loads(plan_path.read_text())) if plan_path.is_file() else "", extra_context=extra)
+ v = judge.judge(inp)
+ raw = v.raw if isinstance(v.raw, dict) else json.loads(v.raw or "{}")
+ row = {"id": run.name, "round": int(rnd.get("index", 0)), "overall": v.overall, "loop_overall": (rnd.get("judgment") or {}).get("overall"),
+ "uncapped": raw.get("overall_uncapped"), "scores": v.scores,
+ "defects": [d for d, on in (raw.get("defects") or {}).items() if on],
+ "caps": [c.get("rule") for c in (raw.get("caps") or {}).get("caps_applied", [])], "summary": (v.summary or "")[:500]}
+ cache.mkdir(parents=True, exist_ok=True)
+ cached.write_text(json.dumps(row, indent=1))
+ return row
+
+
+def main(argv: list[str] | None = None) -> int:
+ ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+ ap.add_argument("out", help="bench/out/<battery> of the A/B")
+ ap.add_argument("--refs", required=True, help="folder of the reference photos to attach to EVERY run")
+ ap.add_argument("--rubric", default="shader_v2")
+ ap.add_argument("--model", default="gemini:gemini-3.1-pro-preview")
+ ap.add_argument("--n", type=int, default=2)
+ a = ap.parse_args(argv)
+ out = Path(a.out)
+ refs = sorted(p for p in Path(a.refs).iterdir() if p.suffix.lower() in SUFFIXES)
+ judge = LikenessJudge(a.model, n_samples=a.n, rubric=a.rubric)
+ rows = [judge_run(run, refs, judge, out / "refs_ab_judge") for run in sorted((out / "runs").iterdir()) if (run / "spec.json").is_file()]
+ ok = {r["id"]: r for r in rows if "overall" in r}
+ print(f"# refs A/B readout — {len(ok)} runs judged with LikenessJudge({a.rubric}, {a.model}, n={a.n}), photos: {[p.name for p in refs]}\n")
+ print("| run | round | loop score | same-judge score | defects | caps |\n|---|---|---|---|---|---|")
+ for rid, r in sorted(ok.items()):
+ print(f"| {rid} | r{r['round']:02d} | {r['loop_overall']:.3f} | **{r['overall']:.3f}** | {', '.join(r['defects'])} | {', '.join(c for c in r['caps'] if c)} |")
+ seeds: dict[str, dict[str, float]] = {}
+ for rid, r in ok.items():
+ name, arm, seed = rid.rsplit("_", 2)
+ seeds.setdefault(seed, {})[arm] = r["overall"]
+ deltas = [s["ref"] - s["noref"] for s in seeds.values() if "ref" in s and "noref" in s]
+ if deltas:
+ m = st.mean(deltas)
+ sd = st.stdev(deltas) if len(deltas) > 1 else float("nan")
+ print(f"\npaired (ref − noref): n={len(deltas)} mean {m:+.3f} sd {sd:.3f} sign {sum(d > 0 for d in deltas)}/{len(deltas)}"
+ f" (judged-score A/A floor sd 0.202)")
+ for r in rows:
+ if "error" in r:
+ print(f"- {r['id']}: {r['error']}")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())