Official Codebase of 3DCodeVerse
Python
bench: h2h/ab_view/complexity readers load the record once and take round N's GLB from round N (6.5, B4)
gaoypeng567c1f0529a8
diff --git a/eval/bench/ab_view.py b/eval/bench/ab_view.pyindex 22b0060a..d707b254 100644--- a/eval/bench/ab_view.py+++ b/eval/bench/ab_view.py@@ -21,7 +21,6 @@ import argparseimport base64import htmlimport io-import jsonimport statisticsimport sysfrom dataclasses import dataclass, field@@ -77,26 +76,19 @@ def _sheet_data_uri(run_dir: Path) -> str:def load_run(run_dir: Path, arm: str) -> Run:r = Run(slug=run_dir.name, arm=arm)- rec = run_dir / "record.json"- if rec.is_file():- d = json.loads(rec.read_text())- r.brief = str((d.get("spec") or {}).get("prompt") or "")- try: # the round addons/select picks; a record it cannot read keeps only its status- s = select.summarise(run_dir)- r.status, r.picked, r.baseline = s.stop_reason or "?", s.picked_score, s.baseline_score- except Exception: # noqa: BLE001 - one unreadable record must not blank the page- r.status = str(d.get("status") or "?")- r.rounds = len(d.get("rounds") or [])- try: # the ledger's total and the steps' minutes, as every reader reports them- run = load_record(run_dir)- r.usd, r.minutes = round(run.total_usage.cost_usd, 2), round(run.minutes or 0)- except Exception: # noqa: BLE001 - an unreadable record keeps $0 / 0 min- pass- r.error = str(d.get("error") or "")[:300]- for rd in d.get("rounds") or []:- failed = [g.get("gate") for g in (rd.get("gates") or []) if not g.get("passed")]+ try: # read once: the round addons/select picks, the ledger's total, the steps' minutes+ rec = load_record(run_dir)+ s = select.summarise(run_dir, record=rec)+ except Exception: # noqa: BLE001 - a missing or unreadable record must not blank the page+ rec = None+ if rec is not None:+ r.brief, r.rounds, r.error = rec.spec.prompt, len(rec.rounds), (rec.error or "")[:300]+ r.status, r.picked, r.baseline = s.stop_reason or "?", s.picked_score, s.baseline_score+ r.usd, r.minutes = round(rec.total_usage.cost_usd, 2), round(rec.minutes or 0)+ for rd in rec.rounds:+ failed = [g.gate for g in rd.gates if not g.passed]if failed:- r.notes.append(f"r{rd.get('index')} gates: {', '.join(str(g) for g in failed[:4])}")+ r.notes.append(f"r{rd.index} gates: {', '.join(failed[:4])}")r.sheet = _sheet_data_uri(run_dir)return rdiff --git a/eval/bench/complexity_report.py b/eval/bench/complexity_report.pyindex e9f4530b..505f68be 100644--- a/eval/bench/complexity_report.py+++ b/eval/bench/complexity_report.py@@ -34,7 +34,7 @@ for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolfrom bench.stats import correlation # noqa: E402from codeverse3d.addons import select # noqa: E402-from codeverse3d.contracts.run import RunId, RunRecord # noqa: E402+from codeverse3d.contracts.run import RoundRecord, RunId, RunRecord # noqa: E402from codeverse3d.proc import read_json_or_none # noqa: E402from codeverse3d.record.record import find_run_dirs # noqa: E402from codeverse3d.spatial.complexity import ( # noqa: E402@@ -42,6 +42,7 @@ from codeverse3d.spatial.complexity import ( # noqa: E402ComplexityVector,band_of,)+from codeverse3d.workspace import Workspace # noqa: E402CRITERIA = ("intent_fidelity",@@ -70,22 +71,23 @@ _read_json = read_json_or_nonedef _vector_for(run: Path, rnd: dict[str, Any] | None) -> ComplexityVector | None:- """The run's recorded complexity vector, else one recomputed from its GLB.+ """The round's recorded complexity vector, else one recomputed from the round's own GLB.The historic corpus predates the vector, so recomputing is the normal pathfor anything recorded before 2026-08-24 — that is what makes the whole- corpus comparable.+ corpus comparable. The round is named by its index alone (a historic round may not+ validate); ``addons.select`` answers for its own files, never the last build's."""- for meas in (rnd or {}).get("measurement"), (_read_json(run / "artifacts" / "measurement.json") or {}):- block = ((meas or {}).get("extra") or {}).get("complexity")+ ws, one = Workspace(run), RoundRecord(index=int((rnd or {}).get("index") or 0), kind="")+ own = (((rnd or {}).get("measurement") or {}).get("extra") or {}).get("complexity")+ for block in (own, select.round_complexity_block(ws, one)):if isinstance(block, dict):try:return ComplexityVector.model_validate(block)except Exception: # noqa: BLE001 - an older/foreign block must not stop the scanpass- kept = run / "artifacts" / f"r{int((rnd or {}).get('index') or 0):02d}" / "object.glb"- glb = kept if kept.is_file() else run / "artifacts" / "object.glb"- if not glb.is_file():+ glb = select.round_file(ws, one)+ if glb is None:return Nonefrom codeverse3d.spatial.complexity import complexity_of_glb@@ -95,31 +97,26 @@ def _vector_for(run: Path, rnd: dict[str, Any] | None) -> ComplexityVector | Nonreturn None-def _picked_round(run: Path, record: dict[str, Any]) -> dict[str, Any] | None:+def _picked_round(run: Path, record: dict[str, Any], rec: RunRecord | None) -> dict[str, Any] | None:"""The round ``addons/select`` picks, as its raw dict; the last round when the recordwill not load (the scan reads the historic corpus as-is)."""rounds = record.get("rounds") or []try:- idx = select.summarise(run).picked_round+ idx = select.summarise(run, record=rec).picked_round if rec is not None else Noneexcept Exception: # noqa: BLE001 - one unreadable record must not stop the scanidx = Nonereturn next((r for r in rounds if r.get("index") == idx), rounds[-1] if rounds else None)-def _minutes(record: dict[str, Any]) -> float | None:- """``RunRecord.minutes`` (docs/COST.md §31); None for a record that will not load."""- try:- minutes = RunRecord.model_validate(record).minutes- except Exception: # noqa: BLE001 - the historic corpus is read as-is- return None- return None if minutes is None else round(minutes, 2)--def row_for(run: Path, battery: str, slug: str | None = None) -> Row | None:record = _read_json(run / "record.json")if not record:return None- rnd = _picked_round(run, record)+ try: # validated once; the historic corpus is read as a raw dict where it will not load+ rec: RunRecord | None = RunRecord.model_validate(record)+ except Exception: # noqa: BLE001+ rec = None+ rnd = _picked_round(run, record, rec)judgment = (rnd or {}).get("judgment") or {}scores = judgment.get("scores") or {}vec = _vector_for(run, rnd)@@ -141,7 +138,7 @@ def row_for(run: Path, battery: str, slug: str | None = None) -> Row | None:overall=judgment.get("overall"),issues=len(judgment.get("issues") or []),cost_usd=round(float((record.get("total_usage") or {}).get("cost_usd") or 0.0), 4),- minutes=_minutes(record),+ minutes=round(rec.minutes, 2) if rec is not None and rec.minutes is not None else None,index=vec.index,band=vec.band,)diff --git a/eval/bench/h2h_glb.py b/eval/bench/h2h_glb.pyindex 68b46cca..aef7c2c6 100644--- a/eval/bench/h2h_glb.py+++ b/eval/bench/h2h_glb.py@@ -45,9 +45,11 @@ for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolsys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)from bench._fixed_eval import RUBRIC # noqa: E402-from bench.run_bench import Battery, BenchPrompt, record_minutes # noqa: E402+from bench.run_bench import Battery, BenchPrompt # noqa: E402from bench.stats import sign_test # noqa: E402from codeverse3d.addons import select # noqa: E402+from codeverse3d.record.record import load_record # noqa: E402+from codeverse3d.workspace import Workspace # noqa: E402JUDGE_MODEL = "gemini:gemini-3.1-pro-preview"N_SAMPLES = 2@@ -108,31 +110,24 @@ def their_side(gallery: Path, slug: str) -> Side:def our_side(runs_dir: Path, item_id: str) -> Side:ws = runs_dir / item_id- rec_path = ws / "record.json"- if not rec_path.is_file():+ if not (ws / "record.json").is_file():return Side(source="3dcodeverse", status="not_run", error=f"no record.json under {ws}")- rec: dict[str, Any] = json.loads(rec_path.read_text())- status = str(rec.get("status", ""))- side = Side(source="3dcodeverse", status=status, rounds=len(rec.get("rounds", [])),- cost_usd=float((rec.get("total_usage") or {}).get("cost_usd", 0.0)))- side.minutes = record_minutes(ws)- if not status or status in IN_PROGRESS:- side.error = f"run not finished (status={status!r})"- return side- try:- picked = select.summarise(ws).picked_round+ try: # read once: the round addons/select picks+ rec = load_record(ws)+ picked = select.summarise(ws, record=rec).picked_roundexcept Exception as e: # noqa: BLE001 - one unreadable record must not kill the battery- side.error = f"unreadable record: {e}"- return side- if picked is None:+ return Side(source="3dcodeverse", error=f"unreadable record: {e}")+ status = rec.status.value+ side = Side(source="3dcodeverse", status=status, rounds=len(rec.rounds), cost_usd=rec.total_usage.cost_usd,+ minutes=round(rec.minutes or 0.0, 2))+ if status in IN_PROGRESS:+ side.error = f"run not finished (status={status!r})"+ elif picked is None:side.error = f"no judged round to hand over (status={status!r})"- return side- for cand in (ws / "deliverable" / "object.glb", ws / "artifacts" / f"r{picked:02d}" / "object.glb",- ws / "artifacts" / "object.glb"):- if cand.is_file():- side.glb = str(cand)- return side- side.error = f"finished run has no object.glb under {ws}"+ elif (glb := select.round_file(Workspace(ws), next(r for r in rec.rounds if r.index == picked))) is None:+ side.error = f"round {picked} kept no object.glb under {ws}" # never the last build's canonical one+ else:+ side.glb = str(glb)return sidediff --git a/eval/bench/h2h_scene.py b/eval/bench/h2h_scene.pyindex 81355c04..4f9b12f7 100644--- a/eval/bench/h2h_scene.py+++ b/eval/bench/h2h_scene.py@@ -39,12 +39,13 @@ from pydantic import BaseModelfor _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)-from bench.run_bench import Battery, BenchPrompt, record_minutes # noqa: E402+from bench.run_bench import Battery, BenchPrompt # noqa: E402from bench.stats import sign_test # noqa: E402from codeverse3d.addons import select # noqa: E402from codeverse3d.contracts.artifacts import RenderSet, RenderView # noqa: E402from codeverse3d.contracts.common import Language, Track # noqa: E402from codeverse3d.contracts.spec import Spec # noqa: E402+from codeverse3d.record.record import load_record # noqa: E402HERE = Path(__file__).resolve().parentBATTERY = HERE / "prompts" / "h2h_scene_v1.yaml"@@ -129,26 +130,20 @@ def their_meta(out_dir: Path) -> dict[str, Any]:# ----------------------------------------------------------------------------- oursdef our_frames(run_dir: Path) -> tuple[list[RenderView], dict[str, Any]] | None:- rec_path = run_dir / "record.json"- if not rec_path.is_file():- return None- rec = json.loads(rec_path.read_text())- try:- summary = select.summarise(run_dir)- except Exception: # noqa: BLE001 - an unreadable record is a run we cannot compare+ try: # read once: the round addons/select picks+ rec = load_record(run_dir)+ summary = select.summarise(run_dir, record=rec)+ except Exception: # noqa: BLE001 - a missing or unreadable record is a run we cannot comparereturn Nonepicked = summary.picked_round if summary.picked_round is not None else 0- rnd_path = run_dir / "rounds" / f"r{picked:02d}.json"- if not rnd_path.is_file():+ rnd = next((r for r in rec.rounds if r.index == picked), None)+ if rnd is None or rnd.renders is None:return None- rnd = json.loads(rnd_path.read_text())- views = [RenderView.model_validate(v) for v in (rnd.get("renders") or {}).get("views", [])]- stills = [v for v in views if (v.time_s or 0.0) == 0.0 and v.mode == "shaded" and Path(v.path).is_file()]+ stills = [v for v in rnd.renders.views if (v.time_s or 0.0) == 0.0 and v.mode == "shaded" and Path(v.path).is_file()]flagged = [v for v in stills if v.judge] or stillschosen = flagged[:MAX_VIEWS]- meta = {"harness_score": summary.picked_score, "rounds": len(rec.get("rounds") or []), "picked_round": picked,- "cost_usd": float((rec.get("total_usage") or {}).get("cost_usd") or 0.0), "minutes": record_minutes(run_dir),- "status": summary.stop_reason}+ meta = {"harness_score": summary.picked_score, "rounds": len(rec.rounds), "picked_round": picked,+ "cost_usd": rec.total_usage.cost_usd, "minutes": round(rec.minutes or 0.0, 2), "status": summary.stop_reason}return [RenderView(name=v.name, path=v.path, mode="shaded", width=v.width, height=v.height) for v in chosen], meta