Skip to content

Official Codebase of 3DCodeVerse

Python

bench: h2h/ab_view/complexity readers load the record once and take round N's GLB from round N (6.5, B4)

gaoypeng567c1f0529a8
diff --git a/eval/bench/ab_view.py b/eval/bench/ab_view.py
index 22b0060a..d707b254 100644
--- a/eval/bench/ab_view.py
+++ b/eval/bench/ab_view.py
@@ -21,7 +21,6 @@ import argparse
import base64
import html
import io
-import json
import statistics
import sys
from dataclasses import dataclass, field
@@ -77,26 +76,19 @@ def _sheet_data_uri(run_dir: Path) -> str:
def load_run(run_dir: Path, arm: str) -> Run:
r = Run(slug=run_dir.name, arm=arm)
- rec = run_dir / "record.json"
- if rec.is_file():
- d = json.loads(rec.read_text())
- r.brief = str((d.get("spec") or {}).get("prompt") or "")
- try: # the round addons/select picks; a record it cannot read keeps only its status
- s = select.summarise(run_dir)
- r.status, r.picked, r.baseline = s.stop_reason or "?", s.picked_score, s.baseline_score
- except Exception: # noqa: BLE001 - one unreadable record must not blank the page
- r.status = str(d.get("status") or "?")
- r.rounds = len(d.get("rounds") or [])
- try: # the ledger's total and the steps' minutes, as every reader reports them
- run = load_record(run_dir)
- r.usd, r.minutes = round(run.total_usage.cost_usd, 2), round(run.minutes or 0)
- except Exception: # noqa: BLE001 - an unreadable record keeps $0 / 0 min
- pass
- r.error = str(d.get("error") or "")[:300]
- for rd in d.get("rounds") or []:
- failed = [g.get("gate") for g in (rd.get("gates") or []) if not g.get("passed")]
+ try: # read once: the round addons/select picks, the ledger's total, the steps' minutes
+ rec = load_record(run_dir)
+ s = select.summarise(run_dir, record=rec)
+ except Exception: # noqa: BLE001 - a missing or unreadable record must not blank the page
+ rec = None
+ if rec is not None:
+ r.brief, r.rounds, r.error = rec.spec.prompt, len(rec.rounds), (rec.error or "")[:300]
+ r.status, r.picked, r.baseline = s.stop_reason or "?", s.picked_score, s.baseline_score
+ r.usd, r.minutes = round(rec.total_usage.cost_usd, 2), round(rec.minutes or 0)
+ for rd in rec.rounds:
+ failed = [g.gate for g in rd.gates if not g.passed]
if failed:
- r.notes.append(f"r{rd.get('index')} gates: {', '.join(str(g) for g in failed[:4])}")
+ r.notes.append(f"r{rd.index} gates: {', '.join(failed[:4])}")
r.sheet = _sheet_data_uri(run_dir)
return r
diff --git a/eval/bench/complexity_report.py b/eval/bench/complexity_report.py
index e9f4530b..505f68be 100644
--- a/eval/bench/complexity_report.py
+++ b/eval/bench/complexity_report.py
@@ -34,7 +34,7 @@ for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resol
from bench.stats import correlation # noqa: E402
from codeverse3d.addons import select # noqa: E402
-from codeverse3d.contracts.run import RunId, RunRecord # noqa: E402
+from codeverse3d.contracts.run import RoundRecord, RunId, RunRecord # noqa: E402
from codeverse3d.proc import read_json_or_none # noqa: E402
from codeverse3d.record.record import find_run_dirs # noqa: E402
from codeverse3d.spatial.complexity import ( # noqa: E402
@@ -42,6 +42,7 @@ from codeverse3d.spatial.complexity import ( # noqa: E402
ComplexityVector,
band_of,
)
+from codeverse3d.workspace import Workspace # noqa: E402
CRITERIA = (
"intent_fidelity",
@@ -70,22 +71,23 @@ _read_json = read_json_or_none
def _vector_for(run: Path, rnd: dict[str, Any] | None) -> ComplexityVector | None:
- """The run's recorded complexity vector, else one recomputed from its GLB.
+ """The round's recorded complexity vector, else one recomputed from the round's own GLB.
The historic corpus predates the vector, so recomputing is the normal path
for anything recorded before 2026-08-24 — that is what makes the whole
- corpus comparable.
+ corpus comparable. The round is named by its index alone (a historic round may not
+ validate); ``addons.select`` answers for its own files, never the last build's.
"""
- for meas in (rnd or {}).get("measurement"), (_read_json(run / "artifacts" / "measurement.json") or {}):
- block = ((meas or {}).get("extra") or {}).get("complexity")
+ ws, one = Workspace(run), RoundRecord(index=int((rnd or {}).get("index") or 0), kind="")
+ own = (((rnd or {}).get("measurement") or {}).get("extra") or {}).get("complexity")
+ for block in (own, select.round_complexity_block(ws, one)):
if isinstance(block, dict):
try:
return ComplexityVector.model_validate(block)
except Exception: # noqa: BLE001 - an older/foreign block must not stop the scan
pass
- kept = run / "artifacts" / f"r{int((rnd or {}).get('index') or 0):02d}" / "object.glb"
- glb = kept if kept.is_file() else run / "artifacts" / "object.glb"
- if not glb.is_file():
+ glb = select.round_file(ws, one)
+ if glb is None:
return None
from codeverse3d.spatial.complexity import complexity_of_glb
@@ -95,31 +97,26 @@ def _vector_for(run: Path, rnd: dict[str, Any] | None) -> ComplexityVector | Non
return None
-def _picked_round(run: Path, record: dict[str, Any]) -> dict[str, Any] | None:
+def _picked_round(run: Path, record: dict[str, Any], rec: RunRecord | None) -> dict[str, Any] | None:
"""The round ``addons/select`` picks, as its raw dict; the last round when the record
will not load (the scan reads the historic corpus as-is)."""
rounds = record.get("rounds") or []
try:
- idx = select.summarise(run).picked_round
+ idx = select.summarise(run, record=rec).picked_round if rec is not None else None
except Exception: # noqa: BLE001 - one unreadable record must not stop the scan
idx = None
return next((r for r in rounds if r.get("index") == idx), rounds[-1] if rounds else None)
-def _minutes(record: dict[str, Any]) -> float | None:
- """``RunRecord.minutes`` (docs/COST.md §31); None for a record that will not load."""
- try:
- minutes = RunRecord.model_validate(record).minutes
- except Exception: # noqa: BLE001 - the historic corpus is read as-is
- return None
- return None if minutes is None else round(minutes, 2)
-
-
def row_for(run: Path, battery: str, slug: str | None = None) -> Row | None:
record = _read_json(run / "record.json")
if not record:
return None
- rnd = _picked_round(run, record)
+ try: # validated once; the historic corpus is read as a raw dict where it will not load
+ rec: RunRecord | None = RunRecord.model_validate(record)
+ except Exception: # noqa: BLE001
+ rec = None
+ rnd = _picked_round(run, record, rec)
judgment = (rnd or {}).get("judgment") or {}
scores = judgment.get("scores") or {}
vec = _vector_for(run, rnd)
@@ -141,7 +138,7 @@ def row_for(run: Path, battery: str, slug: str | None = None) -> Row | None:
overall=judgment.get("overall"),
issues=len(judgment.get("issues") or []),
cost_usd=round(float((record.get("total_usage") or {}).get("cost_usd") or 0.0), 4),
- minutes=_minutes(record),
+ minutes=round(rec.minutes, 2) if rec is not None and rec.minutes is not None else None,
index=vec.index,
band=vec.band,
)
diff --git a/eval/bench/h2h_glb.py b/eval/bench/h2h_glb.py
index 68b46cca..aef7c2c6 100644
--- a/eval/bench/h2h_glb.py
+++ b/eval/bench/h2h_glb.py
@@ -45,9 +45,11 @@ for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resol
sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)
from bench._fixed_eval import RUBRIC # noqa: E402
-from bench.run_bench import Battery, BenchPrompt, record_minutes # noqa: E402
+from bench.run_bench import Battery, BenchPrompt # noqa: E402
from bench.stats import sign_test # noqa: E402
from codeverse3d.addons import select # noqa: E402
+from codeverse3d.record.record import load_record # noqa: E402
+from codeverse3d.workspace import Workspace # noqa: E402
JUDGE_MODEL = "gemini:gemini-3.1-pro-preview"
N_SAMPLES = 2
@@ -108,31 +110,24 @@ def their_side(gallery: Path, slug: str) -> Side:
def our_side(runs_dir: Path, item_id: str) -> Side:
ws = runs_dir / item_id
- rec_path = ws / "record.json"
- if not rec_path.is_file():
+ if not (ws / "record.json").is_file():
return Side(source="3dcodeverse", status="not_run", error=f"no record.json under {ws}")
- rec: dict[str, Any] = json.loads(rec_path.read_text())
- status = str(rec.get("status", ""))
- side = Side(source="3dcodeverse", status=status, rounds=len(rec.get("rounds", [])),
- cost_usd=float((rec.get("total_usage") or {}).get("cost_usd", 0.0)))
- side.minutes = record_minutes(ws)
- if not status or status in IN_PROGRESS:
- side.error = f"run not finished (status={status!r})"
- return side
- try:
- picked = select.summarise(ws).picked_round
+ try: # read once: the round addons/select picks
+ rec = load_record(ws)
+ picked = select.summarise(ws, record=rec).picked_round
except Exception as e: # noqa: BLE001 - one unreadable record must not kill the battery
- side.error = f"unreadable record: {e}"
- return side
- if picked is None:
+ return Side(source="3dcodeverse", error=f"unreadable record: {e}")
+ status = rec.status.value
+ side = Side(source="3dcodeverse", status=status, rounds=len(rec.rounds), cost_usd=rec.total_usage.cost_usd,
+ minutes=round(rec.minutes or 0.0, 2))
+ if status in IN_PROGRESS:
+ side.error = f"run not finished (status={status!r})"
+ elif picked is None:
side.error = f"no judged round to hand over (status={status!r})"
- return side
- for cand in (ws / "deliverable" / "object.glb", ws / "artifacts" / f"r{picked:02d}" / "object.glb",
- ws / "artifacts" / "object.glb"):
- if cand.is_file():
- side.glb = str(cand)
- return side
- side.error = f"finished run has no object.glb under {ws}"
+ elif (glb := select.round_file(Workspace(ws), next(r for r in rec.rounds if r.index == picked))) is None:
+ side.error = f"round {picked} kept no object.glb under {ws}" # never the last build's canonical one
+ else:
+ side.glb = str(glb)
return side
diff --git a/eval/bench/h2h_scene.py b/eval/bench/h2h_scene.py
index 81355c04..4f9b12f7 100644
--- a/eval/bench/h2h_scene.py
+++ b/eval/bench/h2h_scene.py
@@ -39,12 +39,13 @@ from pydantic import BaseModel
for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):
sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)
-from bench.run_bench import Battery, BenchPrompt, record_minutes # noqa: E402
+from bench.run_bench import Battery, BenchPrompt # noqa: E402
from bench.stats import sign_test # noqa: E402
from codeverse3d.addons import select # noqa: E402
from codeverse3d.contracts.artifacts import RenderSet, RenderView # noqa: E402
from codeverse3d.contracts.common import Language, Track # noqa: E402
from codeverse3d.contracts.spec import Spec # noqa: E402
+from codeverse3d.record.record import load_record # noqa: E402
HERE = Path(__file__).resolve().parent
BATTERY = HERE / "prompts" / "h2h_scene_v1.yaml"
@@ -129,26 +130,20 @@ def their_meta(out_dir: Path) -> dict[str, Any]:
# ----------------------------------------------------------------------------- ours
def our_frames(run_dir: Path) -> tuple[list[RenderView], dict[str, Any]] | None:
- rec_path = run_dir / "record.json"
- if not rec_path.is_file():
- return None
- rec = json.loads(rec_path.read_text())
- try:
- summary = select.summarise(run_dir)
- except Exception: # noqa: BLE001 - an unreadable record is a run we cannot compare
+ try: # read once: the round addons/select picks
+ rec = load_record(run_dir)
+ summary = select.summarise(run_dir, record=rec)
+ except Exception: # noqa: BLE001 - a missing or unreadable record is a run we cannot compare
return None
picked = summary.picked_round if summary.picked_round is not None else 0
- rnd_path = run_dir / "rounds" / f"r{picked:02d}.json"
- if not rnd_path.is_file():
+ rnd = next((r for r in rec.rounds if r.index == picked), None)
+ if rnd is None or rnd.renders is None:
return None
- rnd = json.loads(rnd_path.read_text())
- views = [RenderView.model_validate(v) for v in (rnd.get("renders") or {}).get("views", [])]
- stills = [v for v in views if (v.time_s or 0.0) == 0.0 and v.mode == "shaded" and Path(v.path).is_file()]
+ stills = [v for v in rnd.renders.views if (v.time_s or 0.0) == 0.0 and v.mode == "shaded" and Path(v.path).is_file()]
flagged = [v for v in stills if v.judge] or stills
chosen = flagged[:MAX_VIEWS]
- meta = {"harness_score": summary.picked_score, "rounds": len(rec.get("rounds") or []), "picked_round": picked,
- "cost_usd": float((rec.get("total_usage") or {}).get("cost_usd") or 0.0), "minutes": record_minutes(run_dir),
- "status": summary.stop_reason}
+ meta = {"harness_score": summary.picked_score, "rounds": len(rec.rounds), "picked_round": picked,
+ "cost_usd": rec.total_usage.cost_usd, "minutes": round(rec.minutes or 0.0, 2), "status": summary.stop_reason}
return [RenderView(name=v.name, path=v.path, mode="shaded", width=v.width, height=v.height) for v in chosen], meta