Official Codebase of 3DCodeVerse
Python
bench: one record walk (bench/_records.py) for six surveys; scene_stats.STAGES goes — nothing read it
gaoypeng8d28fef20029
diff --git a/eval/bench/_records.py b/eval/bench/_records.pynew file mode 100644index 00000000..a02ef729--- /dev/null+++ b/eval/bench/_records.py@@ -0,0 +1,38 @@+"""The one walk over a battery's run records, and the one "which prompt is this" rule.++The walk is ``record.unique_files`` (symlinked cells followed and counted once, the+``_assets`` / ``_cand`` sub-workspaces skipped) — each hand-rolled copy of it has at some+point counted a run twice, or (a bare ``rglob``) missed every symlinked cell."""++from __future__ import annotations++import json+from pathlib import Path++from codeverse3d.record.record import unique_files+++def records(root: Path, *, track: str | None = None) -> list[tuple[Path, dict]]:+ """``(record.json path, raw record)`` for every run under ``root``, once per run on+ disk; an unreadable file is skipped, and ``track`` keeps only that track's runs."""+ out = []+ for rec in unique_files(root, "record.json"):+ try:+ data = json.loads(rec.read_text())+ except (OSError, ValueError):+ continue+ if track is None or (data.get("spec") or {}).get("track") == track:+ out.append((rec, data))+ return out+++def prompt_of(path: Path, root: Path | None = None) -> str:+ """The battery cell a file belongs to: the directory after ``cells/`` (``ab_plan``,+ ``compare_backends``) or ``runs/`` (``bench run``). With ``root`` only the part below+ it is read, and a file in no such layout is named by its first directory there;+ otherwise by its parent directory."""+ parts = path.relative_to(root).parts if root is not None else path.parts+ for anchor in ("cells", "runs"):+ if anchor in parts:+ return parts[parts.index(anchor) + 1]+ return parts[0] if root is not None and len(parts) > 1 else path.parent.namediff --git a/eval/bench/coupling_stats.py b/eval/bench/coupling_stats.pyindex 35fa21a1..455f9998 100644--- a/eval/bench/coupling_stats.py+++ b/eval/bench/coupling_stats.py@@ -16,7 +16,6 @@ a coupling at all — the mechanism claim in docs/PAPER_WRITING.md §5.3.from __future__ import annotationsimport argparse-import jsonimport statisticsimport sysfrom pathlib import Path@@ -24,6 +23,7 @@ from pathlib import Pathfor _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)+from bench._records import prompt_of, records # noqa: E402from codeverse3d.record.record import unique_files # noqa: E402from codeverse3d.spatial.joints_model import UrdfError, load_urdf # noqa: E402@@ -39,19 +39,6 @@ def _urdfs(root: Path) -> list[Path]:return [p for p in unique_files(root, "robot.urdf") if p.parent.name == "artifacts"]-def _prompt_of(path: Path) -> str:- """The battery cell a URDF belongs to.-- Two layouts: ``ab_plan``/``compare_backends`` write ``cells/<prompt>/<arm>/run/...``- and ``bench run`` writes ``runs/<prompt>/...``. Falling back to the parent directory- named every URDF "artifacts"."""- parts = path.parts- for marker in ("cells", "runs"):- if marker in parts:- return parts[parts.index(marker) + 1]- return path.parent.name--def per_prompt(root: Path) -> str:"""One row per prompt that declared a coupling: what it moves, what drives it."""rows: dict[str, list[tuple[int, int]]] = {}@@ -62,7 +49,7 @@ def per_prompt(root: Path) -> str:continuemoves, free = len(robot.movable_joints()), len(robot.independent_joints())if moves != free:- rows.setdefault(_prompt_of(path), []).append((moves, free))+ rows.setdefault(prompt_of(path), []).append((moves, free))out = [f"{root.name}, per prompt (coupled URDFs only):","| prompt | URDFs | movable joints (median) | driven per pose (median) | fewest driven |","|---|--:|--:|--:|--:|"]@@ -87,11 +74,11 @@ def survey(root: Path) -> dict:continuemoves = [j for j in robot.joints.values() if j.movable]free = robot.independent_joints()- prompts.add(_prompt_of(path))+ prompts.add(prompt_of(path))movable.append(len(moves))independent.append(len(free))if len(free) != len(moves):- coupled_prompts.add(_prompt_of(path))+ coupled_prompts.add(prompt_of(path))coupled = [(m, i) for m, i in zip(movable, independent, strict=True) if m != i]return {"urdfs": len(movable), "unreadable": unreadable, "prompts": len(prompts),"coupled_prompts": len(coupled_prompts), "movable": movable, "independent": independent,@@ -104,11 +91,7 @@ def gate_stats(root: Path) -> dict:The mechanical counts below say which poses were SAMPLED; this says what the gate thatjudges those poses reported, which is the loss event a coupled battery is run for."""rounds = ran = failed = errors = 0- for rec in unique_files(root, "record.json"): # the same cells _urdfs saw, symlinks included- try:- data = json.loads(rec.read_text())- except (OSError, ValueError):- continue+ for _, data in records(root): # the same cells _urdfs saw, symlinks includedfor rnd in data.get("rounds") or []:rounds += 1for g in rnd.get("gates") or []:diff --git a/eval/bench/penetration_thresholds.py b/eval/bench/penetration_thresholds.pyindex aad11a10..237d1863 100644--- a/eval/bench/penetration_thresholds.py+++ b/eval/bench/penetration_thresholds.py@@ -18,7 +18,6 @@ from __future__ import annotationsimport argparseimport inspect-import jsonimport statisticsimport sysfrom collections import Counter@@ -27,8 +26,8 @@ from pathlib import Pathfor _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)+from bench._records import records # noqa: E402from codeverse3d.languages.urdf import REST_PENETRATION_MAX_M # noqa: E402-from codeverse3d.record.record import unique_files # noqa: E402from codeverse3d.spatial.connectivity import ( # noqa: E402PENETRATION_ERROR_M,PENETRATION_WARN_M,@@ -47,26 +46,6 @@ SWEEP_REST_ERROR_M = float(inspect.signature(sweep_findings).parameters["rest_maassert SWEEP_REST_ERROR_M == REST_PENETRATION_MAX_M, "the urdf runtime disagrees with the sweep default"---def records(root: Path) -> list[dict]:- """Every articulated run record under ``root``, once per run.-- The walk is ``flywheel.record.unique_files``: it follows symlinked cells and the- ``run/telemetry/trajectories`` link and counts each file once, and skips the- sub-workspaces a run owns — the trap that inflated COST §30, handled in one place.- """- out = []- for rec in unique_files(root, "record.json"):- try:- data = json.loads(rec.read_text())- except (OSError, ValueError):- continue- if (data.get("spec") or {}).get("track") == "articulated_object":- out.append(data)- return out--def sweep_pairs(sweep: dict | None, *, rest_only: bool) -> dict[tuple[str, ...], float]:"""``{link pair: worst depth}`` from a ``joint_sweep`` gate's ERROR findings.@@ -166,7 +145,7 @@ def corroborate(recs: list[dict], threshold_m: float) -> dict:def report(root: Path) -> str:- recs = records(root) # read once: this used to walk every record.json twice+ recs = [data for _, data in records(root, track="articulated_object")] # read once: this used to walk every record.json twices = survey(recs)rounds, worst = s["rounds"], s["worst"]nonzero = [w for w in worst if w > 0]diff --git a/eval/bench/scene_regate.py b/eval/bench/scene_regate.pyindex f93b0b08..de2f50f1 100644--- a/eval/bench/scene_regate.py+++ b/eval/bench/scene_regate.py@@ -32,7 +32,7 @@ from pathlib import Pathfor _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)-from bench.scene_stats import scene_records # noqa: E402+from bench._records import records # noqa: E402from codeverse3d.config import get_settings # noqa: E402from codeverse3d.contracts.artifacts import Severity # noqa: E402from codeverse3d.spatial.scene_placement import placement_gate # noqa: E402@@ -43,7 +43,7 @@ GATE = "scene_placement"def scene_runs(root: Path) -> list[Path]:"""Every recorded scene run under ``root`` that still has a workspace to probe."""- return [run for run, _ in scene_records(root) if (run / "src" / "scene.js").is_file()]+ return [rec.parent for rec, _ in records(root, track="scene") if (rec.parent / "src" / "scene.js").is_file()]def recorded_errors(record: dict) -> list[str]:diff --git a/eval/bench/scene_stats.py b/eval/bench/scene_stats.pyindex af60017c..4c014881 100644--- a/eval/bench/scene_stats.py+++ b/eval/bench/scene_stats.py@@ -18,7 +18,6 @@ a defect in the plan or in the assets follows any change to the assembly languagfrom __future__ import annotationsimport argparse-import jsonimport statisticsimport sysfrom collections import Counter@@ -28,34 +27,15 @@ for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolsys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)from bench._jsonl import read_jsonl # noqa: E402-from codeverse3d.record.record import unique_files # noqa: E402+from bench._records import records # noqa: E402from codeverse3d.spatial.node import browser_was_lost # noqa: E402-#: pipeline stages in the order the scene track runs them-STAGES = ("plan", "skeleton", "assets", "env", "layouts", "assemble", "generate", "build")---def scene_records(root: Path) -> list[tuple[Path, dict]]:- """``(run dir, record)`` for every scene run under ``root``, once per run on disk.-- The walk is ``flywheel.record.unique_files`` (symlinked cells collapsed, the- ``_assets`` / ``_cand`` sub-workspaces skipped) — the one walker every scene survey- shares (``scene_regate`` re-gates exactly the runs this counts)."""- out = []- for rec in unique_files(root, "record.json"):- try:- data = json.loads(rec.read_text())- except (OSError, ValueError):- continue- if (data.get("spec") or {}).get("track") == "scene":- out.append((rec.parent, data))- return out-def runs(root: Path) -> list[tuple[str, dict, list[dict]]]:"""``(prompt id, record, events)`` per scene run under ``root``, once per run."""out = []- for run, data in scene_records(root):+ for rec, data in records(root, track="scene"):+ run = rec.parentevents = read_jsonl(run / "events.jsonl")out.append((str((data.get("spec") or {}).get("id", run.name)).split("/")[-1], data, events))# by id, then by path: two runs of one prompt (a battery with reps) share the id, and adiff --git a/eval/bench/session_stats.py b/eval/bench/session_stats.pyindex cba77070..4323e79e 100644--- a/eval/bench/session_stats.py+++ b/eval/bench/session_stats.py@@ -25,6 +25,7 @@ from pathlib import Pathfor _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)+from bench._records import records # noqa: E402from codeverse3d.record.record import unique_files # noqa: E402#: gemini-cli names an MCP tool ``mcp_<server>_<tool>``; ours is the ``3dcode`` server@@ -103,12 +104,8 @@ def token_rates(sessions: list[Path]) -> dict[str, float]:def round_costs(root: Path) -> list[float]:"""Generator dollars per round, from the run records (judging is billed elsewhere)."""out: list[float] = []- for rec in unique_files(root, "record.json"):- try:- rounds = (json.loads(rec.read_text()) or {}).get("rounds") or []- except (OSError, ValueError):- continue- out += [float((r.get("usage") or {}).get("cost_usd") or 0.0) for r in rounds+ for _, data in records(root):+ out += [float((r.get("usage") or {}).get("cost_usd") or 0.0) for r in data.get("rounds") or []if (r.get("usage") or {}).get("cost_usd")]return outdiff --git a/eval/bench/skill_targets.py b/eval/bench/skill_targets.pyindex 32da9edd..811b2dd9 100644--- a/eval/bench/skill_targets.py+++ b/eval/bench/skill_targets.py@@ -33,6 +33,7 @@ from typing import Anyfor _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)+from bench._records import prompt_of, records # noqa: E402from bench.stats import mean_ci, n_to_resolve # noqa: E402from codeverse3d.addons.skill_targets import ( # noqa: E402SRC_ARTIFACT,@@ -95,24 +96,8 @@ class Run:def load_runs(root: Path) -> list[Run]:- out: list[Run] = []- for rec in sorted(root.rglob("record.json")):- try:- data = json.loads(rec.read_text())- except (OSError, json.JSONDecodeError):- continue- out.append(Run(key=_key_of(rec, root), label=str(rec.parent.relative_to(root)),- path=rec, record=data))- return out---def _key_of(rec: Path, root: Path) -> str:- """A stable prompt id: the cell/run directory name, not the whole path."""- parts = rec.relative_to(root).parts- for anchor in ("cells", "runs"):- if anchor in parts:- return parts[parts.index(anchor) + 1]- return parts[0] if parts else rec.parent.name+ return [Run(key=prompt_of(rec, root), label=str(rec.parent.relative_to(root)), path=rec, record=data)+ for rec, data in records(root)]def ab_arms(root: Path) -> dict[str, Path] | None:diff --git a/eval/tests/test_session_stats.py b/eval/tests/test_session_stats.pyindex ec199d90..a0f5a7f1 100644--- a/eval/tests/test_session_stats.py+++ b/eval/tests/test_session_stats.py@@ -148,7 +148,8 @@ def test_the_coupling_survey_counts_runs_not_copies_of_the_same_urdf(tmp_path: Pdeliverable/ (finalised) — and `bench run` lays cells out as runs/<id>/, notcells/<id>/. Counting files tripled every mechanical number and named every prompt"artifacts"."""- from bench.coupling_stats import _prompt_of, _urdfs+ from bench._records import prompt_of+ from bench.coupling_stats import _urdfsurdf = """<?xml version="1.0"?><robot name="rig"><link name="base"/><link name="a"/>@@ -163,7 +164,7 @@ def test_the_coupling_survey_counts_runs_not_copies_of_the_same_urdf(tmp_path: Pfound = _urdfs(tmp_path)assert [p.parent.name for p in found] == ["artifacts"], "one URDF per run, the built one"- assert _prompt_of(found[0]) == "cpl_umbrella"+ assert prompt_of(found[0]) == "cpl_umbrella"def _sweep(pair: str, depth_m: float, *, poses: list[dict], worst_pose: dict) -> dict: