Skip to content

Official Codebase of 3DCodeVerse

Python

bench: one record walk (bench/_records.py) for six surveys; scene_stats.STAGES goes — nothing read it

gaoypeng8d28fef20029
diff --git a/eval/bench/_records.py b/eval/bench/_records.py
new file mode 100644
index 00000000..a02ef729
--- /dev/null
+++ b/eval/bench/_records.py
@@ -0,0 +1,38 @@
+"""The one walk over a battery's run records, and the one "which prompt is this" rule.
+
+The walk is ``record.unique_files`` (symlinked cells followed and counted once, the
+``_assets`` / ``_cand`` sub-workspaces skipped) — each hand-rolled copy of it has at some
+point counted a run twice, or (a bare ``rglob``) missed every symlinked cell."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from codeverse3d.record.record import unique_files
+
+
+def records(root: Path, *, track: str | None = None) -> list[tuple[Path, dict]]:
+ """``(record.json path, raw record)`` for every run under ``root``, once per run on
+ disk; an unreadable file is skipped, and ``track`` keeps only that track's runs."""
+ out = []
+ for rec in unique_files(root, "record.json"):
+ try:
+ data = json.loads(rec.read_text())
+ except (OSError, ValueError):
+ continue
+ if track is None or (data.get("spec") or {}).get("track") == track:
+ out.append((rec, data))
+ return out
+
+
+def prompt_of(path: Path, root: Path | None = None) -> str:
+ """The battery cell a file belongs to: the directory after ``cells/`` (``ab_plan``,
+ ``compare_backends``) or ``runs/`` (``bench run``). With ``root`` only the part below
+ it is read, and a file in no such layout is named by its first directory there;
+ otherwise by its parent directory."""
+ parts = path.relative_to(root).parts if root is not None else path.parts
+ for anchor in ("cells", "runs"):
+ if anchor in parts:
+ return parts[parts.index(anchor) + 1]
+ return parts[0] if root is not None and len(parts) > 1 else path.parent.name
diff --git a/eval/bench/coupling_stats.py b/eval/bench/coupling_stats.py
index 35fa21a1..455f9998 100644
--- a/eval/bench/coupling_stats.py
+++ b/eval/bench/coupling_stats.py
@@ -16,7 +16,6 @@ a coupling at all — the mechanism claim in docs/PAPER_WRITING.md §5.3.
from __future__ import annotations
import argparse
-import json
import statistics
import sys
from pathlib import Path
@@ -24,6 +23,7 @@ from pathlib import Path
for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):
sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)
+from bench._records import prompt_of, records # noqa: E402
from codeverse3d.record.record import unique_files # noqa: E402
from codeverse3d.spatial.joints_model import UrdfError, load_urdf # noqa: E402
@@ -39,19 +39,6 @@ def _urdfs(root: Path) -> list[Path]:
return [p for p in unique_files(root, "robot.urdf") if p.parent.name == "artifacts"]
-def _prompt_of(path: Path) -> str:
- """The battery cell a URDF belongs to.
-
- Two layouts: ``ab_plan``/``compare_backends`` write ``cells/<prompt>/<arm>/run/...``
- and ``bench run`` writes ``runs/<prompt>/...``. Falling back to the parent directory
- named every URDF "artifacts"."""
- parts = path.parts
- for marker in ("cells", "runs"):
- if marker in parts:
- return parts[parts.index(marker) + 1]
- return path.parent.name
-
-
def per_prompt(root: Path) -> str:
"""One row per prompt that declared a coupling: what it moves, what drives it."""
rows: dict[str, list[tuple[int, int]]] = {}
@@ -62,7 +49,7 @@ def per_prompt(root: Path) -> str:
continue
moves, free = len(robot.movable_joints()), len(robot.independent_joints())
if moves != free:
- rows.setdefault(_prompt_of(path), []).append((moves, free))
+ rows.setdefault(prompt_of(path), []).append((moves, free))
out = [f"{root.name}, per prompt (coupled URDFs only):",
"| prompt | URDFs | movable joints (median) | driven per pose (median) | fewest driven |",
"|---|--:|--:|--:|--:|"]
@@ -87,11 +74,11 @@ def survey(root: Path) -> dict:
continue
moves = [j for j in robot.joints.values() if j.movable]
free = robot.independent_joints()
- prompts.add(_prompt_of(path))
+ prompts.add(prompt_of(path))
movable.append(len(moves))
independent.append(len(free))
if len(free) != len(moves):
- coupled_prompts.add(_prompt_of(path))
+ coupled_prompts.add(prompt_of(path))
coupled = [(m, i) for m, i in zip(movable, independent, strict=True) if m != i]
return {"urdfs": len(movable), "unreadable": unreadable, "prompts": len(prompts),
"coupled_prompts": len(coupled_prompts), "movable": movable, "independent": independent,
@@ -104,11 +91,7 @@ def gate_stats(root: Path) -> dict:
The mechanical counts below say which poses were SAMPLED; this says what the gate that
judges those poses reported, which is the loss event a coupled battery is run for."""
rounds = ran = failed = errors = 0
- for rec in unique_files(root, "record.json"): # the same cells _urdfs saw, symlinks included
- try:
- data = json.loads(rec.read_text())
- except (OSError, ValueError):
- continue
+ for _, data in records(root): # the same cells _urdfs saw, symlinks included
for rnd in data.get("rounds") or []:
rounds += 1
for g in rnd.get("gates") or []:
diff --git a/eval/bench/penetration_thresholds.py b/eval/bench/penetration_thresholds.py
index aad11a10..237d1863 100644
--- a/eval/bench/penetration_thresholds.py
+++ b/eval/bench/penetration_thresholds.py
@@ -18,7 +18,6 @@ from __future__ import annotations
import argparse
import inspect
-import json
import statistics
import sys
from collections import Counter
@@ -27,8 +26,8 @@ from pathlib import Path
for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):
sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)
+from bench._records import records # noqa: E402
from codeverse3d.languages.urdf import REST_PENETRATION_MAX_M # noqa: E402
-from codeverse3d.record.record import unique_files # noqa: E402
from codeverse3d.spatial.connectivity import ( # noqa: E402
PENETRATION_ERROR_M,
PENETRATION_WARN_M,
@@ -47,26 +46,6 @@ SWEEP_REST_ERROR_M = float(inspect.signature(sweep_findings).parameters["rest_ma
assert SWEEP_REST_ERROR_M == REST_PENETRATION_MAX_M, "the urdf runtime disagrees with the sweep default"
-
-
-def records(root: Path) -> list[dict]:
- """Every articulated run record under ``root``, once per run.
-
- The walk is ``flywheel.record.unique_files``: it follows symlinked cells and the
- ``run/telemetry/trajectories`` link and counts each file once, and skips the
- sub-workspaces a run owns — the trap that inflated COST §30, handled in one place.
- """
- out = []
- for rec in unique_files(root, "record.json"):
- try:
- data = json.loads(rec.read_text())
- except (OSError, ValueError):
- continue
- if (data.get("spec") or {}).get("track") == "articulated_object":
- out.append(data)
- return out
-
-
def sweep_pairs(sweep: dict | None, *, rest_only: bool) -> dict[tuple[str, ...], float]:
"""``{link pair: worst depth}`` from a ``joint_sweep`` gate's ERROR findings.
@@ -166,7 +145,7 @@ def corroborate(recs: list[dict], threshold_m: float) -> dict:
def report(root: Path) -> str:
- recs = records(root) # read once: this used to walk every record.json twice
+ recs = [data for _, data in records(root, track="articulated_object")] # read once: this used to walk every record.json twice
s = survey(recs)
rounds, worst = s["rounds"], s["worst"]
nonzero = [w for w in worst if w > 0]
diff --git a/eval/bench/scene_regate.py b/eval/bench/scene_regate.py
index f93b0b08..de2f50f1 100644
--- a/eval/bench/scene_regate.py
+++ b/eval/bench/scene_regate.py
@@ -32,7 +32,7 @@ from pathlib import Path
for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):
sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)
-from bench.scene_stats import scene_records # noqa: E402
+from bench._records import records # noqa: E402
from codeverse3d.config import get_settings # noqa: E402
from codeverse3d.contracts.artifacts import Severity # noqa: E402
from codeverse3d.spatial.scene_placement import placement_gate # noqa: E402
@@ -43,7 +43,7 @@ GATE = "scene_placement"
def scene_runs(root: Path) -> list[Path]:
"""Every recorded scene run under ``root`` that still has a workspace to probe."""
- return [run for run, _ in scene_records(root) if (run / "src" / "scene.js").is_file()]
+ return [rec.parent for rec, _ in records(root, track="scene") if (rec.parent / "src" / "scene.js").is_file()]
def recorded_errors(record: dict) -> list[str]:
diff --git a/eval/bench/scene_stats.py b/eval/bench/scene_stats.py
index af60017c..4c014881 100644
--- a/eval/bench/scene_stats.py
+++ b/eval/bench/scene_stats.py
@@ -18,7 +18,6 @@ a defect in the plan or in the assets follows any change to the assembly languag
from __future__ import annotations
import argparse
-import json
import statistics
import sys
from collections import Counter
@@ -28,34 +27,15 @@ for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resol
sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)
from bench._jsonl import read_jsonl # noqa: E402
-from codeverse3d.record.record import unique_files # noqa: E402
+from bench._records import records # noqa: E402
from codeverse3d.spatial.node import browser_was_lost # noqa: E402
-#: pipeline stages in the order the scene track runs them
-STAGES = ("plan", "skeleton", "assets", "env", "layouts", "assemble", "generate", "build")
-
-
-def scene_records(root: Path) -> list[tuple[Path, dict]]:
- """``(run dir, record)`` for every scene run under ``root``, once per run on disk.
-
- The walk is ``flywheel.record.unique_files`` (symlinked cells collapsed, the
- ``_assets`` / ``_cand`` sub-workspaces skipped) — the one walker every scene survey
- shares (``scene_regate`` re-gates exactly the runs this counts)."""
- out = []
- for rec in unique_files(root, "record.json"):
- try:
- data = json.loads(rec.read_text())
- except (OSError, ValueError):
- continue
- if (data.get("spec") or {}).get("track") == "scene":
- out.append((rec.parent, data))
- return out
-
def runs(root: Path) -> list[tuple[str, dict, list[dict]]]:
"""``(prompt id, record, events)`` per scene run under ``root``, once per run."""
out = []
- for run, data in scene_records(root):
+ for rec, data in records(root, track="scene"):
+ run = rec.parent
events = read_jsonl(run / "events.jsonl")
out.append((str((data.get("spec") or {}).get("id", run.name)).split("/")[-1], data, events))
# by id, then by path: two runs of one prompt (a battery with reps) share the id, and a
diff --git a/eval/bench/session_stats.py b/eval/bench/session_stats.py
index cba77070..4323e79e 100644
--- a/eval/bench/session_stats.py
+++ b/eval/bench/session_stats.py
@@ -25,6 +25,7 @@ from pathlib import Path
for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):
sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)
+from bench._records import records # noqa: E402
from codeverse3d.record.record import unique_files # noqa: E402
#: gemini-cli names an MCP tool ``mcp_<server>_<tool>``; ours is the ``3dcode`` server
@@ -103,12 +104,8 @@ def token_rates(sessions: list[Path]) -> dict[str, float]:
def round_costs(root: Path) -> list[float]:
"""Generator dollars per round, from the run records (judging is billed elsewhere)."""
out: list[float] = []
- for rec in unique_files(root, "record.json"):
- try:
- rounds = (json.loads(rec.read_text()) or {}).get("rounds") or []
- except (OSError, ValueError):
- continue
- out += [float((r.get("usage") or {}).get("cost_usd") or 0.0) for r in rounds
+ for _, data in records(root):
+ out += [float((r.get("usage") or {}).get("cost_usd") or 0.0) for r in data.get("rounds") or []
if (r.get("usage") or {}).get("cost_usd")]
return out
diff --git a/eval/bench/skill_targets.py b/eval/bench/skill_targets.py
index 32da9edd..811b2dd9 100644
--- a/eval/bench/skill_targets.py
+++ b/eval/bench/skill_targets.py
@@ -33,6 +33,7 @@ from typing import Any
for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):
sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)
+from bench._records import prompt_of, records # noqa: E402
from bench.stats import mean_ci, n_to_resolve # noqa: E402
from codeverse3d.addons.skill_targets import ( # noqa: E402
SRC_ARTIFACT,
@@ -95,24 +96,8 @@ class Run:
def load_runs(root: Path) -> list[Run]:
- out: list[Run] = []
- for rec in sorted(root.rglob("record.json")):
- try:
- data = json.loads(rec.read_text())
- except (OSError, json.JSONDecodeError):
- continue
- out.append(Run(key=_key_of(rec, root), label=str(rec.parent.relative_to(root)),
- path=rec, record=data))
- return out
-
-
-def _key_of(rec: Path, root: Path) -> str:
- """A stable prompt id: the cell/run directory name, not the whole path."""
- parts = rec.relative_to(root).parts
- for anchor in ("cells", "runs"):
- if anchor in parts:
- return parts[parts.index(anchor) + 1]
- return parts[0] if parts else rec.parent.name
+ return [Run(key=prompt_of(rec, root), label=str(rec.parent.relative_to(root)), path=rec, record=data)
+ for rec, data in records(root)]
def ab_arms(root: Path) -> dict[str, Path] | None:
diff --git a/eval/tests/test_session_stats.py b/eval/tests/test_session_stats.py
index ec199d90..a0f5a7f1 100644
--- a/eval/tests/test_session_stats.py
+++ b/eval/tests/test_session_stats.py
@@ -148,7 +148,8 @@ def test_the_coupling_survey_counts_runs_not_copies_of_the_same_urdf(tmp_path: P
deliverable/ (finalised) — and `bench run` lays cells out as runs/<id>/, not
cells/<id>/. Counting files tripled every mechanical number and named every prompt
"artifacts"."""
- from bench.coupling_stats import _prompt_of, _urdfs
+ from bench._records import prompt_of
+ from bench.coupling_stats import _urdfs
urdf = """<?xml version="1.0"?>
<robot name="rig"><link name="base"/><link name="a"/>
@@ -163,7 +164,7 @@ def test_the_coupling_survey_counts_runs_not_copies_of_the_same_urdf(tmp_path: P
found = _urdfs(tmp_path)
assert [p.parent.name for p in found] == ["artifacts"], "one URDF per run, the built one"
- assert _prompt_of(found[0]) == "cpl_umbrella"
+ assert prompt_of(found[0]) == "cpl_umbrella"
def _sweep(pair: str, depth_m: float, *, poses: list[dict], worst_pose: dict) -> dict: