Official Codebase of 3DCodeVerse
Python
readers: a runs root is walked once per run (symlinked cells found), a refine row is named by its RunId, and a scene render uses the in-run cameras
gaoypeng481a8ab1749a
diff --git a/harness/codeverse3d/addons/costreport/audit.py b/harness/codeverse3d/addons/costreport/audit.pyindex 724fad2b..eaf083f0 100644--- a/harness/codeverse3d/addons/costreport/audit.py+++ b/harness/codeverse3d/addons/costreport/audit.py@@ -28,7 +28,13 @@ from codeverse3d.contracts.run import RunIdfrom codeverse3d.cost.ledger import load_ledger, price_call, summarisefrom codeverse3d.cost.types import CallCost, Stage, Summaryfrom codeverse3d.proc import read_json_or_none-from codeverse3d.record.record import SUBRUN_DIRS, RecordError, load_record+from codeverse3d.record.record import (+ RecordError,+ find_run_dirs,+ is_run_dir,+ load_record,+ stop_reason,+)@dataclass@@ -83,7 +89,7 @@ def read_run(run_dir: Path, *, name: str = "", recheck: bool = False) -> RunLedgrec = Noneif rec is not None:led.track, led.language, led.status = rec.spec.track.value, rec.spec.language.value, rec.status.value- led.stop_reason = str(rec.extra.get("stop_reason") or "")+ led.stop_reason = stop_reason(rec)led.n_rounds, led.minutes = len(rec.rounds), rec.minutesled.round_built = [None if r.build is None else r.build.ok for r in rec.rounds]cands = read_json_or_none(run_dir / "rounds" / "candidates.json") or rec.extra.get("candidates") or {}@@ -114,18 +120,16 @@ def read_cell(cell_dir: Path, *, name: str, recheck: bool = False) -> RunLedger:def find_runs(root: str | Path) -> list[Path]:"""Every run directory under ``root`` (a dir with ``record.json``) and every compare cell(a dir with ``cell.json``; one that wraps a harness run is named by the cell, not its- ``run/``). Sub-workspaces (``_assets/``, ``_cand/``) are part of their run."""+ ``run/``), found by ``record.find_run_dirs``: once each — a cell another battery symlinks in+ included — and never inside a run (its ``_assets/``, ``_cand/`` sub-workspaces are part of it)."""base = Path(root)- if (base / "record.json").is_file() or (base / "cell.json").is_file():+ if _is_run_or_cell(base):return [base]- out: set[Path] = set()- for marker in ("record.json", "cell.json"):- for path in base.rglob(marker):- d = path.parent- if SUBRUN_DIRS & set(d.relative_to(base).parts):- continue- out.add(d.parent if d.name == "run" and (d.parent / "cell.json").is_file() else d)- return sorted(out)+ return find_run_dirs(base, predicate=_is_run_or_cell)+++def _is_run_or_cell(d: Path) -> bool:+ return is_run_dir(d) or (d / "cell.json").is_file()#: dimensions the audit always aggregates ondiff --git a/harness/codeverse3d/addons/dataset/refine.py b/harness/codeverse3d/addons/dataset/refine.pyindex e0194732..c2ef87f9 100644--- a/harness/codeverse3d/addons/dataset/refine.py+++ b/harness/codeverse3d/addons/dataset/refine.py@@ -36,12 +36,7 @@ from codeverse3d.record.git_history import (diff_between,read_tree_at,)-from codeverse3d.record.record import (- BATTERY_MARKERS,- effective_judgment,- effective_score,- iter_runs,-)+from codeverse3d.record.record import effective_judgment, effective_score, iter_runsfrom codeverse3d.workspace import Workspace#: rounds that answer a judgment. A texture or asset round is driven by its own pass, and@@ -136,35 +131,15 @@ def _transition(ws: Workspace, rec: RunRecord, prev: RoundRecord, cur: RoundRecooutcome=outcome_of(delta, threshold), before_files=files_before, after_files=files_after)-#: path segments that are run LAYOUT, not a battery name — the same markers ``iter_runs``-#: discovers batteries by-_LAYOUT_DIRS = frozenset(BATTERY_MARKERS)---def _identity(root: Path, found: Any) -> tuple[str, str]:- """``(battery, run)`` for a run, taken from where it PHYSICALLY lives.-- ``eval/bench/out``'s batteries symlink each other's cells (54 of 239 runs are reachable- twice), so labelling by the path a scan happened to reach first credits a run to a- battery it never ran in."""- try:- rel = found.ws.root.resolve().relative_to(root.resolve()).as_posix()- except ValueError:- return found.run_id.battery, found.run_id.rel- head, _, rest = rel.partition("/")- if not rest or head in _LAYOUT_DIRS:- return found.run_id.battery, rel- return head, rest--def transitions(runs_dir: Path | str, *, threshold: float = MIN_PREFERENCE_DELTA,max_diff_bytes: int = MAX_DIFF_BYTES, with_code: bool = False,drops: Counter[str] | None = None) -> Iterator[RefineTransition]:"""Every exportable refine transition under ``runs_dir``; every one that is not is counted.A run reachable through more than one path (54 of 239 under ``eval/bench/out``, where the- batteries symlink each other's cells) is found once (``find_run_dirs``) and exported under- the battery it physically lives in."""+ batteries symlink each other's cells) is found once (``find_run_dirs``), at the path it+ physically lives at. A row is named by its ``RunId`` (``battery``, ``run`` = the slug), the+ key of ``index.sqlite`` and of the exported samples."""drops = drops if drops is not None else Counter()root = Path(runs_dir)for found in iter_runs(root, on_error=lambda p, e: drops.update([f"unreadable_record: {type(e).__name__}"])):@@ -183,8 +158,7 @@ def transitions(runs_dir: Path | str, *, threshold: float = MIN_PREFERENCE_DELTAdrops["predecessor_unjudged"] += 1else:try:- battery, run = _identity(root, found)- yield _transition(found.ws, found.record, prev, cur, battery=battery, run=run,+ yield _transition(found.ws, found.record, prev, cur, battery=found.run_id.battery, run=found.run_id.slug,threshold=threshold, max_diff_bytes=max_diff_bytes, with_code=with_code)except GitReadError as e:drops[f"git_read_failed: {str(e)[:60]}"] += 1diff --git a/harness/codeverse3d/addons/dataset/sample.py b/harness/codeverse3d/addons/dataset/sample.pyindex 12cacff3..cf52c614 100644--- a/harness/codeverse3d/addons/dataset/sample.py+++ b/harness/codeverse3d/addons/dataset/sample.py@@ -39,10 +39,7 @@ from codeverse3d.contracts.common import (from codeverse3d.contracts.run import RoundRecord, RunRecordfrom codeverse3d.record import git_historyfrom codeverse3d.record.deliverable import deliverable_path, round_outputs, texture_shipped_for-from codeverse3d.record.record import (- effective_judgment,- round_summary,-)+from codeverse3d.record.record import effective_judgment, round_summary, stop_reasonfrom codeverse3d.workspace import WorkspaceMAX_GLB_BYTES = 20 * 1024 * 1024@@ -341,7 +338,7 @@ def build_meta(rounds=len(record.rounds),round=rnd.index if rnd is not None else None,rounds_summary=[round_summary(r) for r in record.rounds],- stop_reason=str(record.extra.get("stop_reason", "") or ""),+ stop_reason=stop_reason(record),code_commit=rnd.commit if rnd is not None else "",code_source=code_source,code_fingerprint=code_fingerprint(code) if code else "",diff --git a/harness/codeverse3d/addons/select.py b/harness/codeverse3d/addons/select.pyindex 743347f1..ea558606 100644--- a/harness/codeverse3d/addons/select.py+++ b/harness/codeverse3d/addons/select.py@@ -48,6 +48,7 @@ from codeverse3d.record.record import (repackage,round_complexity,rubric_of,+ stop_reason,)from codeverse3d.workspace import Workspace@@ -239,7 +240,7 @@ def summarise(run_dir: Path | str, *, record: RunRecord | None = None) -> RunSumpicked, method = pick(run_dir, record=rec), "score"baseline = by_index[0].score if 0 in by_index else Nonescore = by_index[picked].score if picked is not None else None- return RunSummary(rounds=len(rows), stop_reason=str(rec.extra.get("stop_reason") or rec.status.value),+ return RunSummary(rounds=len(rows), stop_reason=stop_reason(rec),baseline_score=baseline, picked_round=picked, picked_score=score, method=method,delta=round(score - baseline, 6) if score is not None and baseline is not None else None)diff --git a/harness/codeverse3d/cli/inspect_cmd.py b/harness/codeverse3d/cli/inspect_cmd.pyindex a04ec949..6fa94f32 100644--- a/harness/codeverse3d/cli/inspect_cmd.py+++ b/harness/codeverse3d/cli/inspect_cmd.py@@ -20,6 +20,7 @@ from codeverse3d.cli._common import RunsDirOpt, console, kv_tablefrom codeverse3d.config import get_settingsfrom codeverse3d.contracts.artifacts import RENDER_MODESfrom codeverse3d.contracts.common import Track+from codeverse3d.contracts.plan import ScenePlanfrom codeverse3d.contracts.spec import Spec@@ -64,9 +65,14 @@ def render(r = get_settings().renderif spec.track is Track.SCENE:from codeverse3d.spatial.render_scene import render_scene-- rs = render_scene(ws, out_dir, cameras=None, width=width or r.scene_width,- height=height or r.scene_height)+ from codeverse3d.spatial.tool_common import load_plan+ from codeverse3d.tracks.scene import SCENE_TIMES++ # the in-run rig (tracks/scene.ScenePipeline.render): the plan's cameras at the judged+ # times — not the cameras createScene() authored, which the judge never saw on their own+ plan = load_plan(ws.plan_path) if ws.plan_path.is_file() else None+ rs = render_scene(ws, out_dir, cameras=list(plan.cameras) if isinstance(plan, ScenePlan) else None,+ times=SCENE_TIMES, width=width or r.scene_width, height=height or r.scene_height)else:glb = ws.artifacts / "object.glb"if not glb.is_file():diff --git a/harness/codeverse3d/cli/skills_cmd.py b/harness/codeverse3d/cli/skills_cmd.pyindex 575e237c..7fc19bfb 100644--- a/harness/codeverse3d/cli/skills_cmd.py+++ b/harness/codeverse3d/cli/skills_cmd.py@@ -131,17 +131,23 @@ def report("""from rich.table import Table+ from codeverse3d.contracts.run import SkillsUsage+ from codeverse3d.record.record import unique_files+ from codeverse3d.skills.telemetry import SKILLS_FILE, TELEMETRY_DIR+rows: list[dict] = []- for p in sorted(Path(runs_dir).rglob("telemetry/skills.jsonl")):- rows.extend(read_jsonl_lenient(p))+ for p in unique_files(runs_dir, SKILLS_FILE, subruns=True): # a symlinked cell once; candidates' sessions too+ if p.parent.name == TELEMETRY_DIR:+ rows.extend(read_jsonl_lenient(p))if not rows:err_console.print(f"[yellow]no telemetry/skills.jsonl under {runs_dir}[/] (was C3D_SKILLS=0?)")raise typer.Exit(code=1)per: dict[tuple[str, str], list[int]] = {}- def _blind(r: dict) -> bool: # an atime row whose control fired saw nothing- return bool(r.get("control_read")) and r.get("evidence", "atime") != "transcript"+ def _blind(r: dict) -> bool: # the probe saw nothing: SkillsUsage.deep_read_rate says None+ u = SkillsUsage.model_validate(r)+ return bool(u.listed) and u.deep_read_rate is Noneblind = sum(1 for r in rows if _blind(r))exact = sum(1 for r in rows if r.get("evidence") == "transcript")diff --git a/harness/codeverse3d/record/record.py b/harness/codeverse3d/record/record.pyindex 52545e1f..b2788291 100644--- a/harness/codeverse3d/record/record.py+++ b/harness/codeverse3d/record/record.py@@ -141,6 +141,11 @@ def rubric_of(record: RunRecord) -> str:return str(record.extra.get("rubric") or "") or next(judged, "") or run_rubric(record.spec)+def stop_reason(record: RunRecord) -> str:+ """Why the run stopped: ``extra["stop_reason"]`` (what the loop stamped), else its status."""+ return str(record.extra.get("stop_reason") or record.status.value)++def effective_score(r: RoundRecord) -> float | None:"""``r.score`` with degraded judgments filtered out (None = unscored)."""j = effective_judgment(r)@@ -301,23 +306,23 @@ RUN_SEARCH_DEPTH = 7BATTERY_MARKERS = ("runs", "cells", "arms")#: run-layout directories that hold a SUB-workspace (a scene asset candidate, a rejected#: best-of-N candidate): their files belong to that sub-run, not to the run above them.-#: ``addons.costreport.audit.find_runs`` and the bench survey scripts skip them by this one name.+#: :func:`unique_files` and the bench survey scripts skip them by this one name.SUBRUN_DIRS: frozenset[str] = frozenset({"_cand", "_assets"})-def unique_files(root: Path | str, name: str) -> list[Path]:+def unique_files(root: Path | str, name: str, *, subruns: bool = False) -> list[Path]:"""Every file called ``name`` under ``root``, once per file on disk.Battery trees symlink each other's cells and every run has``run/telemetry/trajectories -> run/trajectories``, so a walk must say what it doesabout symlinks: this one follows them (``recurse_symlinks=True``) and collapses thetwo paths of one file by its resolved path. Sub-workspaces (:data:`SUBRUN_DIRS`) are- skipped. The one walker for every survey that counts sessions, records or artefacts —+ skipped unless ``subruns`` (a best-of-N candidate's session is a session). The one walker for every survey that counts sessions, records or artefacts —each hand-rolled copy of it has at some point counted a file twice."""seen: dict[Path, Path] = {}root = Path(root)for p in sorted(root.rglob(name, recurse_symlinks=True)):- if SUBRUN_DIRS & set(p.relative_to(root).parts):+ if not subruns and SUBRUN_DIRS & set(p.relative_to(root).parts):continueseen.setdefault(p.resolve(), p)return sorted(seen.values())diff --git a/harness/tests/cost/test_audit_read.py b/harness/tests/cost/test_audit_read.pyindex 52aa8c46..a9d76c03 100644--- a/harness/tests/cost/test_audit_read.py+++ b/harness/tests/cost/test_audit_read.py@@ -26,15 +26,25 @@ def test_a_run_reads_as_its_ledger_rows_and_its_record_facts(fake_run: Path, tmpshutil.copytree(fake_run, bare)shutil.rmtree(bare / "telemetry")assert audit_runs([tmp_path]).n_runs == 1+ # a record with no extra["stop_reason"] stops where its status says (it read blank, N60)+ rec = json.loads((bare / "record.json").read_text())+ rec["extra"].pop("stop_reason")+ (bare / "record.json").write_text(json.dumps(rec))+ assert read_run(bare).stop_reason == "max_rounds"-def test_find_runs_skips_sub_workspaces(tmp_path: Path):- (tmp_path / "a").mkdir()- (tmp_path / "a" / "record.json").write_text("{}")- sub = tmp_path / "a" / "_assets" / "x"- sub.mkdir(parents=True)- (sub / "record.json").write_text("{}")- assert find_runs(tmp_path) == [tmp_path / "a"]+def test_find_runs_counts_each_run_once_and_never_one_inside_a_run(tmp_path: Path):+ """N58: `record.find_run_dirs` — a sub-workspace or a directory nested in a run is part of that run,+ and a cell another battery symlinks in is found (a bare rglob missed it)."""+ bat = tmp_path / "bat"+ for d in (bat / "runs" / "a", bat / "runs" / "a" / "_assets" / "x", bat / "runs" / "a" / "eval" / "inner",+ tmp_path / "other" / "cells" / "c" / "run"):+ d.mkdir(parents=True)+ (d / "record.json").write_text("{}")+ (tmp_path / "other" / "cells" / "c" / "cell.json").write_text("{}")+ (bat / "cells").mkdir()+ (bat / "cells" / "c").symlink_to(tmp_path / "other" / "cells" / "c")+ assert find_runs(bat) == [bat / "cells" / "c", bat / "runs" / "a"]def test_an_ab_batterys_control_and_variant_cells_stay_two_runs(fake_run: Path, tmp_path: Path):diff --git a/harness/tests/flywheel_cli/conftest.py b/harness/tests/flywheel_cli/conftest.pyindex 3410ae6e..3960e591 100644--- a/harness/tests/flywheel_cli/conftest.py+++ b/harness/tests/flywheel_cli/conftest.py@@ -22,7 +22,7 @@ from codeverse3d.contracts.artifacts import (RenderView,Severity,)-from codeverse3d.contracts.common import Backends, Language, Track, Usage+from codeverse3d.contracts.common import ENTRY_FILE, TRACK_LANGUAGES, Backends, Language, Usagefrom codeverse3d.contracts.run import RoundRecord, RunRecord, RunStatus, StepTimefrom codeverse3d.contracts.spec import Specfrom codeverse3d.cost.ledger import record_call@@ -59,15 +59,10 @@ def make_fake_run(scores: tuple[float, float] = (0.55, 0.80), with_repair: bool = False,) -> tuple[Workspace, RunRecord]:ws = Workspace(runs_dir / slug).create()- track = Track.SCENE if language is Language.SCENE_THREEJS else (- Track.ARTICULATED_OBJECT if language is Language.URDF_BLENDER else (- Track.GRAPHICS if language in (Language.GLSL_SHADER, Language.OPENGL_PYTHON) else Track.STATIC_OBJECT))+ track = next(t for t, languages in TRACK_LANGUAGES.items() if language in languages)spec = Spec(id=slug, track=track, language=language, prompt=prompt, backends=Backends(generator=generator))ws.write_json(ws.spec_path, spec)- entry = {"blender": "src/model.py", "cadquery": "src/model.py", "urdf_blender": "src/model.py",- "threejs": "src/object.js", "scene_threejs": "src/scene.js",- "glsl_shader": "src/shader.frag", "opengl_python": "src/program.py"}[language.value]- e = ws.root / entry+ e = ws.root / ENTRY_FILE[language]e.parent.mkdir(parents=True, exist_ok=True)e.write_text("# round 0\nimport bpy # comment\n\nbpy.ops.mesh.primitive_cube_add(size=1.0)\n")if language is Language.URDF_BLENDER:diff --git a/harness/tests/flywheel_cli/test_cli.py b/harness/tests/flywheel_cli/test_cli.pyindex 181f44e3..49a4d664 100644--- a/harness/tests/flywheel_cli/test_cli.py+++ b/harness/tests/flywheel_cli/test_cli.py@@ -637,3 +637,25 @@ def test_a_replay_judges_the_picked_round_on_the_runs_rubric_and_books_into_the_assert load_record(ws).total_usage.cost_usd == pytest.approx(ledger_usage(load_ledger(ws.root)).cost_usd)assert load_record(ws).total_usage.cost_usd == pytest.approx(total + 3 * 0.05)++def test_render_of_a_scene_uses_the_in_run_cameras_and_times(tmp_path: Path, monkeypatch):+ """N69: `3dcode render` on a scene renders the plan's cameras at SCENE_TIMES — what the in-run judge+ saw (tracks/scene.ScenePipeline.render) — not the cameras createScene() authored."""+ import codeverse3d.spatial.render_scene as RS+ from codeverse3d.contracts.artifacts import RenderSet+ from codeverse3d.contracts.plan import BBox, CameraPlan, ScenePlan, ZonePlan+ from codeverse3d.tracks.scene import SCENE_TIMES+ from codeverse3d.workspace import Workspace++ runs = tmp_path / "runs"+ assert runner.invoke(app, ["make", "x", "--track", "scene", "--no-run", "--runs-dir", str(runs), "--slug", "s"]).exit_code == 0+ ws = Workspace(runs / "s")+ cam = CameraPlan(name="hero", position=(4, 2, 4), look_at=(0, 0, 0))+ ws.write_json(ws.plan_path, ScenePlan(title="t", summary="s", setting="x", environment="e", bounds=BBox(center=(0, 0, 0), extents=(10, 4, 10)),+ zones=[ZonePlan(name="z", description="d", bbox=BBox(center=(0, 0, 0), extents=(1, 1, 1)))],+ cameras=[cam]))+ seen: dict = {}+ monkeypatch.setattr(RS, "render_scene", lambda ws, out_dir, **kw: seen.update(kw) or RenderSet(views=[], renderer="fake"))+ r = runner.invoke(app, ["render", "s", "--runs-dir", str(runs)])+ assert r.exit_code == 0, r.output+ assert [c.name for c in seen["cameras"]] == ["hero"] and tuple(seen["times"]) == SCENE_TIMESdiff --git a/harness/tests/flywheel_cli/test_refine.py b/harness/tests/flywheel_cli/test_refine.pyindex 5318820d..bdc02f39 100644--- a/harness/tests/flywheel_cli/test_refine.py+++ b/harness/tests/flywheel_cli/test_refine.py@@ -39,7 +39,8 @@ def test_a_refine_round_carries_its_brief_its_diff_and_its_delta(tmp_path):rows, drops = _rows(tmp_path)assert len(rows) == 1 and not dropst = rows[0]- assert t.round == 1 and t.round_kind == "refine" and t.battery == "batt"+ assert t.round == 1 and t.round_kind == "refine"+ assert (t.battery, t.run) == (tmp_path.name, "batt__chair") # its RunId: the key index.sqlite joins on (N59)assert t.score_before == 0.30 and t.score_after == 0.80 and t.score_delta == 0.50 and t.outcome == "improved"assert t.instructions == ["thicken the legs"]assert [g["message"] for g in t.gate_errors] == ["part 'Leg' is floating"] # ERRORs onlydiff --git a/harness/tests/skills/test_telemetry.py b/harness/tests/skills/test_telemetry.pyindex 78ef7e4b..76271b36 100644--- a/harness/tests/skills/test_telemetry.py+++ b/harness/tests/skills/test_telemetry.py@@ -192,7 +192,8 @@ def test_the_report_counts_exact_sessions_and_drops_only_blind_ones(tmp_path):from codeverse3d.cli.main import app- tele = tmp_path / "run" / TELEMETRY_DIR+ root = tmp_path / "bat"+ tele = root / "run" / TELEMETRY_DIRtele.mkdir(parents=True)rows = [ # exact, the agent opened the control too: counted, and said so{"agent": "gemini-cli:gemini-3.7-flash", "listed": ["c3d-a"], "evidence": "transcript",@@ -202,9 +203,15 @@ def test_the_report_counts_exact_sessions_and_drops_only_blind_ones(tmp_path):"reads": [{"name": "c3d-a", "surfaced": True, "deep": True}]},](tele / SKILLS_FILE).write_text("".join(json.dumps(r) + "\n" for r in rows))- r = CliRunner().invoke(app, ["skills", "report", str(tmp_path), "--json"])+ # N58: a best-of-N candidate's session counts, and a cell another battery symlinks in is read, once+ for d in (root / "run" / "_cand" / "c0", tmp_path / "other" / "cell"):+ (d / TELEMETRY_DIR).mkdir(parents=True)+ (d / TELEMETRY_DIR / SKILLS_FILE).write_text(json.dumps(rows[0]) + "\n")+ (root / "cell").symlink_to(tmp_path / "other" / "cell")+ (root / "alias").symlink_to(tmp_path / "other" / "cell")+ r = CliRunner().invoke(app, ["skills", "report", str(root), "--json"])assert r.exit_code == 0, r.outputout = json.loads(r.output)- assert out["transcript_sessions"] == 1 and out["control_opened_by_agent_sessions"] == 1+ assert out["transcript_sessions"] == 3 and out["control_opened_by_agent_sessions"] == 3assert out["control_read_sessions"] == 1- assert [(s["backend"], s["deep"]) for s in out["skills"]] == [("gemini-cli", 1)]+ assert [(s["backend"], s["deep"]) for s in out["skills"]] == [("gemini-cli", 3)]