Skip to content

Official Codebase of 3DCodeVerse

Python

readers: a runs root is walked once per run (symlinked cells found), a refine row is named by its RunId, and a scene render uses the in-run cameras

gaoypeng481a8ab1749a
diff --git a/harness/codeverse3d/addons/costreport/audit.py b/harness/codeverse3d/addons/costreport/audit.py
index 724fad2b..eaf083f0 100644
--- a/harness/codeverse3d/addons/costreport/audit.py
+++ b/harness/codeverse3d/addons/costreport/audit.py
@@ -28,7 +28,13 @@ from codeverse3d.contracts.run import RunId
from codeverse3d.cost.ledger import load_ledger, price_call, summarise
from codeverse3d.cost.types import CallCost, Stage, Summary
from codeverse3d.proc import read_json_or_none
-from codeverse3d.record.record import SUBRUN_DIRS, RecordError, load_record
+from codeverse3d.record.record import (
+ RecordError,
+ find_run_dirs,
+ is_run_dir,
+ load_record,
+ stop_reason,
+)
@dataclass
@@ -83,7 +89,7 @@ def read_run(run_dir: Path, *, name: str = "", recheck: bool = False) -> RunLedg
rec = None
if rec is not None:
led.track, led.language, led.status = rec.spec.track.value, rec.spec.language.value, rec.status.value
- led.stop_reason = str(rec.extra.get("stop_reason") or "")
+ led.stop_reason = stop_reason(rec)
led.n_rounds, led.minutes = len(rec.rounds), rec.minutes
led.round_built = [None if r.build is None else r.build.ok for r in rec.rounds]
cands = read_json_or_none(run_dir / "rounds" / "candidates.json") or rec.extra.get("candidates") or {}
@@ -114,18 +120,16 @@ def read_cell(cell_dir: Path, *, name: str, recheck: bool = False) -> RunLedger:
def find_runs(root: str | Path) -> list[Path]:
"""Every run directory under ``root`` (a dir with ``record.json``) and every compare cell
(a dir with ``cell.json``; one that wraps a harness run is named by the cell, not its
- ``run/``). Sub-workspaces (``_assets/``, ``_cand/``) are part of their run."""
+ ``run/``), found by ``record.find_run_dirs``: once each — a cell another battery symlinks in
+ included — and never inside a run (its ``_assets/``, ``_cand/`` sub-workspaces are part of it)."""
base = Path(root)
- if (base / "record.json").is_file() or (base / "cell.json").is_file():
+ if _is_run_or_cell(base):
return [base]
- out: set[Path] = set()
- for marker in ("record.json", "cell.json"):
- for path in base.rglob(marker):
- d = path.parent
- if SUBRUN_DIRS & set(d.relative_to(base).parts):
- continue
- out.add(d.parent if d.name == "run" and (d.parent / "cell.json").is_file() else d)
- return sorted(out)
+ return find_run_dirs(base, predicate=_is_run_or_cell)
+
+
+def _is_run_or_cell(d: Path) -> bool:
+ return is_run_dir(d) or (d / "cell.json").is_file()
#: dimensions the audit always aggregates on
diff --git a/harness/codeverse3d/addons/dataset/refine.py b/harness/codeverse3d/addons/dataset/refine.py
index e0194732..c2ef87f9 100644
--- a/harness/codeverse3d/addons/dataset/refine.py
+++ b/harness/codeverse3d/addons/dataset/refine.py
@@ -36,12 +36,7 @@ from codeverse3d.record.git_history import (
diff_between,
read_tree_at,
)
-from codeverse3d.record.record import (
- BATTERY_MARKERS,
- effective_judgment,
- effective_score,
- iter_runs,
-)
+from codeverse3d.record.record import effective_judgment, effective_score, iter_runs
from codeverse3d.workspace import Workspace
#: rounds that answer a judgment. A texture or asset round is driven by its own pass, and
@@ -136,35 +131,15 @@ def _transition(ws: Workspace, rec: RunRecord, prev: RoundRecord, cur: RoundReco
outcome=outcome_of(delta, threshold), before_files=files_before, after_files=files_after)
-#: path segments that are run LAYOUT, not a battery name — the same markers ``iter_runs``
-#: discovers batteries by
-_LAYOUT_DIRS = frozenset(BATTERY_MARKERS)
-
-
-def _identity(root: Path, found: Any) -> tuple[str, str]:
- """``(battery, run)`` for a run, taken from where it PHYSICALLY lives.
-
- ``eval/bench/out``'s batteries symlink each other's cells (54 of 239 runs are reachable
- twice), so labelling by the path a scan happened to reach first credits a run to a
- battery it never ran in."""
- try:
- rel = found.ws.root.resolve().relative_to(root.resolve()).as_posix()
- except ValueError:
- return found.run_id.battery, found.run_id.rel
- head, _, rest = rel.partition("/")
- if not rest or head in _LAYOUT_DIRS:
- return found.run_id.battery, rel
- return head, rest
-
-
def transitions(runs_dir: Path | str, *, threshold: float = MIN_PREFERENCE_DELTA,
max_diff_bytes: int = MAX_DIFF_BYTES, with_code: bool = False,
drops: Counter[str] | None = None) -> Iterator[RefineTransition]:
"""Every exportable refine transition under ``runs_dir``; every one that is not is counted.
A run reachable through more than one path (54 of 239 under ``eval/bench/out``, where the
- batteries symlink each other's cells) is found once (``find_run_dirs``) and exported under
- the battery it physically lives in."""
+ batteries symlink each other's cells) is found once (``find_run_dirs``), at the path it
+ physically lives at. A row is named by its ``RunId`` (``battery``, ``run`` = the slug), the
+ key of ``index.sqlite`` and of the exported samples."""
drops = drops if drops is not None else Counter()
root = Path(runs_dir)
for found in iter_runs(root, on_error=lambda p, e: drops.update([f"unreadable_record: {type(e).__name__}"])):
@@ -183,8 +158,7 @@ def transitions(runs_dir: Path | str, *, threshold: float = MIN_PREFERENCE_DELTA
drops["predecessor_unjudged"] += 1
else:
try:
- battery, run = _identity(root, found)
- yield _transition(found.ws, found.record, prev, cur, battery=battery, run=run,
+ yield _transition(found.ws, found.record, prev, cur, battery=found.run_id.battery, run=found.run_id.slug,
threshold=threshold, max_diff_bytes=max_diff_bytes, with_code=with_code)
except GitReadError as e:
drops[f"git_read_failed: {str(e)[:60]}"] += 1
diff --git a/harness/codeverse3d/addons/dataset/sample.py b/harness/codeverse3d/addons/dataset/sample.py
index 12cacff3..cf52c614 100644
--- a/harness/codeverse3d/addons/dataset/sample.py
+++ b/harness/codeverse3d/addons/dataset/sample.py
@@ -39,10 +39,7 @@ from codeverse3d.contracts.common import (
from codeverse3d.contracts.run import RoundRecord, RunRecord
from codeverse3d.record import git_history
from codeverse3d.record.deliverable import deliverable_path, round_outputs, texture_shipped_for
-from codeverse3d.record.record import (
- effective_judgment,
- round_summary,
-)
+from codeverse3d.record.record import effective_judgment, round_summary, stop_reason
from codeverse3d.workspace import Workspace
MAX_GLB_BYTES = 20 * 1024 * 1024
@@ -341,7 +338,7 @@ def build_meta(
rounds=len(record.rounds),
round=rnd.index if rnd is not None else None,
rounds_summary=[round_summary(r) for r in record.rounds],
- stop_reason=str(record.extra.get("stop_reason", "") or ""),
+ stop_reason=stop_reason(record),
code_commit=rnd.commit if rnd is not None else "",
code_source=code_source,
code_fingerprint=code_fingerprint(code) if code else "",
diff --git a/harness/codeverse3d/addons/select.py b/harness/codeverse3d/addons/select.py
index 743347f1..ea558606 100644
--- a/harness/codeverse3d/addons/select.py
+++ b/harness/codeverse3d/addons/select.py
@@ -48,6 +48,7 @@ from codeverse3d.record.record import (
repackage,
round_complexity,
rubric_of,
+ stop_reason,
)
from codeverse3d.workspace import Workspace
@@ -239,7 +240,7 @@ def summarise(run_dir: Path | str, *, record: RunRecord | None = None) -> RunSum
picked, method = pick(run_dir, record=rec), "score"
baseline = by_index[0].score if 0 in by_index else None
score = by_index[picked].score if picked is not None else None
- return RunSummary(rounds=len(rows), stop_reason=str(rec.extra.get("stop_reason") or rec.status.value),
+ return RunSummary(rounds=len(rows), stop_reason=stop_reason(rec),
baseline_score=baseline, picked_round=picked, picked_score=score, method=method,
delta=round(score - baseline, 6) if score is not None and baseline is not None else None)
diff --git a/harness/codeverse3d/cli/inspect_cmd.py b/harness/codeverse3d/cli/inspect_cmd.py
index a04ec949..6fa94f32 100644
--- a/harness/codeverse3d/cli/inspect_cmd.py
+++ b/harness/codeverse3d/cli/inspect_cmd.py
@@ -20,6 +20,7 @@ from codeverse3d.cli._common import RunsDirOpt, console, kv_table
from codeverse3d.config import get_settings
from codeverse3d.contracts.artifacts import RENDER_MODES
from codeverse3d.contracts.common import Track
+from codeverse3d.contracts.plan import ScenePlan
from codeverse3d.contracts.spec import Spec
@@ -64,9 +65,14 @@ def render(
r = get_settings().render
if spec.track is Track.SCENE:
from codeverse3d.spatial.render_scene import render_scene
-
- rs = render_scene(ws, out_dir, cameras=None, width=width or r.scene_width,
- height=height or r.scene_height)
+ from codeverse3d.spatial.tool_common import load_plan
+ from codeverse3d.tracks.scene import SCENE_TIMES
+
+ # the in-run rig (tracks/scene.ScenePipeline.render): the plan's cameras at the judged
+ # times — not the cameras createScene() authored, which the judge never saw on their own
+ plan = load_plan(ws.plan_path) if ws.plan_path.is_file() else None
+ rs = render_scene(ws, out_dir, cameras=list(plan.cameras) if isinstance(plan, ScenePlan) else None,
+ times=SCENE_TIMES, width=width or r.scene_width, height=height or r.scene_height)
else:
glb = ws.artifacts / "object.glb"
if not glb.is_file():
diff --git a/harness/codeverse3d/cli/skills_cmd.py b/harness/codeverse3d/cli/skills_cmd.py
index 575e237c..7fc19bfb 100644
--- a/harness/codeverse3d/cli/skills_cmd.py
+++ b/harness/codeverse3d/cli/skills_cmd.py
@@ -131,17 +131,23 @@ def report(
"""
from rich.table import Table
+ from codeverse3d.contracts.run import SkillsUsage
+ from codeverse3d.record.record import unique_files
+ from codeverse3d.skills.telemetry import SKILLS_FILE, TELEMETRY_DIR
+
rows: list[dict] = []
- for p in sorted(Path(runs_dir).rglob("telemetry/skills.jsonl")):
- rows.extend(read_jsonl_lenient(p))
+ for p in unique_files(runs_dir, SKILLS_FILE, subruns=True): # a symlinked cell once; candidates' sessions too
+ if p.parent.name == TELEMETRY_DIR:
+ rows.extend(read_jsonl_lenient(p))
if not rows:
err_console.print(f"[yellow]no telemetry/skills.jsonl under {runs_dir}[/] (was C3D_SKILLS=0?)")
raise typer.Exit(code=1)
per: dict[tuple[str, str], list[int]] = {}
- def _blind(r: dict) -> bool: # an atime row whose control fired saw nothing
- return bool(r.get("control_read")) and r.get("evidence", "atime") != "transcript"
+ def _blind(r: dict) -> bool: # the probe saw nothing: SkillsUsage.deep_read_rate says None
+ u = SkillsUsage.model_validate(r)
+ return bool(u.listed) and u.deep_read_rate is None
blind = sum(1 for r in rows if _blind(r))
exact = sum(1 for r in rows if r.get("evidence") == "transcript")
diff --git a/harness/codeverse3d/record/record.py b/harness/codeverse3d/record/record.py
index 52545e1f..b2788291 100644
--- a/harness/codeverse3d/record/record.py
+++ b/harness/codeverse3d/record/record.py
@@ -141,6 +141,11 @@ def rubric_of(record: RunRecord) -> str:
return str(record.extra.get("rubric") or "") or next(judged, "") or run_rubric(record.spec)
+def stop_reason(record: RunRecord) -> str:
+ """Why the run stopped: ``extra["stop_reason"]`` (what the loop stamped), else its status."""
+ return str(record.extra.get("stop_reason") or record.status.value)
+
+
def effective_score(r: RoundRecord) -> float | None:
"""``r.score`` with degraded judgments filtered out (None = unscored)."""
j = effective_judgment(r)
@@ -301,23 +306,23 @@ RUN_SEARCH_DEPTH = 7
BATTERY_MARKERS = ("runs", "cells", "arms")
#: run-layout directories that hold a SUB-workspace (a scene asset candidate, a rejected
#: best-of-N candidate): their files belong to that sub-run, not to the run above them.
-#: ``addons.costreport.audit.find_runs`` and the bench survey scripts skip them by this one name.
+#: :func:`unique_files` and the bench survey scripts skip them by this one name.
SUBRUN_DIRS: frozenset[str] = frozenset({"_cand", "_assets"})
-def unique_files(root: Path | str, name: str) -> list[Path]:
+def unique_files(root: Path | str, name: str, *, subruns: bool = False) -> list[Path]:
"""Every file called ``name`` under ``root``, once per file on disk.
Battery trees symlink each other's cells and every run has
``run/telemetry/trajectories -> run/trajectories``, so a walk must say what it does
about symlinks: this one follows them (``recurse_symlinks=True``) and collapses the
two paths of one file by its resolved path. Sub-workspaces (:data:`SUBRUN_DIRS`) are
- skipped. The one walker for every survey that counts sessions, records or artefacts —
+ skipped unless ``subruns`` (a best-of-N candidate's session is a session). The one walker for every survey that counts sessions, records or artefacts —
each hand-rolled copy of it has at some point counted a file twice."""
seen: dict[Path, Path] = {}
root = Path(root)
for p in sorted(root.rglob(name, recurse_symlinks=True)):
- if SUBRUN_DIRS & set(p.relative_to(root).parts):
+ if not subruns and SUBRUN_DIRS & set(p.relative_to(root).parts):
continue
seen.setdefault(p.resolve(), p)
return sorted(seen.values())
diff --git a/harness/tests/cost/test_audit_read.py b/harness/tests/cost/test_audit_read.py
index 52aa8c46..a9d76c03 100644
--- a/harness/tests/cost/test_audit_read.py
+++ b/harness/tests/cost/test_audit_read.py
@@ -26,15 +26,25 @@ def test_a_run_reads_as_its_ledger_rows_and_its_record_facts(fake_run: Path, tmp
shutil.copytree(fake_run, bare)
shutil.rmtree(bare / "telemetry")
assert audit_runs([tmp_path]).n_runs == 1
+ # a record with no extra["stop_reason"] stops where its status says (it read blank, N60)
+ rec = json.loads((bare / "record.json").read_text())
+ rec["extra"].pop("stop_reason")
+ (bare / "record.json").write_text(json.dumps(rec))
+ assert read_run(bare).stop_reason == "max_rounds"
-def test_find_runs_skips_sub_workspaces(tmp_path: Path):
- (tmp_path / "a").mkdir()
- (tmp_path / "a" / "record.json").write_text("{}")
- sub = tmp_path / "a" / "_assets" / "x"
- sub.mkdir(parents=True)
- (sub / "record.json").write_text("{}")
- assert find_runs(tmp_path) == [tmp_path / "a"]
+def test_find_runs_counts_each_run_once_and_never_one_inside_a_run(tmp_path: Path):
+ """N58: `record.find_run_dirs` — a sub-workspace or a directory nested in a run is part of that run,
+ and a cell another battery symlinks in is found (a bare rglob missed it)."""
+ bat = tmp_path / "bat"
+ for d in (bat / "runs" / "a", bat / "runs" / "a" / "_assets" / "x", bat / "runs" / "a" / "eval" / "inner",
+ tmp_path / "other" / "cells" / "c" / "run"):
+ d.mkdir(parents=True)
+ (d / "record.json").write_text("{}")
+ (tmp_path / "other" / "cells" / "c" / "cell.json").write_text("{}")
+ (bat / "cells").mkdir()
+ (bat / "cells" / "c").symlink_to(tmp_path / "other" / "cells" / "c")
+ assert find_runs(bat) == [bat / "cells" / "c", bat / "runs" / "a"]
def test_an_ab_batterys_control_and_variant_cells_stay_two_runs(fake_run: Path, tmp_path: Path):
diff --git a/harness/tests/flywheel_cli/conftest.py b/harness/tests/flywheel_cli/conftest.py
index 3410ae6e..3960e591 100644
--- a/harness/tests/flywheel_cli/conftest.py
+++ b/harness/tests/flywheel_cli/conftest.py
@@ -22,7 +22,7 @@ from codeverse3d.contracts.artifacts import (
RenderView,
Severity,
)
-from codeverse3d.contracts.common import Backends, Language, Track, Usage
+from codeverse3d.contracts.common import ENTRY_FILE, TRACK_LANGUAGES, Backends, Language, Usage
from codeverse3d.contracts.run import RoundRecord, RunRecord, RunStatus, StepTime
from codeverse3d.contracts.spec import Spec
from codeverse3d.cost.ledger import record_call
@@ -59,15 +59,10 @@ def make_fake_run(
scores: tuple[float, float] = (0.55, 0.80), with_repair: bool = False,
) -> tuple[Workspace, RunRecord]:
ws = Workspace(runs_dir / slug).create()
- track = Track.SCENE if language is Language.SCENE_THREEJS else (
- Track.ARTICULATED_OBJECT if language is Language.URDF_BLENDER else (
- Track.GRAPHICS if language in (Language.GLSL_SHADER, Language.OPENGL_PYTHON) else Track.STATIC_OBJECT))
+ track = next(t for t, languages in TRACK_LANGUAGES.items() if language in languages)
spec = Spec(id=slug, track=track, language=language, prompt=prompt, backends=Backends(generator=generator))
ws.write_json(ws.spec_path, spec)
- entry = {"blender": "src/model.py", "cadquery": "src/model.py", "urdf_blender": "src/model.py",
- "threejs": "src/object.js", "scene_threejs": "src/scene.js",
- "glsl_shader": "src/shader.frag", "opengl_python": "src/program.py"}[language.value]
- e = ws.root / entry
+ e = ws.root / ENTRY_FILE[language]
e.parent.mkdir(parents=True, exist_ok=True)
e.write_text("# round 0\nimport bpy # comment\n\nbpy.ops.mesh.primitive_cube_add(size=1.0)\n")
if language is Language.URDF_BLENDER:
diff --git a/harness/tests/flywheel_cli/test_cli.py b/harness/tests/flywheel_cli/test_cli.py
index 181f44e3..49a4d664 100644
--- a/harness/tests/flywheel_cli/test_cli.py
+++ b/harness/tests/flywheel_cli/test_cli.py
@@ -637,3 +637,25 @@ def test_a_replay_judges_the_picked_round_on_the_runs_rubric_and_books_into_the_
assert load_record(ws).total_usage.cost_usd == pytest.approx(ledger_usage(load_ledger(ws.root)).cost_usd)
assert load_record(ws).total_usage.cost_usd == pytest.approx(total + 3 * 0.05)
+
+def test_render_of_a_scene_uses_the_in_run_cameras_and_times(tmp_path: Path, monkeypatch):
+ """N69: `3dcode render` on a scene renders the plan's cameras at SCENE_TIMES — what the in-run judge
+ saw (tracks/scene.ScenePipeline.render) — not the cameras createScene() authored."""
+ import codeverse3d.spatial.render_scene as RS
+ from codeverse3d.contracts.artifacts import RenderSet
+ from codeverse3d.contracts.plan import BBox, CameraPlan, ScenePlan, ZonePlan
+ from codeverse3d.tracks.scene import SCENE_TIMES
+ from codeverse3d.workspace import Workspace
+
+ runs = tmp_path / "runs"
+ assert runner.invoke(app, ["make", "x", "--track", "scene", "--no-run", "--runs-dir", str(runs), "--slug", "s"]).exit_code == 0
+ ws = Workspace(runs / "s")
+ cam = CameraPlan(name="hero", position=(4, 2, 4), look_at=(0, 0, 0))
+ ws.write_json(ws.plan_path, ScenePlan(title="t", summary="s", setting="x", environment="e", bounds=BBox(center=(0, 0, 0), extents=(10, 4, 10)),
+ zones=[ZonePlan(name="z", description="d", bbox=BBox(center=(0, 0, 0), extents=(1, 1, 1)))],
+ cameras=[cam]))
+ seen: dict = {}
+ monkeypatch.setattr(RS, "render_scene", lambda ws, out_dir, **kw: seen.update(kw) or RenderSet(views=[], renderer="fake"))
+ r = runner.invoke(app, ["render", "s", "--runs-dir", str(runs)])
+ assert r.exit_code == 0, r.output
+ assert [c.name for c in seen["cameras"]] == ["hero"] and tuple(seen["times"]) == SCENE_TIMES
diff --git a/harness/tests/flywheel_cli/test_refine.py b/harness/tests/flywheel_cli/test_refine.py
index 5318820d..bdc02f39 100644
--- a/harness/tests/flywheel_cli/test_refine.py
+++ b/harness/tests/flywheel_cli/test_refine.py
@@ -39,7 +39,8 @@ def test_a_refine_round_carries_its_brief_its_diff_and_its_delta(tmp_path):
rows, drops = _rows(tmp_path)
assert len(rows) == 1 and not drops
t = rows[0]
- assert t.round == 1 and t.round_kind == "refine" and t.battery == "batt"
+ assert t.round == 1 and t.round_kind == "refine"
+ assert (t.battery, t.run) == (tmp_path.name, "batt__chair") # its RunId: the key index.sqlite joins on (N59)
assert t.score_before == 0.30 and t.score_after == 0.80 and t.score_delta == 0.50 and t.outcome == "improved"
assert t.instructions == ["thicken the legs"]
assert [g["message"] for g in t.gate_errors] == ["part 'Leg' is floating"] # ERRORs only
diff --git a/harness/tests/skills/test_telemetry.py b/harness/tests/skills/test_telemetry.py
index 78ef7e4b..76271b36 100644
--- a/harness/tests/skills/test_telemetry.py
+++ b/harness/tests/skills/test_telemetry.py
@@ -192,7 +192,8 @@ def test_the_report_counts_exact_sessions_and_drops_only_blind_ones(tmp_path):
from codeverse3d.cli.main import app
- tele = tmp_path / "run" / TELEMETRY_DIR
+ root = tmp_path / "bat"
+ tele = root / "run" / TELEMETRY_DIR
tele.mkdir(parents=True)
rows = [ # exact, the agent opened the control too: counted, and said so
{"agent": "gemini-cli:gemini-3.7-flash", "listed": ["c3d-a"], "evidence": "transcript",
@@ -202,9 +203,15 @@ def test_the_report_counts_exact_sessions_and_drops_only_blind_ones(tmp_path):
"reads": [{"name": "c3d-a", "surfaced": True, "deep": True}]},
]
(tele / SKILLS_FILE).write_text("".join(json.dumps(r) + "\n" for r in rows))
- r = CliRunner().invoke(app, ["skills", "report", str(tmp_path), "--json"])
+ # N58: a best-of-N candidate's session counts, and a cell another battery symlinks in is read, once
+ for d in (root / "run" / "_cand" / "c0", tmp_path / "other" / "cell"):
+ (d / TELEMETRY_DIR).mkdir(parents=True)
+ (d / TELEMETRY_DIR / SKILLS_FILE).write_text(json.dumps(rows[0]) + "\n")
+ (root / "cell").symlink_to(tmp_path / "other" / "cell")
+ (root / "alias").symlink_to(tmp_path / "other" / "cell")
+ r = CliRunner().invoke(app, ["skills", "report", str(root), "--json"])
assert r.exit_code == 0, r.output
out = json.loads(r.output)
- assert out["transcript_sessions"] == 1 and out["control_opened_by_agent_sessions"] == 1
+ assert out["transcript_sessions"] == 3 and out["control_opened_by_agent_sessions"] == 3
assert out["control_read_sessions"] == 1
- assert [(s["backend"], s["deep"]) for s in out["skills"]] == [("gemini-cli", 1)]
+ assert [(s["backend"], s["deep"]) for s in out["skills"]] == [("gemini-cli", 3)]