Official Codebase of 3DCodeVerse
Python
gallery/flywheel: a compare or A/B battery dir exported "0 runs" and exited 0
gaoypeng856c22c6d40b
diff --git a/harness/codeverse/flywheel/record.py b/harness/codeverse/flywheel/record.pyindex f99a01ed..47f56ee8 100644--- a/harness/codeverse/flywheel/record.py+++ b/harness/codeverse/flywheel/record.py@@ -275,30 +275,51 @@ def load_record(ws: Workspace | Path | str) -> RunRecord:raise RecordError(f"invalid record.json in {ws.root}: {e}") from e-#: how deep to look for a nested run before giving up (ab_plan's layout is-#: arms/<arm>/cells/<prompt>/<slug>/run — five levels below the battery root)-_NESTED_SCAN_DEPTH = 6+# Battery layouts nest their runs at these depths below the battery root: run_bench+# ``runs/<id>`` = 2, compare_backends ``cells/<id>/<arm>/run`` = 4, ab_plan+# ``arms/<arm>/cells/<id>/<slug>/run`` = 6. Seven covers all three with a margin and+# stops a mistyped root from walking a whole home directory.+RUN_SEARCH_DEPTH = 7+#: subdirectories that mark a bench battery directory rather than a plain runs root+BATTERY_MARKERS = ("runs", "cells", "arms")-def _find_nested_run(root: Path, depth: int = _NESTED_SCAN_DEPTH) -> Path | None:- """The first run directory strictly below ``root``, or None. Breadth-first and- depth-bounded: this only has to prove that the caller pointed one level too high."""- level = [root]- for _ in range(depth):++def is_run_dir(p: Path) -> bool:+ return p.is_dir() and (p / "record.json").is_file()+++def find_run_dirs(root: Path | str, *, predicate: Callable[[Path], bool] = is_run_dir,+ max_depth: int = RUN_SEARCH_DEPTH) -> list[Path]:+ """Run directories under ``root``, sorted.++ Direct children win when there are any — that is the ``3dcv`` runs root and the+ run_bench ``runs/`` layout, and it keeps the common case a single ``iterdir()``.+ Only when there are none do we descend, which is what makes a compare_backends or+ ab_plan BATTERY directory work: those hold their runs four and five levels down, so+ every exporter used to scan one level, find nothing and report "0 runs" as a success.+ A directory that IS a run is never descended into (a run holds candidate workspaces+ of its own).+ """+ root = Path(root)+ if not root.is_dir():+ return []+ direct = sorted(d for d in root.iterdir() if predicate(d))+ if direct:+ return direct+ found: list[Path] = []+ frontier = [d for d in sorted(root.iterdir()) if d.is_dir()] # depth 1+ for _ in range(max_depth):+ if not frontier:+ breaknxt: list[Path] = []- for d in level:- try:- kids = sorted(p for p in d.iterdir() if p.is_dir())- except OSError: # pragma: no cover - unreadable dir- continue- for k in kids:- if (k / "record.json").is_file():- return k- nxt.append(k)- if not nxt:- return None- level = nxt- return None+ for d in frontier:+ if predicate(d):+ found.append(d)+ continue # a run's own subdirectories are not runs+ nxt.extend(c for c in sorted(d.iterdir()) if c.is_dir())+ frontier = nxt+ return sorted(found)def iter_runs(@@ -312,22 +333,20 @@ def iter_runs(root = Path(runs_dir)if not root.is_dir():raise FileNotFoundError(f"runs dir not found: {root}")- children = sorted(p for p in root.iterdir() if p.is_dir())- if not any((d / "record.json").is_file() for d in children):- # Pointing at a BATTERY dir instead of its runs root is the easy mistake, and every- # exporter here reports "0 runs" as a SUCCESS — writing an empty metadata.parquet.- # Silence must not look like an empty dataset. Keying the guard on root/runs alone- # left it silent for the two drivers that produce most batteries: compare_backends- # writes cells/<prompt>/<arm>/run and ab_plan writes arms/<arm>/cells/<prompt>/<slug>/run.- found = _find_nested_run(root)- if found is not None:- hint = (f"did you mean {root / 'runs'}?" if found.parent == root / "runs"- else f"e.g. {found} — batteries nest their runs (cells/<prompt>/<arm>/run)")- raise FileNotFoundError(- f"no run directories directly under {root}, but there are runs below it — {hint}")+ children = find_run_dirs(root)+ battery_dirs = [m for m in BATTERY_MARKERS if (root / m).is_dir()]+ if not children and battery_dirs:+ # Silence must not look like an empty dataset. This guard used to test only for+ # ``<root>/runs`` — true for the run_bench layout, false for compare_backends+ # (cells/) and ab_plan (arms/), i.e. exactly today's batteries, which therefore+ # reported "0 runs" as a success. find_run_dirs now FINDS all three layouts, so+ # reaching here means a battery directory that really is empty; an ordinary runs+ # root with no runs in it stays a legitimate empty result.+ raise FileNotFoundError(+ f"no run directories under {root} — it looks like a battery directory "+ f"({', '.join(battery_dirs)}/) but holds no record.json within "+ f"{RUN_SEARCH_DEPTH} levels")for d in children:- if not (d / "record.json").is_file():- continuews = Workspace(d)try:rec = load_record(ws)diff --git a/harness/codeverse/gallery/index.py b/harness/codeverse/gallery/index.pyindex ad8d9bb3..4c5c37ee 100644--- a/harness/codeverse/gallery/index.py+++ b/harness/codeverse/gallery/index.py@@ -18,7 +18,13 @@ from pathlib import Pathfrom codeverse.contracts.run import RunRecordfrom codeverse.flywheel.quality import quality_tier-from codeverse.flywheel.record import RecordError, best_round_index, effective_judgment, load_record+from codeverse.flywheel.record import (+ RecordError,+ best_round_index,+ effective_judgment,+ find_run_dirs,+ load_record,+)from codeverse.flywheel.sample import gate_error_summary, telemetry_digestfrom codeverse.gallery.labels import humanize_viewfrom codeverse.gallery.model import GalleryIndex, RootSection, RoundRow, RunEntry, RunLink@@ -283,8 +289,12 @@ def is_run_dir(p: Path) -> bool:def scan_root(root: Path, label: str | None = None) -> RootSection:+ """Runs under ``root``. Uses ``find_run_dirs`` rather than a one-level+ ``iterdir()``: a compare_backends or ab_plan battery directory holds its runs four+ and five levels down, so pointing the gallery at one built a page "of 0 runs" and+ exited 0 — the silent-empty failure, on a directory full of real runs."""label = label or root_label(root)- entries = [entry_for_dir(label, d) for d in sorted(root.iterdir()) if is_run_dir(d)] if root.is_dir() else []+ entries = [entry_for_dir(label, d) for d in find_run_dirs(root, predicate=is_run_dir)]return RootSection(label=label, path=str(Path(root).resolve()), entries=entries)diff --git a/harness/tests/flywheel_cli/test_flywheel.py b/harness/tests/flywheel_cli/test_flywheel.pyindex 6b17523c..e7d847bc 100644--- a/harness/tests/flywheel_cli/test_flywheel.py+++ b/harness/tests/flywheel_cli/test_flywheel.py@@ -371,48 +371,89 @@ def test_export_includes_textured_assets_when_shipped(fake_run, tmp_path: Path):assert not (sdir2 / "textures").exists()-def test_pointing_at_a_battery_dir_says_so_instead_of_exporting_nothing(tmp_path):- """`bench/out/<battery>` holds its runs in `runs/`, and pointing one level too high- used to report "runs 0, exported 0" as a SUCCESS — an empty dataset that looks like- a finished one. Silence must not read as an empty result."""- import pytest-- from codeverse.flywheel.record import iter_runs-- battery = tmp_path / "static_v2_flash"+def test_a_battery_dir_is_discovered_rather_than_reported_as_zero_runs(tmp_path):+ """CP-6: pointing an exporter at `bench/out/<battery>` used to report "runs 0,+ exported 0" as a SUCCESS. The old guard only fired when `<root>/runs` existed —+ true for the run_bench layout, FALSE for compare_backends (cells/<id>/<arm>/run) and+ ab_plan (arms/<arm>/cells/<id>/<slug>/run), i.e. exactly today's batteries. All+ three layouts are now discovered instead."""+ from codeverse.flywheel.record import find_run_dirs++ battery = tmp_path / "static_v2_flash" # run_bench: runs/<id>(battery / "runs" / "some_run").mkdir(parents=True)(battery / "runs" / "some_run" / "record.json").write_text("{}")+ assert [d.name for d in find_run_dirs(battery)] == ["some_run"]- with pytest.raises(FileNotFoundError, match=r"did you mean .*runs"):- list(iter_runs(battery))+ cmp_bat = tmp_path / "compare_v2" # compare_backends: cells/<id>/<arm>/run+ cell = cmp_bat / "cells" / "cmp_med_chair" / "harness_api-agent" / "run"+ cell.mkdir(parents=True)+ (cell / "record.json").write_text("{}")+ assert find_run_dirs(cmp_bat) == [cell]++ ab = tmp_path / "ab_aa_noise" # ab_plan: arms/<arm>/cells/<id>/<slug>/run+ for arm in ("control", "variant"):+ d = ab / "arms" / arm / "cells" / "ctrl_med_chair" / "harness_api-agent" / "run"+ d.mkdir(parents=True)+ (d / "record.json").write_text("{}")+ assert len(find_run_dirs(ab)) == 2+++def test_a_gallery_of_a_battery_dir_is_not_a_gallery_of_zero_runs(tmp_path):+ """CP-6, the other half: gallery/index.scan_root did a single `root.iterdir()`, so+ `3dcv gallery build bench/out/<ab battery>` printed "gallery of 0 runs" and exit 0."""+ from codeverse.gallery.index import scan_root+ ab = tmp_path / "ab_aa_noise"+ d = ab / "arms" / "control" / "cells" / "ctrl_med_chair" / "harness_api-agent" / "run"+ d.mkdir(parents=True)+ (d / "spec.json").write_text("{}")+ assert len(scan_root(ab).entries) == 1-def test_a_compare_or_ab_battery_dir_says_so_too(tmp_path):- """RS-8: the guard only knew the runs/ layout, but compare_backends writes- cells/<prompt>/<arm>/run and ab_plan writes arms/<arm>/cells/<prompt>/<slug>/run —- so for the two drivers that produced today's compare_v3 / A-B batteries the exact- mistake the guard was added for was still silent (n_runs=0 reported as success,- plus an empty metadata.parquet)."""++def test_an_empty_battery_dir_still_refuses_to_look_like_an_empty_dataset(tmp_path):+ """A battery directory with the layout but no records is the case the anti-silence+ guard is still for — it must raise, not export nothing and exit 0."""import pytestfrom codeverse.flywheel.record import iter_runs+ battery = tmp_path / "compare_empty"+ (battery / "cells" / "cmp_med_chair").mkdir(parents=True)+ with pytest.raises(FileNotFoundError, match=r"looks like a battery directory"):+ list(iter_runs(battery))+++def test_a_compare_or_ab_battery_dir_is_exported_not_refused(tmp_path):+ """RS-8 / CP-6, the two reports of one defect. The original guard only knew the+ runs/ layout, so for compare_backends (``cells/<prompt>/<arm>/run``) and ab_plan+ (``arms/<arm>/cells/<prompt>/<slug>/run``) — the drivers that produced today's+ compare_v3 and A-B batteries — pointing an exporter at the battery reported+ n_runs=0 as a SUCCESS and wrote an empty metadata.parquet.++ Sign-off note: two waves fixed this, one by raising a loud hint and one by+ discovering the runs. Discovery wins and subsumes the hint — every other battery+ reader (``3dcv cost --runs-dir``) already accepts a battery root, so refusing here+ would have been the only command that did not. The anti-silence guard is kept for+ the case discovery cannot rescue: a battery-SHAPED directory holding no records+ (``test_an_empty_battery_dir_still_refuses_to_look_like_an_empty_dataset``)."""+ from codeverse.flywheel.record import find_run_dirs, iter_runs+compare = tmp_path / "compare_v3"for pid in ("p0", "p1"):for arm in ("harness_codex", "oneshot_gemini"):run = compare / "cells" / pid / arm / "run"run.mkdir(parents=True)(run / "record.json").write_text("{}")-- with pytest.raises(FileNotFoundError, match="runs below it"):- list(iter_runs(compare))+ assert len(find_run_dirs(compare)) == 4+ seen: list[str] = []+ list(iter_runs(compare, on_error=lambda d, e: seen.append(d.name))) # must not raise+ assert len(seen) == 4, "every cell is reached; these stub records are invalid, not absent"ab = tmp_path / "ab_plan_v1"deep = ab / "arms" / "control" / "cells" / "p0" / "stool_ab12" / "run"deep.mkdir(parents=True)(deep / "record.json").write_text("{}")- with pytest.raises(FileNotFoundError, match="runs below it"):- list(iter_runs(ab))+ assert find_run_dirs(ab) == [deep]def test_a_genuinely_empty_runs_root_is_still_just_empty(tmp_path):