Skip to content

Official Codebase of 3DCodeVerse

Python

gallery/flywheel: a compare or A/B battery dir exported "0 runs" and exited 0

gaoypeng856c22c6d40b
diff --git a/harness/codeverse/flywheel/record.py b/harness/codeverse/flywheel/record.py
index f99a01ed..47f56ee8 100644
--- a/harness/codeverse/flywheel/record.py
+++ b/harness/codeverse/flywheel/record.py
@@ -275,30 +275,51 @@ def load_record(ws: Workspace | Path | str) -> RunRecord:
raise RecordError(f"invalid record.json in {ws.root}: {e}") from e
-#: how deep to look for a nested run before giving up (ab_plan's layout is
-#: arms/<arm>/cells/<prompt>/<slug>/run — five levels below the battery root)
-_NESTED_SCAN_DEPTH = 6
+# Battery layouts nest their runs at these depths below the battery root: run_bench
+# ``runs/<id>`` = 2, compare_backends ``cells/<id>/<arm>/run`` = 4, ab_plan
+# ``arms/<arm>/cells/<id>/<slug>/run`` = 6. Seven covers all three with a margin and
+# stops a mistyped root from walking a whole home directory.
+RUN_SEARCH_DEPTH = 7
+#: subdirectories that mark a bench battery directory rather than a plain runs root
+BATTERY_MARKERS = ("runs", "cells", "arms")
-def _find_nested_run(root: Path, depth: int = _NESTED_SCAN_DEPTH) -> Path | None:
- """The first run directory strictly below ``root``, or None. Breadth-first and
- depth-bounded: this only has to prove that the caller pointed one level too high."""
- level = [root]
- for _ in range(depth):
+
+def is_run_dir(p: Path) -> bool:
+ return p.is_dir() and (p / "record.json").is_file()
+
+
+def find_run_dirs(root: Path | str, *, predicate: Callable[[Path], bool] = is_run_dir,
+ max_depth: int = RUN_SEARCH_DEPTH) -> list[Path]:
+ """Run directories under ``root``, sorted.
+
+ Direct children win when there are any — that is the ``3dcv`` runs root and the
+ run_bench ``runs/`` layout, and it keeps the common case a single ``iterdir()``.
+ Only when there are none do we descend, which is what makes a compare_backends or
+ ab_plan BATTERY directory work: those hold their runs four and five levels down, so
+ every exporter used to scan one level, find nothing and report "0 runs" as a success.
+ A directory that IS a run is never descended into (a run holds candidate workspaces
+ of its own).
+ """
+ root = Path(root)
+ if not root.is_dir():
+ return []
+ direct = sorted(d for d in root.iterdir() if predicate(d))
+ if direct:
+ return direct
+ found: list[Path] = []
+ frontier = [d for d in sorted(root.iterdir()) if d.is_dir()] # depth 1
+ for _ in range(max_depth):
+ if not frontier:
+ break
nxt: list[Path] = []
- for d in level:
- try:
- kids = sorted(p for p in d.iterdir() if p.is_dir())
- except OSError: # pragma: no cover - unreadable dir
- continue
- for k in kids:
- if (k / "record.json").is_file():
- return k
- nxt.append(k)
- if not nxt:
- return None
- level = nxt
- return None
+ for d in frontier:
+ if predicate(d):
+ found.append(d)
+ continue # a run's own subdirectories are not runs
+ nxt.extend(c for c in sorted(d.iterdir()) if c.is_dir())
+ frontier = nxt
+ return sorted(found)
def iter_runs(
@@ -312,22 +333,20 @@ def iter_runs(
root = Path(runs_dir)
if not root.is_dir():
raise FileNotFoundError(f"runs dir not found: {root}")
- children = sorted(p for p in root.iterdir() if p.is_dir())
- if not any((d / "record.json").is_file() for d in children):
- # Pointing at a BATTERY dir instead of its runs root is the easy mistake, and every
- # exporter here reports "0 runs" as a SUCCESS — writing an empty metadata.parquet.
- # Silence must not look like an empty dataset. Keying the guard on root/runs alone
- # left it silent for the two drivers that produce most batteries: compare_backends
- # writes cells/<prompt>/<arm>/run and ab_plan writes arms/<arm>/cells/<prompt>/<slug>/run.
- found = _find_nested_run(root)
- if found is not None:
- hint = (f"did you mean {root / 'runs'}?" if found.parent == root / "runs"
- else f"e.g. {found} — batteries nest their runs (cells/<prompt>/<arm>/run)")
- raise FileNotFoundError(
- f"no run directories directly under {root}, but there are runs below it — {hint}")
+ children = find_run_dirs(root)
+ battery_dirs = [m for m in BATTERY_MARKERS if (root / m).is_dir()]
+ if not children and battery_dirs:
+ # Silence must not look like an empty dataset. This guard used to test only for
+ # ``<root>/runs`` — true for the run_bench layout, false for compare_backends
+ # (cells/) and ab_plan (arms/), i.e. exactly today's batteries, which therefore
+ # reported "0 runs" as a success. find_run_dirs now FINDS all three layouts, so
+ # reaching here means a battery directory that really is empty; an ordinary runs
+ # root with no runs in it stays a legitimate empty result.
+ raise FileNotFoundError(
+ f"no run directories under {root} — it looks like a battery directory "
+ f"({', '.join(battery_dirs)}/) but holds no record.json within "
+ f"{RUN_SEARCH_DEPTH} levels")
for d in children:
- if not (d / "record.json").is_file():
- continue
ws = Workspace(d)
try:
rec = load_record(ws)
diff --git a/harness/codeverse/gallery/index.py b/harness/codeverse/gallery/index.py
index ad8d9bb3..4c5c37ee 100644
--- a/harness/codeverse/gallery/index.py
+++ b/harness/codeverse/gallery/index.py
@@ -18,7 +18,13 @@ from pathlib import Path
from codeverse.contracts.run import RunRecord
from codeverse.flywheel.quality import quality_tier
-from codeverse.flywheel.record import RecordError, best_round_index, effective_judgment, load_record
+from codeverse.flywheel.record import (
+ RecordError,
+ best_round_index,
+ effective_judgment,
+ find_run_dirs,
+ load_record,
+)
from codeverse.flywheel.sample import gate_error_summary, telemetry_digest
from codeverse.gallery.labels import humanize_view
from codeverse.gallery.model import GalleryIndex, RootSection, RoundRow, RunEntry, RunLink
@@ -283,8 +289,12 @@ def is_run_dir(p: Path) -> bool:
def scan_root(root: Path, label: str | None = None) -> RootSection:
+ """Runs under ``root``. Uses ``find_run_dirs`` rather than a one-level
+ ``iterdir()``: a compare_backends or ab_plan battery directory holds its runs four
+ and five levels down, so pointing the gallery at one built a page "of 0 runs" and
+ exited 0 — the silent-empty failure, on a directory full of real runs."""
label = label or root_label(root)
- entries = [entry_for_dir(label, d) for d in sorted(root.iterdir()) if is_run_dir(d)] if root.is_dir() else []
+ entries = [entry_for_dir(label, d) for d in find_run_dirs(root, predicate=is_run_dir)]
return RootSection(label=label, path=str(Path(root).resolve()), entries=entries)
diff --git a/harness/tests/flywheel_cli/test_flywheel.py b/harness/tests/flywheel_cli/test_flywheel.py
index 6b17523c..e7d847bc 100644
--- a/harness/tests/flywheel_cli/test_flywheel.py
+++ b/harness/tests/flywheel_cli/test_flywheel.py
@@ -371,48 +371,89 @@ def test_export_includes_textured_assets_when_shipped(fake_run, tmp_path: Path):
assert not (sdir2 / "textures").exists()
-def test_pointing_at_a_battery_dir_says_so_instead_of_exporting_nothing(tmp_path):
- """`bench/out/<battery>` holds its runs in `runs/`, and pointing one level too high
- used to report "runs 0, exported 0" as a SUCCESS — an empty dataset that looks like
- a finished one. Silence must not read as an empty result."""
- import pytest
-
- from codeverse.flywheel.record import iter_runs
-
- battery = tmp_path / "static_v2_flash"
+def test_a_battery_dir_is_discovered_rather_than_reported_as_zero_runs(tmp_path):
+ """CP-6: pointing an exporter at `bench/out/<battery>` used to report "runs 0,
+ exported 0" as a SUCCESS. The old guard only fired when `<root>/runs` existed —
+ true for the run_bench layout, FALSE for compare_backends (cells/<id>/<arm>/run) and
+ ab_plan (arms/<arm>/cells/<id>/<slug>/run), i.e. exactly today's batteries. All
+ three layouts are now discovered instead."""
+ from codeverse.flywheel.record import find_run_dirs
+
+ battery = tmp_path / "static_v2_flash" # run_bench: runs/<id>
(battery / "runs" / "some_run").mkdir(parents=True)
(battery / "runs" / "some_run" / "record.json").write_text("{}")
+ assert [d.name for d in find_run_dirs(battery)] == ["some_run"]
- with pytest.raises(FileNotFoundError, match=r"did you mean .*runs"):
- list(iter_runs(battery))
+ cmp_bat = tmp_path / "compare_v2" # compare_backends: cells/<id>/<arm>/run
+ cell = cmp_bat / "cells" / "cmp_med_chair" / "harness_api-agent" / "run"
+ cell.mkdir(parents=True)
+ (cell / "record.json").write_text("{}")
+ assert find_run_dirs(cmp_bat) == [cell]
+
+ ab = tmp_path / "ab_aa_noise" # ab_plan: arms/<arm>/cells/<id>/<slug>/run
+ for arm in ("control", "variant"):
+ d = ab / "arms" / arm / "cells" / "ctrl_med_chair" / "harness_api-agent" / "run"
+ d.mkdir(parents=True)
+ (d / "record.json").write_text("{}")
+ assert len(find_run_dirs(ab)) == 2
+
+
+def test_a_gallery_of_a_battery_dir_is_not_a_gallery_of_zero_runs(tmp_path):
+ """CP-6, the other half: gallery/index.scan_root did a single `root.iterdir()`, so
+ `3dcv gallery build bench/out/<ab battery>` printed "gallery of 0 runs" and exit 0."""
+ from codeverse.gallery.index import scan_root
+ ab = tmp_path / "ab_aa_noise"
+ d = ab / "arms" / "control" / "cells" / "ctrl_med_chair" / "harness_api-agent" / "run"
+ d.mkdir(parents=True)
+ (d / "spec.json").write_text("{}")
+ assert len(scan_root(ab).entries) == 1
-def test_a_compare_or_ab_battery_dir_says_so_too(tmp_path):
- """RS-8: the guard only knew the runs/ layout, but compare_backends writes
- cells/<prompt>/<arm>/run and ab_plan writes arms/<arm>/cells/<prompt>/<slug>/run —
- so for the two drivers that produced today's compare_v3 / A-B batteries the exact
- mistake the guard was added for was still silent (n_runs=0 reported as success,
- plus an empty metadata.parquet)."""
+
+def test_an_empty_battery_dir_still_refuses_to_look_like_an_empty_dataset(tmp_path):
+ """A battery directory with the layout but no records is the case the anti-silence
+ guard is still for — it must raise, not export nothing and exit 0."""
import pytest
from codeverse.flywheel.record import iter_runs
+ battery = tmp_path / "compare_empty"
+ (battery / "cells" / "cmp_med_chair").mkdir(parents=True)
+ with pytest.raises(FileNotFoundError, match=r"looks like a battery directory"):
+ list(iter_runs(battery))
+
+
+def test_a_compare_or_ab_battery_dir_is_exported_not_refused(tmp_path):
+ """RS-8 / CP-6, the two reports of one defect. The original guard only knew the
+ runs/ layout, so for compare_backends (``cells/<prompt>/<arm>/run``) and ab_plan
+ (``arms/<arm>/cells/<prompt>/<slug>/run``) — the drivers that produced today's
+ compare_v3 and A-B batteries — pointing an exporter at the battery reported
+ n_runs=0 as a SUCCESS and wrote an empty metadata.parquet.
+
+ Sign-off note: two waves fixed this, one by raising a loud hint and one by
+ discovering the runs. Discovery wins and subsumes the hint — every other battery
+ reader (``3dcv cost --runs-dir``) already accepts a battery root, so refusing here
+ would have been the only command that did not. The anti-silence guard is kept for
+ the case discovery cannot rescue: a battery-SHAPED directory holding no records
+ (``test_an_empty_battery_dir_still_refuses_to_look_like_an_empty_dataset``)."""
+ from codeverse.flywheel.record import find_run_dirs, iter_runs
+
compare = tmp_path / "compare_v3"
for pid in ("p0", "p1"):
for arm in ("harness_codex", "oneshot_gemini"):
run = compare / "cells" / pid / arm / "run"
run.mkdir(parents=True)
(run / "record.json").write_text("{}")
-
- with pytest.raises(FileNotFoundError, match="runs below it"):
- list(iter_runs(compare))
+ assert len(find_run_dirs(compare)) == 4
+ seen: list[str] = []
+ list(iter_runs(compare, on_error=lambda d, e: seen.append(d.name))) # must not raise
+ assert len(seen) == 4, "every cell is reached; these stub records are invalid, not absent"
ab = tmp_path / "ab_plan_v1"
deep = ab / "arms" / "control" / "cells" / "p0" / "stool_ab12" / "run"
deep.mkdir(parents=True)
(deep / "record.json").write_text("{}")
- with pytest.raises(FileNotFoundError, match="runs below it"):
- list(iter_runs(ab))
+ assert find_run_dirs(ab) == [deep]
def test_a_genuinely_empty_runs_root_is_still_just_empty(tmp_path):