Skip to content

Official Codebase of 3DCodeVerse

Python

gallery: the default battery glob is eval/bench/out/*/runs — bench/out/ moved into eval/ (D77)

gaoypenge94564df4825
diff --git a/harness/codeverse3d/addons/gallery/index.py b/harness/codeverse3d/addons/gallery/index.py
index 97062a56..14c684e4 100644
--- a/harness/codeverse3d/addons/gallery/index.py
+++ b/harness/codeverse3d/addons/gallery/index.py
@@ -41,11 +41,11 @@ from codeverse3d.workspace import Workspace
#: run roots the CLI defaults to when the user names none
DEFAULT_RUNS = "runs"
-DEFAULT_BENCH_GLOB = "bench/out/*/runs"
+DEFAULT_BENCH_GLOB = "eval/bench/out/*/runs"
def default_roots(base: Path | str = ".") -> list[Path]:
- """``runs/`` plus every ``bench/out/*/runs`` that exists, in that order."""
+ """``runs/`` plus every ``eval/bench/out/*/runs`` that exists, in that order."""
base = Path(base)
roots: list[Path] = []
if (base / DEFAULT_RUNS).is_dir():
diff --git a/harness/codeverse3d/cli/main.py b/harness/codeverse3d/cli/main.py
index 652ff504..ef1eb658 100644
--- a/harness/codeverse3d/cli/main.py
+++ b/harness/codeverse3d/cli/main.py
@@ -92,7 +92,7 @@ def tools(
gallery_app = typer.Typer(no_args_is_help=True)
RootsArg = Annotated[list[Path] | None, typer.Argument(
- help="run roots (default: ./runs, every ./bench/out/*/runs and every <repo>/eval/bench/out/*/runs that exists)")]
+ help="run roots (default: ./runs, every ./eval/bench/out/*/runs and every <repo>/eval/bench/out/*/runs that exists)")]
def resolve_roots(roots: list[Path] | None) -> list[Path]:
@@ -113,7 +113,7 @@ def resolve_roots(roots: list[Path] | None) -> list[Path]:
fallback = Path(get_settings().runs_dir)
if fallback.is_dir():
return [fallback]
- raise C.CliError(f"no run roots found (looked for runs/ and bench/out/*/runs under {Path.cwd()}, "
+ raise C.CliError(f"no run roots found (looked for runs/ and eval/bench/out/*/runs under {Path.cwd()}, "
f"and {C.EVAL_ROOT / 'bench' / 'out'}/*/runs); "
f"pass one explicitly: `3dcode gallery serve path/to/runs`")
return found
diff --git a/harness/tests/gallery/conftest.py b/harness/tests/gallery/conftest.py
index 3d993b7d..657c186b 100644
--- a/harness/tests/gallery/conftest.py
+++ b/harness/tests/gallery/conftest.py
@@ -19,7 +19,7 @@ from tests.flywheel_cli.conftest import make_fake_run, tiny_png
def gallery_tree(tmp_path: Path) -> dict[str, Path]:
"""``{"runs": …, "battery": …}`` — two roots holding six run directories."""
runs = tmp_path / "runs"
- battery = tmp_path / "bench" / "out" / "static_v9" / "runs"
+ battery = tmp_path / "eval" / "bench" / "out" / "static_v9" / "runs"
battery.mkdir(parents=True)
make_fake_run(runs, "wooden_chair_ab12cd34") # passes
diff --git a/harness/tests/gallery/test_cli.py b/harness/tests/gallery/test_cli.py
index 4c3e6a71..ee86fd0a 100644
--- a/harness/tests/gallery/test_cli.py
+++ b/harness/tests/gallery/test_cli.py
@@ -52,20 +52,20 @@ def test_default_roots_from_cwd(gallery_tree: dict[str, Path], monkeypatch: pyte
monkeypatch.chdir(gallery_tree["root"])
roots = resolve_roots(None)
assert [r.name for r in roots] == ["runs", "runs"]
- assert str(roots[1]).endswith(os.path.join("bench", "out", "static_v9", "runs"))
+ assert str(roots[1]).endswith(os.path.join("eval", "bench", "out", "static_v9", "runs"))
-def test_default_roots_find_the_eval_batteries_from_any_cwd(gallery_tree: dict[str, Path], tmp_path: Path,
- monkeypatch: pytest.MonkeyPatch):
+def test_default_roots_find_the_eval_batteries_from_any_cwd(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
"""D77 moved the batteries to <repo>/eval/bench/out/<b>/runs; the defaults only looked
under the cwd, so `3dcode gallery serve` from harness/ listed none of them."""
- battery = tmp_path / "eval" / "bench" / "out" / "b1" / "runs"
+ repo = tmp_path / "repo"
+ battery = repo / "eval" / "bench" / "out" / "b1" / "runs"
battery.mkdir(parents=True)
- monkeypatch.setattr("codeverse3d.cli._common.EVAL_ROOT", tmp_path / "eval")
- monkeypatch.chdir(gallery_tree["root"])
- assert resolve_roots(None)[-1] == battery and len(resolve_roots(None)) == 3
- monkeypatch.chdir(tmp_path / "eval") # cwd IS the eval root: each battery listed once
- assert resolve_roots(None) == [Path.cwd() / "bench" / "out" / "b1" / "runs"]
+ (repo / "harness").mkdir()
+ monkeypatch.setattr("codeverse3d.cli._common.EVAL_ROOT", repo / "eval")
+ for cwd in (repo / "harness", repo, repo / "eval"): # each battery listed once, from anywhere
+ monkeypatch.chdir(cwd)
+ assert [p.resolve() for p in resolve_roots(None)] == [battery.resolve()], cwd
def test_no_roots_anywhere_is_a_clear_error(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):