Official Codebase of 3DCodeVerse
Python
flywheel export: a core-only install crashed after writing a partial dataset
gaoypeng098a6c3c6301
diff --git a/harness/codeverse/cli/flywheel_cmd.py b/harness/codeverse/cli/flywheel_cmd.pyindex debb8513..b23be261 100644--- a/harness/codeverse/cli/flywheel_cmd.py+++ b/harness/codeverse/cli/flywheel_cmd.py@@ -6,6 +6,7 @@ from pathlib import Pathfrom typing import Annotatedimport typer+from rich.markup import escapefrom codeverse.cli import _common as Cfrom codeverse.cli._fmt import console, kv_table, ok, warn@@ -25,15 +26,29 @@ def export_cmd(captions_dir: Annotated[Path | None, typer.Option("--captions-dir", help="side-car captions written by `caption --out`")] = None,drop_duplicates: Annotated[bool, typer.Option("--drop-duplicates", help="leave exact duplicates (code fingerprint + prompt) out of the index")] = False,) -> None:- """Export runs → sample folders + metadata.parquet (+ optional plain tars)."""+ """Export runs → sample folders + metadata.jsonl/.parquet (+ optional plain tars)."""from codeverse.flywheel.export import export_samples+ if pack:+ # pack_samples rewrites the index and reads it back with pyarrow, so --pack+ # cannot degrade the way a plain export can. Say so BEFORE the first sample is+ # written rather than after a few hundred, with a raw ModuleNotFoundError.+ try:+ import pyarrow # noqa: F401+ except ImportError:+ # escaped: rich reads "[flywheel]" as markup and would print the tag away+ raise C.CliError(escape("--pack needs the flywheel extra: "+ "pip install -e 'harness[flywheel]'")) from None+rep = export_samples(runs_dir, out_dir, min_score=min_score, only_passed=only_passed, include_unbuilt=include_unbuilt,captions_dir=captions_dir, drop_duplicates=drop_duplicates)tiers = " ".join(f"{t}:{rep.tiers.get(t, 0)}" for t in "ABCD")console.print(kv_table("export", {"runs": rep.n_runs, "exported": rep.n_exported, "indexed": rep.n_indexed,"duplicates": rep.n_duplicates, "tiers": tiers,- "skipped": len(rep.skipped), "parquet": rep.parquet}))+ "skipped": len(rep.skipped), "jsonl": rep.jsonl,+ "parquet": rep.parquet or "(skipped: no pyarrow)"}))+ for note in rep.notes:+ warn(escape(note))for d, why in list(rep.skipped.items())[:20]:warn(f"skip {Path(d).name}: {why}")for g in rep.duplicates[:20]:diff --git a/harness/codeverse/flywheel/export.py b/harness/codeverse/flywheel/export.pyindex a2a5172b..91c4bd9f 100644--- a/harness/codeverse/flywheel/export.py+++ b/harness/codeverse/flywheel/export.py@@ -55,6 +55,8 @@ class ExportReport(BaseModel):tiers: dict[str, int] = Field(default_factory=dict, description="quality tier → count (indexed rows)")parquet: str = ""jsonl: str = ""+ notes: list[str] = Field(default_factory=list,+ description="non-fatal degradations, e.g. parquet skipped for a missing extra")def load_captions(ws: Workspace, record: RunRecord, captions_dir: Path | None = None) -> dict[str, Any]:@@ -183,8 +185,19 @@ def export_samples(rep.n_indexed = len(rows)for r in rows:rep.tiers[r["quality_tier"]] = rep.tiers.get(r["quality_tier"], 0) + 1- rep.parquet = str(write_parquet(rows, out / PARQUET_NAME))+ # JSONL first, and unconditionally: it carries the same rows with no third-party+ # dependency, so a core-only install still gets a complete, indexed dataset.rep.jsonl = str(write_jsonl(rows, out / JSONL_NAME))+ try:+ rep.parquet = str(write_parquet(rows, out / PARQUET_NAME))+ except ImportError:+ # docs/INSTALL.md §4 scopes pyarrow to the `flywheel` extra and promises the+ # harness only raises at the moment the named feature is used. Crashing here+ # left a partial dataset on disk (samples written, no index at all) and the+ # rows were already computed — degrade loudly instead.+ rep.notes.append(f"{PARQUET_NAME} skipped: pyarrow is not installed "+ f"(pip install -e 'harness[flywheel]'); {JSONL_NAME} has the same rows")+ log.warning("%s", rep.notes[-1])return repdiff --git a/harness/docs/INSTALL.md b/harness/docs/INSTALL.mdindex 74ac076a..b577cf8b 100644--- a/harness/docs/INSTALL.md+++ b/harness/docs/INSTALL.md@@ -135,7 +135,7 @@ openai, playwright.| `graphics` | moderngl | the `graphics` track (`glsl_shader`, `opengl_python`) — headless GL rendering in `spatial/gl_render.py` || `mesh` | shapely, networkx, manifold3d | `cross_section` filled-area/hollow ratio (shapely), mesh split / connectivity (networkx), joint-sweep boolean intersections (`trimesh.boolean(engine="manifold")`) || `mcp` | mcp | `3dcv mcp` / `codeverse.spatial.mcp_server`, i.e. spatial tools for the `gemini-cli`, `claude-code` and `codex` backends |-| `flywheel` | pyarrow | `3dcv flywheel export --pack` (parquet shards) |+| `flywheel` | pyarrow | `3dcv flywheel export --pack` (parquet shards). A plain `flywheel export` needs nothing: without pyarrow it writes `metadata.jsonl` and skips `metadata.parquet` with a warning. || `all` | all of the above | — || `dev` | pytest, pytest-timeout, ruff | the test suite and the linter |diff --git a/harness/tests/flywheel_cli/test_flywheel.py b/harness/tests/flywheel_cli/test_flywheel.pyindex fc9dcafb..6b17523c 100644--- a/harness/tests/flywheel_cli/test_flywheel.py+++ b/harness/tests/flywheel_cli/test_flywheel.py@@ -448,3 +448,55 @@ def test_parquet_keeps_the_complexity_columns_the_exporter_writes(tmp_path):# and the schema can never fall behind row_for_sample againwith pytest.raises(ValueError, match="missing column"):write_parquet([{**row, "a_new_metric": 1.0}], tmp_path / "later.parquet")+++def test_a_core_only_install_still_gets_a_complete_dataset(runs_dir: Path, tmp_path: Path, monkeypatch):+ """PORT-5: `3dcv flywheel export` (no --pack) called write_parquet unconditionally,+ so on an install without the optional `flywheel` extra it died with a raw+ ModuleNotFoundError AFTER writing every sample folder — a partial dataset with no+ index of any kind. docs/INSTALL.md §4 scopes pyarrow to the extra and promises the+ harness only raises when the named feature is used, so a plain export must degrade:+ metadata.jsonl always, metadata.parquet only when pyarrow is importable."""+ import builtins++ real_import = builtins.__import__++ def no_pyarrow(name, *a, **kw):+ if name == "pyarrow" or name.startswith("pyarrow."):+ raise ModuleNotFoundError("No module named 'pyarrow'")+ return real_import(name, *a, **kw)++ monkeypatch.setattr(builtins, "__import__", no_pyarrow)+ out = tmp_path / "ds_core"+ rep = export_samples(runs_dir, out) # must NOT raise+ assert rep.n_exported == 3 and rep.n_indexed == 3+ # the index the operator can actually use is there, and complete+ assert (out / "metadata.jsonl").is_file()+ assert len((out / "metadata.jsonl").read_text().splitlines()) == 3+ # the parquet is skipped, loudly and by name — never silently+ assert not (out / "metadata.parquet").exists() and rep.parquet == ""+ assert rep.notes and "pyarrow" in rep.notes[0] and "harness[flywheel]" in rep.notes[0]+++def test_pack_refuses_before_writing_anything_when_pyarrow_is_missing(runs_dir: Path, tmp_path: Path, monkeypatch):+ """--pack genuinely needs pyarrow (pack.py reads the index back with pq.read_table),+ so it must say so BEFORE the first sample is written, not after a few hundred."""+ import builtins++ from typer.testing import CliRunner++ from codeverse.cli.main import app++ real_import = builtins.__import__++ def no_pyarrow(name, *a, **kw):+ if name == "pyarrow" or name.startswith("pyarrow."):+ raise ModuleNotFoundError("No module named 'pyarrow'")+ return real_import(name, *a, **kw)++ monkeypatch.setattr(builtins, "__import__", no_pyarrow)+ out = tmp_path / "ds_pack"+ res = CliRunner().invoke(app, ["flywheel", "export", str(runs_dir), str(out), "--pack"])+ assert res.exit_code == 1+ assert "harness[flywheel]" in res.output+ assert not out.exists(), "no partial dataset may be left behind"