Skip to content

Official Codebase of 3DCodeVerse

Python

flywheel export: a core-only install crashed after writing a partial dataset

gaoypeng098a6c3c6301
diff --git a/harness/codeverse/cli/flywheel_cmd.py b/harness/codeverse/cli/flywheel_cmd.py
index debb8513..b23be261 100644
--- a/harness/codeverse/cli/flywheel_cmd.py
+++ b/harness/codeverse/cli/flywheel_cmd.py
@@ -6,6 +6,7 @@ from pathlib import Path
from typing import Annotated
import typer
+from rich.markup import escape
from codeverse.cli import _common as C
from codeverse.cli._fmt import console, kv_table, ok, warn
@@ -25,15 +26,29 @@ def export_cmd(
captions_dir: Annotated[Path | None, typer.Option("--captions-dir", help="side-car captions written by `caption --out`")] = None,
drop_duplicates: Annotated[bool, typer.Option("--drop-duplicates", help="leave exact duplicates (code fingerprint + prompt) out of the index")] = False,
) -> None:
- """Export runs → sample folders + metadata.parquet (+ optional plain tars)."""
+ """Export runs → sample folders + metadata.jsonl/.parquet (+ optional plain tars)."""
from codeverse.flywheel.export import export_samples
+ if pack:
+ # pack_samples rewrites the index and reads it back with pyarrow, so --pack
+ # cannot degrade the way a plain export can. Say so BEFORE the first sample is
+ # written rather than after a few hundred, with a raw ModuleNotFoundError.
+ try:
+ import pyarrow # noqa: F401
+ except ImportError:
+ # escaped: rich reads "[flywheel]" as markup and would print the tag away
+ raise C.CliError(escape("--pack needs the flywheel extra: "
+ "pip install -e 'harness[flywheel]'")) from None
+
rep = export_samples(runs_dir, out_dir, min_score=min_score, only_passed=only_passed, include_unbuilt=include_unbuilt,
captions_dir=captions_dir, drop_duplicates=drop_duplicates)
tiers = " ".join(f"{t}:{rep.tiers.get(t, 0)}" for t in "ABCD")
console.print(kv_table("export", {"runs": rep.n_runs, "exported": rep.n_exported, "indexed": rep.n_indexed,
"duplicates": rep.n_duplicates, "tiers": tiers,
- "skipped": len(rep.skipped), "parquet": rep.parquet}))
+ "skipped": len(rep.skipped), "jsonl": rep.jsonl,
+ "parquet": rep.parquet or "(skipped: no pyarrow)"}))
+ for note in rep.notes:
+ warn(escape(note))
for d, why in list(rep.skipped.items())[:20]:
warn(f"skip {Path(d).name}: {why}")
for g in rep.duplicates[:20]:
diff --git a/harness/codeverse/flywheel/export.py b/harness/codeverse/flywheel/export.py
index a2a5172b..91c4bd9f 100644
--- a/harness/codeverse/flywheel/export.py
+++ b/harness/codeverse/flywheel/export.py
@@ -55,6 +55,8 @@ class ExportReport(BaseModel):
tiers: dict[str, int] = Field(default_factory=dict, description="quality tier → count (indexed rows)")
parquet: str = ""
jsonl: str = ""
+ notes: list[str] = Field(default_factory=list,
+ description="non-fatal degradations, e.g. parquet skipped for a missing extra")
def load_captions(ws: Workspace, record: RunRecord, captions_dir: Path | None = None) -> dict[str, Any]:
@@ -183,8 +185,19 @@ def export_samples(
rep.n_indexed = len(rows)
for r in rows:
rep.tiers[r["quality_tier"]] = rep.tiers.get(r["quality_tier"], 0) + 1
- rep.parquet = str(write_parquet(rows, out / PARQUET_NAME))
+ # JSONL first, and unconditionally: it carries the same rows with no third-party
+ # dependency, so a core-only install still gets a complete, indexed dataset.
rep.jsonl = str(write_jsonl(rows, out / JSONL_NAME))
+ try:
+ rep.parquet = str(write_parquet(rows, out / PARQUET_NAME))
+ except ImportError:
+ # docs/INSTALL.md §4 scopes pyarrow to the `flywheel` extra and promises the
+ # harness only raises at the moment the named feature is used. Crashing here
+ # left a partial dataset on disk (samples written, no index at all) and the
+ # rows were already computed — degrade loudly instead.
+ rep.notes.append(f"{PARQUET_NAME} skipped: pyarrow is not installed "
+ f"(pip install -e 'harness[flywheel]'); {JSONL_NAME} has the same rows")
+ log.warning("%s", rep.notes[-1])
return rep
diff --git a/harness/docs/INSTALL.md b/harness/docs/INSTALL.md
index 74ac076a..b577cf8b 100644
--- a/harness/docs/INSTALL.md
+++ b/harness/docs/INSTALL.md
@@ -135,7 +135,7 @@ openai, playwright.
| `graphics` | moderngl | the `graphics` track (`glsl_shader`, `opengl_python`) — headless GL rendering in `spatial/gl_render.py` |
| `mesh` | shapely, networkx, manifold3d | `cross_section` filled-area/hollow ratio (shapely), mesh split / connectivity (networkx), joint-sweep boolean intersections (`trimesh.boolean(engine="manifold")`) |
| `mcp` | mcp | `3dcv mcp` / `codeverse.spatial.mcp_server`, i.e. spatial tools for the `gemini-cli`, `claude-code` and `codex` backends |
-| `flywheel` | pyarrow | `3dcv flywheel export --pack` (parquet shards) |
+| `flywheel` | pyarrow | `3dcv flywheel export --pack` (parquet shards). A plain `flywheel export` needs nothing: without pyarrow it writes `metadata.jsonl` and skips `metadata.parquet` with a warning. |
| `all` | all of the above | — |
| `dev` | pytest, pytest-timeout, ruff | the test suite and the linter |
diff --git a/harness/tests/flywheel_cli/test_flywheel.py b/harness/tests/flywheel_cli/test_flywheel.py
index fc9dcafb..6b17523c 100644
--- a/harness/tests/flywheel_cli/test_flywheel.py
+++ b/harness/tests/flywheel_cli/test_flywheel.py
@@ -448,3 +448,55 @@ def test_parquet_keeps_the_complexity_columns_the_exporter_writes(tmp_path):
# and the schema can never fall behind row_for_sample again
with pytest.raises(ValueError, match="missing column"):
write_parquet([{**row, "a_new_metric": 1.0}], tmp_path / "later.parquet")
+
+
+def test_a_core_only_install_still_gets_a_complete_dataset(runs_dir: Path, tmp_path: Path, monkeypatch):
+ """PORT-5: `3dcv flywheel export` (no --pack) called write_parquet unconditionally,
+ so on an install without the optional `flywheel` extra it died with a raw
+ ModuleNotFoundError AFTER writing every sample folder — a partial dataset with no
+ index of any kind. docs/INSTALL.md §4 scopes pyarrow to the extra and promises the
+ harness only raises when the named feature is used, so a plain export must degrade:
+ metadata.jsonl always, metadata.parquet only when pyarrow is importable."""
+ import builtins
+
+ real_import = builtins.__import__
+
+ def no_pyarrow(name, *a, **kw):
+ if name == "pyarrow" or name.startswith("pyarrow."):
+ raise ModuleNotFoundError("No module named 'pyarrow'")
+ return real_import(name, *a, **kw)
+
+ monkeypatch.setattr(builtins, "__import__", no_pyarrow)
+ out = tmp_path / "ds_core"
+ rep = export_samples(runs_dir, out) # must NOT raise
+ assert rep.n_exported == 3 and rep.n_indexed == 3
+ # the index the operator can actually use is there, and complete
+ assert (out / "metadata.jsonl").is_file()
+ assert len((out / "metadata.jsonl").read_text().splitlines()) == 3
+ # the parquet is skipped, loudly and by name — never silently
+ assert not (out / "metadata.parquet").exists() and rep.parquet == ""
+ assert rep.notes and "pyarrow" in rep.notes[0] and "harness[flywheel]" in rep.notes[0]
+
+
+def test_pack_refuses_before_writing_anything_when_pyarrow_is_missing(runs_dir: Path, tmp_path: Path, monkeypatch):
+ """--pack genuinely needs pyarrow (pack.py reads the index back with pq.read_table),
+ so it must say so BEFORE the first sample is written, not after a few hundred."""
+ import builtins
+
+ from typer.testing import CliRunner
+
+ from codeverse.cli.main import app
+
+ real_import = builtins.__import__
+
+ def no_pyarrow(name, *a, **kw):
+ if name == "pyarrow" or name.startswith("pyarrow."):
+ raise ModuleNotFoundError("No module named 'pyarrow'")
+ return real_import(name, *a, **kw)
+
+ monkeypatch.setattr(builtins, "__import__", no_pyarrow)
+ out = tmp_path / "ds_pack"
+ res = CliRunner().invoke(app, ["flywheel", "export", str(runs_dir), str(out), "--pack"])
+ assert res.exit_code == 1
+ assert "harness[flywheel]" in res.output
+ assert not out.exists(), "no partial dataset may be left behind"