Skip to content

Official Codebase of 3DCodeVerse

Python

flywheel export: metadata.parquet silently dropped the complexity columns

gaoypeng5f39743feee4
diff --git a/harness/codeverse/flywheel/export.py b/harness/codeverse/flywheel/export.py
index 6e530315..a2a5172b 100644
--- a/harness/codeverse/flywheel/export.py
+++ b/harness/codeverse/flywheel/export.py
@@ -259,6 +259,8 @@ def parquet_schema() -> Any:
("passed", pa.bool_()),
("generator", pa.string()),
("quality_tier", pa.string()),
+ ("complexity", pa.float64()),
+ ("complexity_band", pa.string()),
("gate_errors", pa.int32()),
("cost_usd", pa.float64()),
("rounds", pa.int32()),
@@ -272,10 +274,24 @@ def parquet_schema() -> Any:
def write_parquet(rows: list[dict[str, Any]], path: Path) -> Path:
+ """One row per sample, in :func:`parquet_schema` order.
+
+ ``from_pylist(schema=...)`` DROPS any key the schema does not name, silently: that is
+ how ``complexity`` / ``complexity_band`` — filled by :func:`row_for_sample`, kept in
+ metadata.jsonl, and advertised in this module's docstring — went missing from the
+ parquet, so every query of the dataset by complexity band returned nothing. The
+ schema and the row keys are asserted equal here so a future column cannot vanish
+ the same way.
+ """
import pyarrow as pa
import pyarrow.parquet as pq
- table = pa.Table.from_pylist(rows, schema=parquet_schema())
+ schema = parquet_schema()
+ if rows:
+ missing = set(rows[0]) - set(schema.names)
+ if missing:
+ raise ValueError(f"parquet_schema() is missing column(s) written by row_for_sample: {sorted(missing)}")
+ table = pa.Table.from_pylist(rows, schema=schema)
tmp = path.with_suffix(".parquet.tmp")
pq.write_table(table, tmp, compression="zstd")
tmp.replace(path)
diff --git a/harness/tests/flywheel_cli/test_flywheel.py b/harness/tests/flywheel_cli/test_flywheel.py
index c29e98bf..0541e71d 100644
--- a/harness/tests/flywheel_cli/test_flywheel.py
+++ b/harness/tests/flywheel_cli/test_flywheel.py
@@ -395,3 +395,24 @@ def test_a_genuinely_empty_runs_root_is_still_just_empty(tmp_path):
empty = tmp_path / "runs"
empty.mkdir()
assert list(iter_runs(empty)) == []
+
+
+def test_parquet_keeps_the_complexity_columns_the_exporter_writes(tmp_path):
+ """RS-9: ``pa.Table.from_pylist(rows, schema=...)`` DROPS any key the schema does not
+ name, without a warning. ``row_for_sample`` fills complexity / complexity_band,
+ metadata.jsonl keeps them and the module docstring lists them among the parquet's
+ queryable extras — but parquet_schema() had no such fields, so every query of the
+ shipped dataset (STORAGE_RULES §4) by complexity band silently returned nothing."""
+ import pyarrow.parquet as pq
+
+ from codeverse.flywheel.export import parquet_schema, write_parquet
+
+ row = dict.fromkeys(parquet_schema().names)
+ row.update(id="x", key="k", complexity=7.5, complexity_band="high")
+ table = pq.read_table(write_parquet([row], tmp_path / "metadata.parquet"))
+ assert {"complexity", "complexity_band"} <= set(table.column_names)
+ got = table.to_pylist()[0]
+ assert got["complexity"] == 7.5 and got["complexity_band"] == "high"
+ # and the schema can never fall behind row_for_sample again
+ with pytest.raises(ValueError, match="missing column"):
+ write_parquet([{**row, "a_new_metric": 1.0}], tmp_path / "later.parquet")