Official Codebase of 3DCodeVerse
Python
flywheel export: metadata.parquet silently dropped the complexity columns
gaoypeng5f39743feee4
diff --git a/harness/codeverse/flywheel/export.py b/harness/codeverse/flywheel/export.pyindex 6e530315..a2a5172b 100644--- a/harness/codeverse/flywheel/export.py+++ b/harness/codeverse/flywheel/export.py@@ -259,6 +259,8 @@ def parquet_schema() -> Any:("passed", pa.bool_()),("generator", pa.string()),("quality_tier", pa.string()),+ ("complexity", pa.float64()),+ ("complexity_band", pa.string()),("gate_errors", pa.int32()),("cost_usd", pa.float64()),("rounds", pa.int32()),@@ -272,10 +274,24 @@ def parquet_schema() -> Any:def write_parquet(rows: list[dict[str, Any]], path: Path) -> Path:+ """One row per sample, in :func:`parquet_schema` order.++ ``from_pylist(schema=...)`` DROPS any key the schema does not name, silently: that is+ how ``complexity`` / ``complexity_band`` — filled by :func:`row_for_sample`, kept in+ metadata.jsonl, and advertised in this module's docstring — went missing from the+ parquet, so every query of the dataset by complexity band returned nothing. The+ schema and the row keys are asserted equal here so a future column cannot vanish+ the same way.+ """import pyarrow as paimport pyarrow.parquet as pq- table = pa.Table.from_pylist(rows, schema=parquet_schema())+ schema = parquet_schema()+ if rows:+ missing = set(rows[0]) - set(schema.names)+ if missing:+ raise ValueError(f"parquet_schema() is missing column(s) written by row_for_sample: {sorted(missing)}")+ table = pa.Table.from_pylist(rows, schema=schema)tmp = path.with_suffix(".parquet.tmp")pq.write_table(table, tmp, compression="zstd")tmp.replace(path)diff --git a/harness/tests/flywheel_cli/test_flywheel.py b/harness/tests/flywheel_cli/test_flywheel.pyindex c29e98bf..0541e71d 100644--- a/harness/tests/flywheel_cli/test_flywheel.py+++ b/harness/tests/flywheel_cli/test_flywheel.py@@ -395,3 +395,24 @@ def test_a_genuinely_empty_runs_root_is_still_just_empty(tmp_path):empty = tmp_path / "runs"empty.mkdir()assert list(iter_runs(empty)) == []+++def test_parquet_keeps_the_complexity_columns_the_exporter_writes(tmp_path):+ """RS-9: ``pa.Table.from_pylist(rows, schema=...)`` DROPS any key the schema does not+ name, without a warning. ``row_for_sample`` fills complexity / complexity_band,+ metadata.jsonl keeps them and the module docstring lists them among the parquet's+ queryable extras — but parquet_schema() had no such fields, so every query of the+ shipped dataset (STORAGE_RULES §4) by complexity band silently returned nothing."""+ import pyarrow.parquet as pq++ from codeverse.flywheel.export import parquet_schema, write_parquet++ row = dict.fromkeys(parquet_schema().names)+ row.update(id="x", key="k", complexity=7.5, complexity_band="high")+ table = pq.read_table(write_parquet([row], tmp_path / "metadata.parquet"))+ assert {"complexity", "complexity_band"} <= set(table.column_names)+ got = table.to_pylist()[0]+ assert got["complexity"] == 7.5 and got["complexity_band"] == "high"+ # and the schema can never fall behind row_for_sample again+ with pytest.raises(ValueError, match="missing column"):+ write_parquet([{**row, "a_new_metric": 1.0}], tmp_path / "later.parquet")