:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse-sources): parse-fail a corrupt uploaded parquet file instead of retrying it (#129269)
Tom Owersdea20dbcb200
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.pyindex 75a31846e6c3..14c28a1a48aa 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py@@ -338,6 +338,15 @@ def iter_parquet_tables(yield parquet_file.schema_arrow.empty_table()except pa.ArrowInvalid as e:raise ObjectStorageError(f"{FILE_PARSE_ERROR} '{file_path}' as Parquet: {e}") from e+ except OSError as e:+ # pyarrow's own Parquet decoder raises a bare OSError (pyarrow.ArrowIOError is an alias for+ # it) for a corrupt or malformed file, e.g. a data page whose block structure fails its+ # internal validation. A failure reading the underlying stream keeps its own specific+ # OSError subclass (ConnectionError, TimeoutError, ...) instead, so only the exact base+ # type means the file itself is bad rather than a transient read.+ if type(e) is not OSError:+ raise+ raise ObjectStorageError(f"{FILE_PARSE_ERROR} '{file_path}' as Parquet: {e}") from edef with_file_columns(table: pa.Table, stored: StoredObject, first_row_number: int = 0) -> pa.Table:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.pyindex 24aca3b7ffd1..e7d1b0786e12 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py@@ -716,6 +716,29 @@ class TestReadTable:with pytest.raises(ObjectStorageError, match=f"{FILE_PARSE_ERROR} '{path}' as Parquet: Brotli"):list(iter_file_tables(io.BytesIO(content), stored(path), "parquet", CsvOptions()))+ def test_a_corrupt_parquet_block_is_a_parse_error(self) -> None:+ buffer = io.BytesIO()+ pq.write_table(pa.table({"id": ["1", "2"]}), buffer)+ buffer.seek(0)+ # pyarrow's own decoder raises a bare OSError (not a subclass) for a page that fails its+ # internal validation, e.g. a corrupt block. Real file content can't be made to fail that+ # validation deterministically, so the decoder itself is the mocked boundary.+ with (+ patch.object(pq.ParquetFile, "iter_batches", side_effect=OSError("cannot have zero value per block")),+ pytest.raises(ObjectStorageError, match=f"{FILE_PARSE_ERROR} 'orders.parquet' as Parquet"),+ ):+ list(iter_file_tables(buffer, stored("orders.parquet"), "parquet", CsvOptions()))++ def test_a_transient_read_error_while_streaming_parquet_batches_is_not_a_parse_error(self) -> None:+ buffer = io.BytesIO()+ pq.write_table(pa.table({"id": ["1", "2"]}), buffer)+ buffer.seek(0)+ with (+ patch.object(pq.ParquetFile, "iter_batches", side_effect=ConnectionError("connection reset")),+ pytest.raises(ConnectionError),+ ):+ list(iter_file_tables(buffer, stored("orders.parquet"), "parquet", CsvOptions()))+def test_a_parquet_file_with_no_rows_still_yields_its_schema(self, tmp_path: Path) -> None:pq.write_table(pa.table({"id": pa.array([], pa.string())}), tmp_path / "empty.parquet")store = local_store(tmp_path)