Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): parse-fail a corrupt uploaded parquet file instead of retrying it (#129269)

Tom Owersdea20dbcb200
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py
index 75a31846e6c3..14c28a1a48aa 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py
@@ -338,6 +338,15 @@ def iter_parquet_tables(
yield parquet_file.schema_arrow.empty_table()
except pa.ArrowInvalid as e:
raise ObjectStorageError(f"{FILE_PARSE_ERROR} '{file_path}' as Parquet: {e}") from e
+ except OSError as e:
+ # pyarrow's own Parquet decoder raises a bare OSError (pyarrow.ArrowIOError is an alias for
+ # it) for a corrupt or malformed file, e.g. a data page whose block structure fails its
+ # internal validation. A failure reading the underlying stream keeps its own specific
+ # OSError subclass (ConnectionError, TimeoutError, ...) instead, so only the exact base
+ # type means the file itself is bad rather than a transient read.
+ if type(e) is not OSError:
+ raise
+ raise ObjectStorageError(f"{FILE_PARSE_ERROR} '{file_path}' as Parquet: {e}") from e
def with_file_columns(table: pa.Table, stored: StoredObject, first_row_number: int = 0) -> pa.Table:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
index 24aca3b7ffd1..e7d1b0786e12 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
@@ -716,6 +716,29 @@ class TestReadTable:
with pytest.raises(ObjectStorageError, match=f"{FILE_PARSE_ERROR} '{path}' as Parquet: Brotli"):
list(iter_file_tables(io.BytesIO(content), stored(path), "parquet", CsvOptions()))
+ def test_a_corrupt_parquet_block_is_a_parse_error(self) -> None:
+ buffer = io.BytesIO()
+ pq.write_table(pa.table({"id": ["1", "2"]}), buffer)
+ buffer.seek(0)
+ # pyarrow's own decoder raises a bare OSError (not a subclass) for a page that fails its
+ # internal validation, e.g. a corrupt block. Real file content can't be made to fail that
+ # validation deterministically, so the decoder itself is the mocked boundary.
+ with (
+ patch.object(pq.ParquetFile, "iter_batches", side_effect=OSError("cannot have zero value per block")),
+ pytest.raises(ObjectStorageError, match=f"{FILE_PARSE_ERROR} 'orders.parquet' as Parquet"),
+ ):
+ list(iter_file_tables(buffer, stored("orders.parquet"), "parquet", CsvOptions()))
+
+ def test_a_transient_read_error_while_streaming_parquet_batches_is_not_a_parse_error(self) -> None:
+ buffer = io.BytesIO()
+ pq.write_table(pa.table({"id": ["1", "2"]}), buffer)
+ buffer.seek(0)
+ with (
+ patch.object(pq.ParquetFile, "iter_batches", side_effect=ConnectionError("connection reset")),
+ pytest.raises(ConnectionError),
+ ):
+ list(iter_file_tables(buffer, stored("orders.parquet"), "parquet", CsvOptions()))
+
def test_a_parquet_file_with_no_rows_still_yields_its_schema(self, tmp_path: Path) -> None:
pq.write_table(pa.table({"id": pa.array([], pa.string())}), tmp_path / "empty.parquet")
store = local_store(tmp_path)