:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse-sources): refuse brotli-compressed parquet files (#123071)
Tom Piccirellod8571f7b5854
diff --git a/products/autoresearch/backend/inference/sandbox.py b/products/autoresearch/backend/inference/sandbox.pyindex 705bb0775d32..0359662f2ed6 100644--- a/products/autoresearch/backend/inference/sandbox.py+++ b/products/autoresearch/backend/inference/sandbox.py@@ -1105,9 +1105,17 @@ def _check_scores_footer(parquet: pq.ParquetFile, *, expected_rows: int) -> Noneraise SandboxInferenceError(f"scores.parquet distinct_id must be a string column, got {id_type}")if not (pa.types.is_floating(p_type) or pa.types.is_integer(p_type)):raise SandboxInferenceError(f"scores.parquet p_y must be a numeric column, got {p_type}")+ metadata = parquet.metadata+ # pyarrow decodes Brotli with a bundled copy of the C library that CVE-2026-111497 affects, and+ # predict.py picks the codec. Remove this check when pyarrow bundles Brotli 1.2.1 or later.+ if any(+ metadata.row_group(g).column(c).compression == "BROTLI"+ for g in range(metadata.num_row_groups)+ for c in range(metadata.num_columns)+ ):+ raise SandboxInferenceError("scores.parquet must not use Brotli compression; write it with the default codec")# The compressed file passed the readback cap; the decoded columns must fit it too, or# a highly compressible table expands in the worker before any value is checked.- metadata = parquet.metadatadecoded_bytes = sum(metadata.row_group(g).column(c).total_uncompressed_sizefor g in range(metadata.num_row_groups)diff --git a/products/autoresearch/backend/inference/test_sandbox_inference.py b/products/autoresearch/backend/inference/test_sandbox_inference.pyindex 47aa4ebc4a48..030f871b72d3 100644--- a/products/autoresearch/backend/inference/test_sandbox_inference.py+++ b/products/autoresearch/backend/inference/test_sandbox_inference.py@@ -52,9 +52,11 @@ from products.tasks.backend.facade.sandbox import ExecutionResult_VALID_FEATURE_SQL = "SELECT a.person_id AS distinct_id, count() AS events_total FROM {anchors} a GROUP BY a.person_id"-def _scores_parquet(rows: list[tuple[str, object]]) -> bytes:+def _scores_parquet(rows: list[tuple[str, object]], compression: str = "snappy") -> bytes:buf = io.BytesIO()- pd.DataFrame({"distinct_id": [r[0] for r in rows], "p_y": [r[1] for r in rows]}).to_parquet(buf, index=False)+ pd.DataFrame({"distinct_id": [r[0] for r in rows], "p_y": [r[1] for r in rows]}).to_parquet(+ buf, index=False, compression=compression+ )return buf.getvalue()@@ -492,6 +494,11 @@ class TestFileReadback(SimpleTestCase):with self.assertRaises(SandboxInferenceError):_read_scores(fake, expected_rows=1)+ def test_read_scores_rejects_brotli_compression_before_decoding(self):+ fake = _FakeSandbox(scores_parquet=_scores_parquet([("s1", 0.8)], compression="brotli"))+ with self.assertRaisesRegex(SandboxInferenceError, "Brotli"):+ _read_scores(fake, expected_rows=1)+def test_join_scores_fails_when_predictions_omit_rows(self):# A user missing from scores.parquet must fail the run — skipping them would# advance last_scored_at while silently leaving the user unscored.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/aws_cost_and_usage_report.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/aws_cost_and_usage_report.pyindex ff920e6dbdd6..0c4036f179d7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/aws_cost_and_usage_report.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/aws_cost_and_usage_report.py@@ -44,6 +44,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.filiter_file_rows,normalize_column_name,resolve_file_format,+ uses_brotli_compression,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_sessionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.interruptible_wait import (@@ -60,6 +61,7 @@ DOWNLOAD_ATTEMPTS = 3NO_MANIFEST = "NoManifest"DELIVERY_IN_PROGRESS = "DeliveryInProgress"MANIFEST_UNREADABLE = "ManifestUnreadable"+UNSUPPORTED_COMPRESSION = "UnsupportedCompression"BILLING_PERIOD_COLUMN = "billing_period"ERROR_PREFIX = "AWS Cost and Usage Report request failed"WRONG_REGION_CODES = frozenset(@@ -497,6 +499,11 @@ def iter_data_file() from errorif key.endswith(".parquet"):parquet_file = pq.ParquetFile(destination)+ if uses_brotli_compression(parquet_file.metadata):+ raise AwsCostAndUsageReportError(+ f"{ERROR_PREFIX}: {UNSUPPORTED_COMPRESSION} - {key} uses Brotli compression, which PostHog can't read",+ UNSUPPORTED_COMPRESSION,+ )for batch in parquet_file.iter_batches(batch_size=ROWS_PER_BATCH):table = pa.Table.from_batches([batch])if table.num_rows:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/source.pyindex 8922f293da2c..822792b812f2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/source.py@@ -12,6 +12,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.aws_cost_aDELIVERY_IN_PROGRESS,ERROR_PREFIX,MANIFEST_UNREADABLE,+ UNSUPPORTED_COMPRESSION,VALIDATION_ERROR_MESSAGES,WRONG_REGION_CODES,WRONG_REGION_MESSAGE,@@ -74,6 +75,10 @@ class AwsCostAndUsageReportSource(ResumableSource[AwsCostAndUsageReportSourceConerrors["Enter the first billing period as YYYY-MM"] = ("The first billing period on the source is not in YYYY-MM form. Fix it and retry the sync.")+ errors[f"{ERROR_PREFIX}: {UNSUPPORTED_COMPRESSION}"] = (+ "A file in the export uses Brotli compression, which PostHog can't read. "+ "Check that only AWS writes to the export's S3 prefix, then retry the sync."+ )return errorsdef get_retryable_errors(self) -> set[str]:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/tests/test_aws_cost_and_usage_report.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/tests/test_aws_cost_and_usage_report.pyindex ada41662da9b..71564fbaf2f7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/tests/test_aws_cost_and_usage_report.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/tests/test_aws_cost_and_usage_report.py@@ -2,7 +2,7 @@ import ioimport gzipimport datetime as dtfrom collections.abc import Callable, Mapping, Sequence-from typing import Any+from typing import Any, Literalfrom urllib.parse import unquoteimport pytest@@ -80,7 +80,12 @@ def s3_error(code: str, status: int, headers: Mapping[str, str] | None = None) -)-def parquet_file(rows: list[dict[str, Any]]) -> ScriptedResponse:+ParquetCompression = Literal["gzip", "bz2", "brotli", "lz4", "zstd", "snappy", "none"]+++def parquet_file(+ rows: list[dict[str, Any]], compression: ParquetCompression | dict[str, ParquetCompression] = "snappy"+) -> ScriptedResponse:table = pa.table({"identity_line_item_id": [row["id"] for row in rows],@@ -89,7 +94,7 @@ def parquet_file(rows: list[dict[str, Any]]) -> ScriptedResponse:})buffer = io.BytesIO()- pq.write_table(table, buffer)+ pq.write_table(table, buffer, compression=compression)return ScriptedResponse(body=buffer.getvalue(), headers={"Content-Type": "application/octet-stream"})@@ -308,6 +313,29 @@ class TestSync:assert download.headers["if-unmodified-since"] == manifest_writtenassert str(result.raised).startswith("AWS Cost and Usage Report request failed: DeliveryInProgress")+ def test_a_parquet_file_with_a_brotli_column_stops_the_run_before_any_row_is_yielded(self) -> None:+ key = f"{BASE}/data/BILLING_PERIOD=2024-03/my-export-00001.snappy.parquet"+ result = SourceDriver(AwsCostAndUsageReportSource(), config()).run(+ "cost_and_usage",+ s3(+ {f"{BASE}/data/": [listing(prefixes=[f"{BASE}/data/BILLING_PERIOD=2024-03/"])]},+ {+ f"{BASE}/metadata/BILLING_PERIOD=2024-03/my-export-Manifest.json": ScriptedResponse(+ json={"dataFiles": [key]}+ ),+ key: parquet_file(+ [{"id": "x", "cost": 1.0, "tags": None}], compression={"line_item_unblended_cost": "brotli"}+ ),+ },+ ),+ )++ assert str(result.raised).startswith("AWS Cost and Usage Report request failed: UnsupportedCompression")+ assert table_rows(result.items) == []+ assert any(+ str(result.raised).startswith(prefix) for prefix in AwsCostAndUsageReportSource().get_non_retryable_errors()+ )+def test_a_run_mixing_parquet_and_csv_periods_yields_arrow_tables_only(self) -> None:march = f"{BASE}/data/BILLING_PERIOD=2024-03/my-export-00001.csv.gz"april = f"{BASE}/data/BILLING_PERIOD=2024-04/my-export-00001.snappy.parquet"diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/file_parsing.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/file_parsing.pyindex c258cc97b52e..29f6c8941786 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/file_parsing.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/file_parsing.py@@ -6,10 +6,13 @@ import jsonimport posixpathimport dataclassesfrom collections.abc import Iterator-from typing import IO, Any, Literal, cast+from typing import IO, TYPE_CHECKING, Any, Literal, castfrom structlog.types import FilteringBoundLogger+if TYPE_CHECKING:+ import pyarrow.parquet as pq+FileFormat = Literal["csv", "jsonl", "json"]ConfiguredFileFormat = Literal["infer", "csv", "jsonl", "json"]@@ -122,6 +125,17 @@ def dedupe_headers(headers: list[str]) -> list[str]:return result+def uses_brotli_compression(metadata: "pq.FileMetaData") -> bool:+ # pyarrow decodes Brotli with a bundled copy of the C library that CVE-2026-111497 affects.+ # A Parquet file sets the codec of each column chunk, so a file from user storage must not+ # reach that decoder. Remove this check when pyarrow bundles Brotli 1.2.1 or later.+ return any(+ metadata.row_group(group).column(column).compression == "BROTLI"+ for group in range(metadata.num_row_groups)+ for column in range(metadata.num_columns)+ )++def _decompressed_stream(stream: IO[bytes], compressed: bool) -> IO[bytes]:return cast(IO[bytes], gzip.GzipFile(fileobj=stream)) if compressed else streamdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.pyindex ff92fe7aff30..1e7f56375ffb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py@@ -1,6 +1,6 @@import reimport dataclasses-from collections.abc import Iterator+from collections.abc import Iterable, Iteratorfrom contextlib import contextmanagerfrom typing import ClassVar, cast@@ -12,7 +12,10 @@ from structlog.types import FilteringBoundLoggerfrom posthog.dataclasses import frozen-from products.warehouse_sources.backend.temporal.data_imports.sources.common.file_parsing import FILE_PATH_COLUMN+from products.warehouse_sources.backend.temporal.data_imports.sources.common.file_parsing import (+ FILE_PATH_COLUMN,+ uses_brotli_compression,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (ACCESS_DENIED_ERROR,DELTA_UNSUPPORTED_ERROR,@@ -112,6 +115,7 @@ class DeltaRead:yielded = Falsetry:for file_name, scan_fragments in scans:+ self._refuse_brotli(scan_fragments if scan_fragments is not None else self.dataset.get_fragments())for batch in self._scanner(scan_fragments, batch_rows).to_batches():if batch.num_rows:yielded = True@@ -129,6 +133,15 @@ class DeltaRead:# never recreated.yield self._tagged(self.dataset.schema.empty_table(), with_version, with_file_names, None)+ def _refuse_brotli(self, fragments: Iterable[ds.Fragment]) -> None:+ for fragment in fragments:+ parquet_fragment = cast(ds.ParquetFileFragment, fragment)+ if uses_brotli_compression(parquet_fragment.metadata):+ raise ObjectStorageError(+ f"{FILE_PARSE_ERROR} '{parquet_fragment.path}' in the Delta table '{self.table_name}': Brotli "+ "compression isn't supported. Write the table with Snappy, Gzip, or Zstd compression instead."+ )+def _scanner(self, fragments: list[ds.FileFragment] | None, batch_rows: int) -> ds.Scanner:dataset: ds.Dataset = self.datasetif fragments is not None:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.pyindex 782e4ff6c079..ffc468836a4e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py@@ -16,6 +16,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.file_parsing import (FILE_MODIFIED_AT_COLUMN,FILE_PATH_COLUMN,+ uses_brotli_compression,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (CsvOptions,@@ -285,6 +286,11 @@ def iter_parquet_tables() -> Iterator[pa.Table]:try:parquet_file = pq.ParquetFile(stream)+ if uses_brotli_compression(parquet_file.metadata):+ raise ObjectStorageError(+ f"{FILE_PARSE_ERROR} '{file_path}' as Parquet: Brotli compression isn't supported. "+ "Write the file with Snappy, Gzip, or Zstd compression instead."+ )yielded = Falsefor batch in parquet_file.iter_batches(batch_size=batch_rows, use_threads=False):if batch.num_rows:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.pyindex 8b92bdd8e84c..7efcfb01d6b6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py@@ -17,7 +17,7 @@ import pyarrow as paimport structlogimport pyarrow.parquet as pqfrom botocore.stub import Stubber-from deltalake import DeltaTable, write_deltalake+from deltalake import DeltaTable, WriterProperties, write_deltalakefrom fsspec.implementations.local import LocalFileSystemfrom parameterized import parameterized@@ -626,6 +626,19 @@ class TestReadTable:with pytest.raises(ObjectStorageError, match=FILE_PARSE_ERROR):read_rows(store, table)+ @parameterized.expand([("plain", "orders.parquet", False), ("gzipped", "orders.parquet.gz", True)])+ def test_a_parquet_file_with_one_brotli_column_is_a_parse_error(self, _name: str, path: str, gzipped: bool) -> None:+ buffer = io.BytesIO()+ pq.write_table(+ pa.table({"id": ["1", "2"], "note": ["a", "b"]}),+ buffer,+ compression={"id": "snappy", "note": "brotli"},+ )+ content = gzip.compress(buffer.getvalue()) if gzipped else buffer.getvalue()++ with pytest.raises(ObjectStorageError, match=f"{FILE_PARSE_ERROR} '{path}' as Parquet: Brotli"):+ list(iter_file_tables(io.BytesIO(content), stored(path), "parquet", CsvOptions()))+def test_a_parquet_file_with_no_rows_still_yields_its_schema(self, tmp_path: Path) -> None:pq.write_table(pa.table({"id": pa.array([], pa.string())}), tmp_path / "empty.parquet")store = local_store(tmp_path)@@ -668,6 +681,17 @@ class TestReadTable:assert [(table.column_names, table.num_rows) for table in tables] == [(["id"], 0)]+ @parameterized.expand([("whole_snapshot", False), ("by_file", True)])+ def test_a_delta_table_with_brotli_compression_is_a_parse_error(self, _name: str, with_file_names: bool) -> None:+ with tempfile.TemporaryDirectory() as folder:+ path = str(Path(folder) / "sales")+ write_deltalake(path, pa.table({"id": [1, 2]}), writer_properties=WriterProperties(compression="BROTLI"))++ read = plan_delta_read(path, {}, "sales")++ with pytest.raises(ObjectStorageError, match=f"{FILE_PARSE_ERROR} '.+' in the Delta table 'sales': Brotli"):+ list(read.iter_tables(with_file_names=with_file_names))+def _append(path: str) -> None:write_deltalake(path, pa.table({"id": [3], "day": ["b"]}), mode="append")