Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): refuse brotli-compressed parquet files (#123071)

Tom Piccirellod8571f7b5854
diff --git a/products/autoresearch/backend/inference/sandbox.py b/products/autoresearch/backend/inference/sandbox.py
index 705bb0775d32..0359662f2ed6 100644
--- a/products/autoresearch/backend/inference/sandbox.py
+++ b/products/autoresearch/backend/inference/sandbox.py
@@ -1105,9 +1105,17 @@ def _check_scores_footer(parquet: pq.ParquetFile, *, expected_rows: int) -> None
raise SandboxInferenceError(f"scores.parquet distinct_id must be a string column, got {id_type}")
if not (pa.types.is_floating(p_type) or pa.types.is_integer(p_type)):
raise SandboxInferenceError(f"scores.parquet p_y must be a numeric column, got {p_type}")
+ metadata = parquet.metadata
+ # pyarrow decodes Brotli with a bundled copy of the C library that CVE-2026-111497 affects, and
+ # predict.py picks the codec. Remove this check when pyarrow bundles Brotli 1.2.1 or later.
+ if any(
+ metadata.row_group(g).column(c).compression == "BROTLI"
+ for g in range(metadata.num_row_groups)
+ for c in range(metadata.num_columns)
+ ):
+ raise SandboxInferenceError("scores.parquet must not use Brotli compression; write it with the default codec")
# The compressed file passed the readback cap; the decoded columns must fit it too, or
# a highly compressible table expands in the worker before any value is checked.
- metadata = parquet.metadata
decoded_bytes = sum(
metadata.row_group(g).column(c).total_uncompressed_size
for g in range(metadata.num_row_groups)
diff --git a/products/autoresearch/backend/inference/test_sandbox_inference.py b/products/autoresearch/backend/inference/test_sandbox_inference.py
index 47aa4ebc4a48..030f871b72d3 100644
--- a/products/autoresearch/backend/inference/test_sandbox_inference.py
+++ b/products/autoresearch/backend/inference/test_sandbox_inference.py
@@ -52,9 +52,11 @@ from products.tasks.backend.facade.sandbox import ExecutionResult
_VALID_FEATURE_SQL = "SELECT a.person_id AS distinct_id, count() AS events_total FROM {anchors} a GROUP BY a.person_id"
-def _scores_parquet(rows: list[tuple[str, object]]) -> bytes:
+def _scores_parquet(rows: list[tuple[str, object]], compression: str = "snappy") -> bytes:
buf = io.BytesIO()
- pd.DataFrame({"distinct_id": [r[0] for r in rows], "p_y": [r[1] for r in rows]}).to_parquet(buf, index=False)
+ pd.DataFrame({"distinct_id": [r[0] for r in rows], "p_y": [r[1] for r in rows]}).to_parquet(
+ buf, index=False, compression=compression
+ )
return buf.getvalue()
@@ -492,6 +494,11 @@ class TestFileReadback(SimpleTestCase):
with self.assertRaises(SandboxInferenceError):
_read_scores(fake, expected_rows=1)
+ def test_read_scores_rejects_brotli_compression_before_decoding(self):
+ fake = _FakeSandbox(scores_parquet=_scores_parquet([("s1", 0.8)], compression="brotli"))
+ with self.assertRaisesRegex(SandboxInferenceError, "Brotli"):
+ _read_scores(fake, expected_rows=1)
+
def test_join_scores_fails_when_predictions_omit_rows(self):
# A user missing from scores.parquet must fail the run — skipping them would
# advance last_scored_at while silently leaving the user unscored.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/aws_cost_and_usage_report.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/aws_cost_and_usage_report.py
index ff920e6dbdd6..0c4036f179d7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/aws_cost_and_usage_report.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/aws_cost_and_usage_report.py
@@ -44,6 +44,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.fil
iter_file_rows,
normalize_column_name,
resolve_file_format,
+ uses_brotli_compression,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.interruptible_wait import (
@@ -60,6 +61,7 @@ DOWNLOAD_ATTEMPTS = 3
NO_MANIFEST = "NoManifest"
DELIVERY_IN_PROGRESS = "DeliveryInProgress"
MANIFEST_UNREADABLE = "ManifestUnreadable"
+UNSUPPORTED_COMPRESSION = "UnsupportedCompression"
BILLING_PERIOD_COLUMN = "billing_period"
ERROR_PREFIX = "AWS Cost and Usage Report request failed"
WRONG_REGION_CODES = frozenset(
@@ -497,6 +499,11 @@ def iter_data_file(
) from error
if key.endswith(".parquet"):
parquet_file = pq.ParquetFile(destination)
+ if uses_brotli_compression(parquet_file.metadata):
+ raise AwsCostAndUsageReportError(
+ f"{ERROR_PREFIX}: {UNSUPPORTED_COMPRESSION} - {key} uses Brotli compression, which PostHog can't read",
+ UNSUPPORTED_COMPRESSION,
+ )
for batch in parquet_file.iter_batches(batch_size=ROWS_PER_BATCH):
table = pa.Table.from_batches([batch])
if table.num_rows:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/source.py
index 8922f293da2c..822792b812f2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/source.py
@@ -12,6 +12,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.aws_cost_a
DELIVERY_IN_PROGRESS,
ERROR_PREFIX,
MANIFEST_UNREADABLE,
+ UNSUPPORTED_COMPRESSION,
VALIDATION_ERROR_MESSAGES,
WRONG_REGION_CODES,
WRONG_REGION_MESSAGE,
@@ -74,6 +75,10 @@ class AwsCostAndUsageReportSource(ResumableSource[AwsCostAndUsageReportSourceCon
errors["Enter the first billing period as YYYY-MM"] = (
"The first billing period on the source is not in YYYY-MM form. Fix it and retry the sync."
)
+ errors[f"{ERROR_PREFIX}: {UNSUPPORTED_COMPRESSION}"] = (
+ "A file in the export uses Brotli compression, which PostHog can't read. "
+ "Check that only AWS writes to the export's S3 prefix, then retry the sync."
+ )
return errors
def get_retryable_errors(self) -> set[str]:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/tests/test_aws_cost_and_usage_report.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/tests/test_aws_cost_and_usage_report.py
index ada41662da9b..71564fbaf2f7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/tests/test_aws_cost_and_usage_report.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/tests/test_aws_cost_and_usage_report.py
@@ -2,7 +2,7 @@ import io
import gzip
import datetime as dt
from collections.abc import Callable, Mapping, Sequence
-from typing import Any
+from typing import Any, Literal
from urllib.parse import unquote
import pytest
@@ -80,7 +80,12 @@ def s3_error(code: str, status: int, headers: Mapping[str, str] | None = None) -
)
-def parquet_file(rows: list[dict[str, Any]]) -> ScriptedResponse:
+ParquetCompression = Literal["gzip", "bz2", "brotli", "lz4", "zstd", "snappy", "none"]
+
+
+def parquet_file(
+ rows: list[dict[str, Any]], compression: ParquetCompression | dict[str, ParquetCompression] = "snappy"
+) -> ScriptedResponse:
table = pa.table(
{
"identity_line_item_id": [row["id"] for row in rows],
@@ -89,7 +94,7 @@ def parquet_file(rows: list[dict[str, Any]]) -> ScriptedResponse:
}
)
buffer = io.BytesIO()
- pq.write_table(table, buffer)
+ pq.write_table(table, buffer, compression=compression)
return ScriptedResponse(body=buffer.getvalue(), headers={"Content-Type": "application/octet-stream"})
@@ -308,6 +313,29 @@ class TestSync:
assert download.headers["if-unmodified-since"] == manifest_written
assert str(result.raised).startswith("AWS Cost and Usage Report request failed: DeliveryInProgress")
+ def test_a_parquet_file_with_a_brotli_column_stops_the_run_before_any_row_is_yielded(self) -> None:
+ key = f"{BASE}/data/BILLING_PERIOD=2024-03/my-export-00001.snappy.parquet"
+ result = SourceDriver(AwsCostAndUsageReportSource(), config()).run(
+ "cost_and_usage",
+ s3(
+ {f"{BASE}/data/": [listing(prefixes=[f"{BASE}/data/BILLING_PERIOD=2024-03/"])]},
+ {
+ f"{BASE}/metadata/BILLING_PERIOD=2024-03/my-export-Manifest.json": ScriptedResponse(
+ json={"dataFiles": [key]}
+ ),
+ key: parquet_file(
+ [{"id": "x", "cost": 1.0, "tags": None}], compression={"line_item_unblended_cost": "brotli"}
+ ),
+ },
+ ),
+ )
+
+ assert str(result.raised).startswith("AWS Cost and Usage Report request failed: UnsupportedCompression")
+ assert table_rows(result.items) == []
+ assert any(
+ str(result.raised).startswith(prefix) for prefix in AwsCostAndUsageReportSource().get_non_retryable_errors()
+ )
+
def test_a_run_mixing_parquet_and_csv_periods_yields_arrow_tables_only(self) -> None:
march = f"{BASE}/data/BILLING_PERIOD=2024-03/my-export-00001.csv.gz"
april = f"{BASE}/data/BILLING_PERIOD=2024-04/my-export-00001.snappy.parquet"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/file_parsing.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/file_parsing.py
index c258cc97b52e..29f6c8941786 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/file_parsing.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/file_parsing.py
@@ -6,10 +6,13 @@ import json
import posixpath
import dataclasses
from collections.abc import Iterator
-from typing import IO, Any, Literal, cast
+from typing import IO, TYPE_CHECKING, Any, Literal, cast
from structlog.types import FilteringBoundLogger
+if TYPE_CHECKING:
+ import pyarrow.parquet as pq
+
FileFormat = Literal["csv", "jsonl", "json"]
ConfiguredFileFormat = Literal["infer", "csv", "jsonl", "json"]
@@ -122,6 +125,17 @@ def dedupe_headers(headers: list[str]) -> list[str]:
return result
+def uses_brotli_compression(metadata: "pq.FileMetaData") -> bool:
+ # pyarrow decodes Brotli with a bundled copy of the C library that CVE-2026-111497 affects.
+ # A Parquet file sets the codec of each column chunk, so a file from user storage must not
+ # reach that decoder. Remove this check when pyarrow bundles Brotli 1.2.1 or later.
+ return any(
+ metadata.row_group(group).column(column).compression == "BROTLI"
+ for group in range(metadata.num_row_groups)
+ for column in range(metadata.num_columns)
+ )
+
+
def _decompressed_stream(stream: IO[bytes], compressed: bool) -> IO[bytes]:
return cast(IO[bytes], gzip.GzipFile(fileobj=stream)) if compressed else stream
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py
index ff92fe7aff30..1e7f56375ffb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py
@@ -1,6 +1,6 @@
import re
import dataclasses
-from collections.abc import Iterator
+from collections.abc import Iterable, Iterator
from contextlib import contextmanager
from typing import ClassVar, cast
@@ -12,7 +12,10 @@ from structlog.types import FilteringBoundLogger
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.file_parsing import FILE_PATH_COLUMN
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.file_parsing import (
+ FILE_PATH_COLUMN,
+ uses_brotli_compression,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (
ACCESS_DENIED_ERROR,
DELTA_UNSUPPORTED_ERROR,
@@ -112,6 +115,7 @@ class DeltaRead:
yielded = False
try:
for file_name, scan_fragments in scans:
+ self._refuse_brotli(scan_fragments if scan_fragments is not None else self.dataset.get_fragments())
for batch in self._scanner(scan_fragments, batch_rows).to_batches():
if batch.num_rows:
yielded = True
@@ -129,6 +133,15 @@ class DeltaRead:
# never recreated.
yield self._tagged(self.dataset.schema.empty_table(), with_version, with_file_names, None)
+ def _refuse_brotli(self, fragments: Iterable[ds.Fragment]) -> None:
+ for fragment in fragments:
+ parquet_fragment = cast(ds.ParquetFileFragment, fragment)
+ if uses_brotli_compression(parquet_fragment.metadata):
+ raise ObjectStorageError(
+ f"{FILE_PARSE_ERROR} '{parquet_fragment.path}' in the Delta table '{self.table_name}': Brotli "
+ "compression isn't supported. Write the table with Snappy, Gzip, or Zstd compression instead."
+ )
+
def _scanner(self, fragments: list[ds.FileFragment] | None, batch_rows: int) -> ds.Scanner:
dataset: ds.Dataset = self.dataset
if fragments is not None:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py
index 782e4ff6c079..ffc468836a4e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py
@@ -16,6 +16,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arr
from products.warehouse_sources.backend.temporal.data_imports.sources.common.file_parsing import (
FILE_MODIFIED_AT_COLUMN,
FILE_PATH_COLUMN,
+ uses_brotli_compression,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (
CsvOptions,
@@ -285,6 +286,11 @@ def iter_parquet_tables(
) -> Iterator[pa.Table]:
try:
parquet_file = pq.ParquetFile(stream)
+ if uses_brotli_compression(parquet_file.metadata):
+ raise ObjectStorageError(
+ f"{FILE_PARSE_ERROR} '{file_path}' as Parquet: Brotli compression isn't supported. "
+ "Write the file with Snappy, Gzip, or Zstd compression instead."
+ )
yielded = False
for batch in parquet_file.iter_batches(batch_size=batch_rows, use_threads=False):
if batch.num_rows:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
index 8b92bdd8e84c..7efcfb01d6b6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
@@ -17,7 +17,7 @@ import pyarrow as pa
import structlog
import pyarrow.parquet as pq
from botocore.stub import Stubber
-from deltalake import DeltaTable, write_deltalake
+from deltalake import DeltaTable, WriterProperties, write_deltalake
from fsspec.implementations.local import LocalFileSystem
from parameterized import parameterized
@@ -626,6 +626,19 @@ class TestReadTable:
with pytest.raises(ObjectStorageError, match=FILE_PARSE_ERROR):
read_rows(store, table)
+ @parameterized.expand([("plain", "orders.parquet", False), ("gzipped", "orders.parquet.gz", True)])
+ def test_a_parquet_file_with_one_brotli_column_is_a_parse_error(self, _name: str, path: str, gzipped: bool) -> None:
+ buffer = io.BytesIO()
+ pq.write_table(
+ pa.table({"id": ["1", "2"], "note": ["a", "b"]}),
+ buffer,
+ compression={"id": "snappy", "note": "brotli"},
+ )
+ content = gzip.compress(buffer.getvalue()) if gzipped else buffer.getvalue()
+
+ with pytest.raises(ObjectStorageError, match=f"{FILE_PARSE_ERROR} '{path}' as Parquet: Brotli"):
+ list(iter_file_tables(io.BytesIO(content), stored(path), "parquet", CsvOptions()))
+
def test_a_parquet_file_with_no_rows_still_yields_its_schema(self, tmp_path: Path) -> None:
pq.write_table(pa.table({"id": pa.array([], pa.string())}), tmp_path / "empty.parquet")
store = local_store(tmp_path)
@@ -668,6 +681,17 @@ class TestReadTable:
assert [(table.column_names, table.num_rows) for table in tables] == [(["id"], 0)]
+ @parameterized.expand([("whole_snapshot", False), ("by_file", True)])
+ def test_a_delta_table_with_brotli_compression_is_a_parse_error(self, _name: str, with_file_names: bool) -> None:
+ with tempfile.TemporaryDirectory() as folder:
+ path = str(Path(folder) / "sales")
+ write_deltalake(path, pa.table({"id": [1, 2]}), writer_properties=WriterProperties(compression="BROTLI"))
+
+ read = plan_delta_read(path, {}, "sales")
+
+ with pytest.raises(ObjectStorageError, match=f"{FILE_PARSE_ERROR} '.+' in the Delta table 'sales': Brotli"):
+ list(read.iter_tables(with_file_names=with_file_names))
+
def _append(path: str) -> None:
write_deltalake(path, pa.table({"id": [3], "day": ["b"]}), mode="append")