Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): read warehouse files in chdb with an explicit codec (#123070)

Felipe Rosa de Almeida71bb63814ea7
diff --git a/posthog/hogql/database/s3_table.py b/posthog/hogql/database/s3_table.py
index e42539cadd2a..66a042084135 100644
--- a/posthog/hogql/database/s3_table.py
+++ b/posthog/hogql/database/s3_table.py
@@ -1,4 +1,6 @@
import re
+import itertools
+from collections.abc import Iterator
from dataclasses import dataclass
from pathlib import PurePosixPath
from typing import Literal, Optional
@@ -24,6 +26,25 @@ _FORMAT_LABELS = {
"Delta": "Delta",
"DeltaS3Wrapper": "Delta",
}
+# Mirrors ClickHouse's chooseCompressionMethod.
+_COMPRESSION_BY_EXTENSION = {
+ "gz": "gzip",
+ "gzip": "gzip",
+ "deflate": "deflate",
+ "br": "br",
+ "brotli": "br",
+ "xz": "xz",
+ "lzma": "xz",
+ "zst": "zstd",
+ "zstd": "zstd",
+ "lz4": "lz4",
+ "bz2": "bz2",
+ "snappy": "snappy",
+}
+_URL_GLOB_CHARACTERS = frozenset("*?{}")
+_BROTLI_FILE_NAME_RE = re.compile(r"\.(?:br|brotli)(?![\w.-])", re.IGNORECASE)
+_BROTLI_EXTENSION_RE = re.compile(r"(?:br|brotli)[*?]*", re.IGNORECASE)
+_MAX_BRACE_EXPANSIONS = 1000
@dataclass(frozen=True, kw_only=True)
@@ -106,6 +127,33 @@ def parse_duckdb_s3_source(url: str) -> DuckDBS3Source | None:
)
+def compression_for_url_pattern(url: str) -> str | None:
+ """None when the pattern can match files with different extensions."""
+ _, dot, extension = url.rsplit("/", 1)[-1].rpartition(".")
+ if _URL_GLOB_CHARACTERS.intersection(extension):
+ return None
+ return _COMPRESSION_BY_EXTENSION.get(extension.lower(), "none") if dot else "none"
+
+
+def _brace_expansions(pattern: str) -> Iterator[str]:
+ match = re.search(r"\{([^{}]*)\}", pattern)
+ if match is None:
+ yield pattern
+ return
+ for alternative in match.group(1).split(","):
+ yield from _brace_expansions(pattern[: match.start()] + alternative + pattern[match.end() :])
+
+
+def _reads_brotli_files(url: str) -> bool:
+ urls = list(itertools.islice(_brace_expansions(url), _MAX_BRACE_EXPANSIONS + 1))
+ if len(urls) > _MAX_BRACE_EXPANSIONS:
+ return _BROTLI_FILE_NAME_RE.search(url) is not None
+ return any(
+ dot and _BROTLI_EXTENSION_RE.fullmatch(extension)
+ for _, dot, extension in (expanded.rsplit("/", 1)[-1].rpartition(".") for expanded in urls)
+ )
+
+
def build_function_call(
url: str,
format: str,
@@ -115,6 +163,7 @@ def build_function_call(
structure: Optional[str] = None,
context: Optional[HogQLContext] = None,
table_size_mib: Optional[float] = None,
+ compression: Optional[str] = None,
) -> str:
if access_key is None and access_secret is None and (settings.DEBUG or settings.TEST or settings.USE_LOCAL_SETUP):
access_key = settings.DATAWAREHOUSE_LOCAL_ACCESS_KEY
@@ -202,6 +251,11 @@ def build_function_call(
return return_expr(expr)
+ if _reads_brotli_files(url):
+ raise ExposedHogQLError(
+ "PostHog can't read Brotli-compressed (.br) files. Compress the files with gzip or zstd instead."
+ )
+
# Azure
if re.match(r"^https:\/\/.+\.blob\.core\.windows\.net\/", url):
regex_result = re.search(r"(https:\/\/.+\.blob\.core\.windows\.net)\/(.+?)\/(.*)", url)
@@ -222,8 +276,9 @@ def build_function_call(
escaped_access_key = add_param(access_key)
escaped_access_secret = add_param(access_secret)
escaped_format = add_param(format, False)
+ escaped_compression = add_param(compression, False) if compression else "'auto'"
- expr = f"azureBlobStorage({storage_account_url}, {container}, {blob_path}, {escaped_access_key}, {escaped_access_secret}, {escaped_format}, 'auto'"
+ expr = f"azureBlobStorage({storage_account_url}, {container}, {blob_path}, {escaped_access_key}, {escaped_access_secret}, {escaped_format}, {escaped_compression}"
if structure:
escaped_structure = add_param(structure, False)
@@ -252,6 +307,12 @@ def build_function_call(
if structure:
expr += f", {escaped_structure}"
+ elif compression:
+ # s3() only reads the codec after a structure argument.
+ expr += ", 'auto'"
+
+ if compression:
+ expr += f", {add_param(compression, False)}"
return return_expr(expr)
diff --git a/posthog/hogql/database/test/test_s3_table.py b/posthog/hogql/database/test/test_s3_table.py
index 3b950860d435..8f7956211fb6 100644
--- a/posthog/hogql/database/test/test_s3_table.py
+++ b/posthog/hogql/database/test/test_s3_table.py
@@ -4,13 +4,15 @@ from posthog.test.base import BaseTest
from unittest import mock
from django.conf import settings
-from django.test import override_settings
+from django.test import SimpleTestCase, override_settings
+
+from parameterized import parameterized
from posthog.hogql.constants import MAX_SELECT_RETURNED_ROWS
from posthog.hogql.context import HogQLContext
from posthog.hogql.database.database import Database
from posthog.hogql.database.models import TableNode
-from posthog.hogql.database.s3_table import build_function_call
+from posthog.hogql.database.s3_table import build_function_call, compression_for_url_pattern
from posthog.hogql.database.test.tables import create_aapl_stock_s3_table
from posthog.hogql.errors import ExposedHogQLError
from posthog.hogql.parser import parse_select
@@ -22,6 +24,82 @@ from posthog.models.event.sql import DISTRIBUTED_EVENTS_JSON_TABLE
from products.warehouse_sources.backend.facade.models import DataWarehouseTable
+class TestS3Compression(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("plain_file", "https://b.example.com/t/data.csv", "none"),
+ ("no_extension", "https://b.example.com/t/data", "none"),
+ ("folder", "https://b.example.com/t/", "none"),
+ ("pinned_glob", "https://b.example.com/t/*.parquet", "none"),
+ ("pinned_compressed_glob", "https://b.example.com/t/*.CSV.GZ", "gzip"),
+ ("braces_before_the_extension", "https://b.example.com/t/{a,b}.json.zst", "zstd"),
+ ("dotted_folder", "https://b.example.com/t.gz/data", "none"),
+ ("open_glob", "https://b.example.com/t/*", None),
+ ("recursive_glob", "https://b.example.com/t/**", None),
+ ("glob_after_the_stem", "https://b.example.com/t/data_*", None),
+ ("braces_in_the_extension", "https://b.example.com/t/*.{csv,csv.gz}", None),
+ ("question_mark_in_the_extension", "https://b.example.com/t/data.cs?", None),
+ ]
+ )
+ def test_compression_for_url_pattern(self, _name: str, url: str, expected: str | None) -> None:
+ assert compression_for_url_pattern(url) == expected
+
+ @parameterized.expand(
+ [
+ (
+ "s3_without_structure",
+ "https://b.example.com/t/*.csv.gz",
+ None,
+ "s3('https://b.example.com/t/*.csv.gz', 'key', 'secret', 'CSVWithNames', 'auto', 'gzip')",
+ ),
+ (
+ "s3_with_structure",
+ "https://b.example.com/t/*.csv.gz",
+ "id String",
+ "s3('https://b.example.com/t/*.csv.gz', 'key', 'secret', 'CSVWithNames', 'id String', 'gzip')",
+ ),
+ (
+ "azure",
+ "https://acct.blob.core.windows.net/c/t/*.csv.gz",
+ "id String",
+ "azureBlobStorage('https://acct.blob.core.windows.net', 'c', 't/*.csv.gz', 'key', 'secret', 'CSVWithNames', 'gzip', 'id String')",
+ ),
+ ]
+ )
+ def test_build_function_call_passes_an_explicit_codec(
+ self, _name: str, url: str, structure: str | None, expected: str
+ ) -> None:
+ res = build_function_call(
+ url, DataWarehouseTable.TableFormat.CSVWithNames, None, "key", "secret", structure, compression="gzip"
+ )
+ assert res == expected
+
+ @parameterized.expand(
+ [
+ ("pinned", "https://b.example.com/t/export.csv.br", True),
+ ("long_extension", "https://b.example.com/t/export.CSV.BROTLI", True),
+ ("brace_alternative", "https://b.example.com/t/*.{csv,csv.br}", True),
+ ("glob_after_the_extension", "https://b.example.com/t/*.br*", True),
+ ("brace_alternative_across_folders", "https://b.example.com/{a/x.csv,b/y.br}", True),
+ ("br_inside_a_brace_alternative", "https://b.example.com/t/{data.br,data.us}.csv", False),
+ ("too_many_brace_expansions", "https://b.example.com/t/" + "{a,b}" * 40 + ".csv.br", True),
+ ("lookalike_extension", "https://b.example.com/t/x.brand.csv", False),
+ ("br_inside_the_name", "https://b.example.com/t/x.br.csv", False),
+ ("br_host", "https://files.example.com.br/t/x.csv", False),
+ ]
+ )
+ def test_build_function_call_rejects_brotli_files(self, _name: str, url: str, rejected: bool) -> None:
+ if rejected:
+ with self.assertRaisesMessage(ExposedHogQLError, "Brotli"):
+ build_function_call(url, DataWarehouseTable.TableFormat.CSVWithNames, None, "key", "secret")
+ else:
+ build_function_call(url, DataWarehouseTable.TableFormat.CSVWithNames, None, "key", "secret")
+
+ @parameterized.expand([(DataWarehouseTable.TableFormat.Delta,), (DataWarehouseTable.TableFormat.DeltaS3Wrapper,)])
+ def test_build_function_call_reads_a_delta_table_in_a_br_folder(self, table_format: str) -> None:
+ build_function_call("https://b.example.com/lake/table.br", table_format, None, "key", "secret")
+
+
class TestS3Table(BaseTest):
def _events_from_sql(self) -> str:
if settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA:
diff --git a/products/warehouse_sources/backend/models/table.py b/products/warehouse_sources/backend/models/table.py
index c491bf417e9f..674a0218a469 100644
--- a/products/warehouse_sources/backend/models/table.py
+++ b/products/warehouse_sources/backend/models/table.py
@@ -31,6 +31,7 @@ from posthog.hogql.database.models import DatabaseField, FieldOrTable, StructDat
from posthog.hogql.database.s3_table import (
DataWarehouseTable as HogQLDataWarehouseTable,
build_function_call,
+ compression_for_url_pattern,
)
from posthog.hogql.escape_sql import (
backquote_clickhouse_identifier,
@@ -747,11 +748,30 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Delet
"""
return read_format != DataWarehouseTableFormat.Delta or self.credential_id is None
+ def _chdb_compression(self, read_format: str) -> str | None:
+ compression = compression_for_url_pattern(self.url_pattern)
+ # A wrong codec fails a Parquet read, which falls back to the cluster, but a CSV or JSON read
+ # can infer a wrong schema instead.
+ if compression is None and read_format == DataWarehouseTableFormat.Parquet:
+ return "none"
+ return compression
+
def _describe_with_chdb(
- self, s3_table_func: str, placeholder_context: HogQLContext, describe_settings: dict[str, str | int]
+ self, read_format: str, compression: str, describe_settings: dict[str, str | int]
) -> list[tuple[str, ...]] | None:
"""The DESCRIBE rows read by the embedded engine, or None when it cannot answer and
ClickHouse has to."""
+ placeholder_context = HogQLContext(team_id=self.team.pk)
+ s3_table_func = build_function_call(
+ url=self.url_pattern,
+ queryable_folder=self.queryable_folder,
+ format=read_format,
+ access_key=self.credential.access_key if self.credential else None,
+ access_secret=self.credential.access_secret if self.credential else None,
+ context=placeholder_context,
+ table_size_mib=0,
+ compression=compression,
+ )
try:
quoted_placeholders = {k: escape_param_clickhouse(v) for k, v in placeholder_context.values.items()}
# chdb doesn't support parameterized queries
@@ -782,9 +802,10 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Delet
context=placeholder_context,
table_size_mib=0, # Use the non-cluster s3 table function for chdb
)
+ chdb_compression = self._chdb_compression(describe_format)
# chdb hangs in CI during tests
- if not TEST and self._chdb_may_read(describe_format):
- result = self._describe_with_chdb(s3_table_func, placeholder_context, describe_settings)
+ if not TEST and chdb_compression is not None and self._chdb_may_read(describe_format):
+ result = self._describe_with_chdb(describe_format, chdb_compression, describe_settings)
if result is None:
tag_queries(
@@ -892,7 +913,7 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Delet
"""
return self.size_in_s3_mib is None or self.size_in_s3_mib < S3_CLUSTER_TABLE_SIZE_MIB
- def _count_with_chdb(self) -> int | None:
+ def _count_with_chdb(self, compression: str) -> int | None:
"""The row count read by the embedded engine, or None when it cannot answer and ClickHouse
has to."""
placeholder_context = HogQLContext(team_id=self.team.pk)
@@ -904,6 +925,7 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Delet
access_secret=self.credential.access_secret if self.credential else None,
context=placeholder_context,
table_size_mib=0, # Use the non-cluster s3 table function for chdb
+ compression=compression,
)
try:
quoted_placeholders = {k: escape_param_clickhouse(v) for k, v in placeholder_context.values.items()}
@@ -922,9 +944,15 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Delet
return int(rows[0][0]) if rows else None
def get_count(self, safe_expose_ch_error=True) -> int:
+ chdb_compression = self._chdb_compression(self.format)
# chdb hangs in CI during tests
- if not TEST and self._count_fits_the_chdb_budget() and self._chdb_may_read(self.format):
- chdb_count = self._count_with_chdb()
+ if (
+ not TEST
+ and chdb_compression is not None
+ and self._count_fits_the_chdb_budget()
+ and self._chdb_may_read(self.format)
+ ):
+ chdb_count = self._count_with_chdb(chdb_compression)
if chdb_count is not None:
return chdb_count
diff --git a/products/warehouse_sources/backend/tests/test_table.py b/products/warehouse_sources/backend/tests/test_table.py
index c5aea52e2b97..df8b293fb814 100644
--- a/products/warehouse_sources/backend/tests/test_table.py
+++ b/products/warehouse_sources/backend/tests/test_table.py
@@ -424,6 +424,44 @@ class TestChdbEngineChoiceByFormat:
assert mock_run_chdb.called is uses_chdb
assert mock_sync_execute.called is not uses_chdb
+ @pytest.mark.parametrize(
+ "table_format,file_pattern,chdb_codec",
+ [
+ (DataWarehouseTable.TableFormat.Parquet, "*.parquet", "none"),
+ (DataWarehouseTable.TableFormat.CSVWithNames, "*.csv.gz", "gzip"),
+ (DataWarehouseTable.TableFormat.Parquet, "*", "none"),
+ (DataWarehouseTable.TableFormat.CSVWithNames, "*", None),
+ (DataWarehouseTable.TableFormat.JSON, "**", None),
+ ],
+ )
+ def test_chdb_reads_with_the_codec_the_url_pattern_names(
+ self, table_format: str, file_pattern: str, chdb_codec: str | None
+ ) -> None:
+ table = self._table(table_format, has_credential=True)
+ table.url_pattern = f"https://bucket.example.com/t/{file_pattern}"
+ with (
+ patch("products.warehouse_sources.backend.models.table.TEST", False),
+ patch(
+ "products.warehouse_sources.backend.models.table.run_chdb_query",
+ side_effect=lambda query: "7\n" if "count()" in query else "id,String\n",
+ ) as mock_run_chdb,
+ patch(
+ "products.warehouse_sources.backend.models.table.sync_execute",
+ side_effect=lambda query, **_: [(7,)] if "count()" in query else [("id", "String")],
+ ) as mock_sync_execute,
+ ):
+ assert table.get_columns()["id"]["clickhouse"] == "String"
+ assert table.get_count() == 7
+
+ if chdb_codec is None:
+ assert not mock_run_chdb.called
+ assert mock_sync_execute.call_count == 2
+ else:
+ chdb_queries = [call.args[0] for call in mock_run_chdb.call_args_list]
+ assert len(chdb_queries) == 2
+ assert all(query.endswith(f"'auto', '{chdb_codec}')") for query in chdb_queries), chdb_queries
+ assert not mock_sync_execute.called
+
def _test_certificate(
name: str,