:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse-sources): read warehouse files in chdb with an explicit codec (#123070)
Felipe Rosa de Almeida71bb63814ea7
diff --git a/posthog/hogql/database/s3_table.py b/posthog/hogql/database/s3_table.pyindex e42539cadd2a..66a042084135 100644--- a/posthog/hogql/database/s3_table.py+++ b/posthog/hogql/database/s3_table.py@@ -1,4 +1,6 @@import re+import itertools+from collections.abc import Iteratorfrom dataclasses import dataclassfrom pathlib import PurePosixPathfrom typing import Literal, Optional@@ -24,6 +26,25 @@ _FORMAT_LABELS = {"Delta": "Delta","DeltaS3Wrapper": "Delta",}+# Mirrors ClickHouse's chooseCompressionMethod.+_COMPRESSION_BY_EXTENSION = {+ "gz": "gzip",+ "gzip": "gzip",+ "deflate": "deflate",+ "br": "br",+ "brotli": "br",+ "xz": "xz",+ "lzma": "xz",+ "zst": "zstd",+ "zstd": "zstd",+ "lz4": "lz4",+ "bz2": "bz2",+ "snappy": "snappy",+}+_URL_GLOB_CHARACTERS = frozenset("*?{}")+_BROTLI_FILE_NAME_RE = re.compile(r"\.(?:br|brotli)(?![\w.-])", re.IGNORECASE)+_BROTLI_EXTENSION_RE = re.compile(r"(?:br|brotli)[*?]*", re.IGNORECASE)+_MAX_BRACE_EXPANSIONS = 1000@dataclass(frozen=True, kw_only=True)@@ -106,6 +127,33 @@ def parse_duckdb_s3_source(url: str) -> DuckDBS3Source | None:)+def compression_for_url_pattern(url: str) -> str | None:+ """None when the pattern can match files with different extensions."""+ _, dot, extension = url.rsplit("/", 1)[-1].rpartition(".")+ if _URL_GLOB_CHARACTERS.intersection(extension):+ return None+ return _COMPRESSION_BY_EXTENSION.get(extension.lower(), "none") if dot else "none"+++def _brace_expansions(pattern: str) -> Iterator[str]:+ match = re.search(r"\{([^{}]*)\}", pattern)+ if match is None:+ yield pattern+ return+ for alternative in match.group(1).split(","):+ yield from _brace_expansions(pattern[: match.start()] + alternative + pattern[match.end() :])+++def _reads_brotli_files(url: str) -> bool:+ urls = list(itertools.islice(_brace_expansions(url), _MAX_BRACE_EXPANSIONS + 1))+ if len(urls) > _MAX_BRACE_EXPANSIONS:+ return _BROTLI_FILE_NAME_RE.search(url) is not None+ return any(+ dot and _BROTLI_EXTENSION_RE.fullmatch(extension)+ for _, dot, extension in (expanded.rsplit("/", 1)[-1].rpartition(".") for expanded in urls)+ )++def build_function_call(url: str,format: str,@@ -115,6 +163,7 @@ def build_function_call(structure: Optional[str] = None,context: Optional[HogQLContext] = None,table_size_mib: Optional[float] = None,+ compression: Optional[str] = None,) -> str:if access_key is None and access_secret is None and (settings.DEBUG or settings.TEST or settings.USE_LOCAL_SETUP):access_key = settings.DATAWAREHOUSE_LOCAL_ACCESS_KEY@@ -202,6 +251,11 @@ def build_function_call(return return_expr(expr)+ if _reads_brotli_files(url):+ raise ExposedHogQLError(+ "PostHog can't read Brotli-compressed (.br) files. Compress the files with gzip or zstd instead."+ )+# Azureif re.match(r"^https:\/\/.+\.blob\.core\.windows\.net\/", url):regex_result = re.search(r"(https:\/\/.+\.blob\.core\.windows\.net)\/(.+?)\/(.*)", url)@@ -222,8 +276,9 @@ def build_function_call(escaped_access_key = add_param(access_key)escaped_access_secret = add_param(access_secret)escaped_format = add_param(format, False)+ escaped_compression = add_param(compression, False) if compression else "'auto'"- expr = f"azureBlobStorage({storage_account_url}, {container}, {blob_path}, {escaped_access_key}, {escaped_access_secret}, {escaped_format}, 'auto'"+ expr = f"azureBlobStorage({storage_account_url}, {container}, {blob_path}, {escaped_access_key}, {escaped_access_secret}, {escaped_format}, {escaped_compression}"if structure:escaped_structure = add_param(structure, False)@@ -252,6 +307,12 @@ def build_function_call(if structure:expr += f", {escaped_structure}"+ elif compression:+ # s3() only reads the codec after a structure argument.+ expr += ", 'auto'"++ if compression:+ expr += f", {add_param(compression, False)}"return return_expr(expr)diff --git a/posthog/hogql/database/test/test_s3_table.py b/posthog/hogql/database/test/test_s3_table.pyindex 3b950860d435..8f7956211fb6 100644--- a/posthog/hogql/database/test/test_s3_table.py+++ b/posthog/hogql/database/test/test_s3_table.py@@ -4,13 +4,15 @@ from posthog.test.base import BaseTestfrom unittest import mockfrom django.conf import settings-from django.test import override_settings+from django.test import SimpleTestCase, override_settings++from parameterized import parameterizedfrom posthog.hogql.constants import MAX_SELECT_RETURNED_ROWSfrom posthog.hogql.context import HogQLContextfrom posthog.hogql.database.database import Databasefrom posthog.hogql.database.models import TableNode-from posthog.hogql.database.s3_table import build_function_call+from posthog.hogql.database.s3_table import build_function_call, compression_for_url_patternfrom posthog.hogql.database.test.tables import create_aapl_stock_s3_tablefrom posthog.hogql.errors import ExposedHogQLErrorfrom posthog.hogql.parser import parse_select@@ -22,6 +24,82 @@ from posthog.models.event.sql import DISTRIBUTED_EVENTS_JSON_TABLEfrom products.warehouse_sources.backend.facade.models import DataWarehouseTable+class TestS3Compression(SimpleTestCase):+ @parameterized.expand(+ [+ ("plain_file", "https://b.example.com/t/data.csv", "none"),+ ("no_extension", "https://b.example.com/t/data", "none"),+ ("folder", "https://b.example.com/t/", "none"),+ ("pinned_glob", "https://b.example.com/t/*.parquet", "none"),+ ("pinned_compressed_glob", "https://b.example.com/t/*.CSV.GZ", "gzip"),+ ("braces_before_the_extension", "https://b.example.com/t/{a,b}.json.zst", "zstd"),+ ("dotted_folder", "https://b.example.com/t.gz/data", "none"),+ ("open_glob", "https://b.example.com/t/*", None),+ ("recursive_glob", "https://b.example.com/t/**", None),+ ("glob_after_the_stem", "https://b.example.com/t/data_*", None),+ ("braces_in_the_extension", "https://b.example.com/t/*.{csv,csv.gz}", None),+ ("question_mark_in_the_extension", "https://b.example.com/t/data.cs?", None),+ ]+ )+ def test_compression_for_url_pattern(self, _name: str, url: str, expected: str | None) -> None:+ assert compression_for_url_pattern(url) == expected++ @parameterized.expand(+ [+ (+ "s3_without_structure",+ "https://b.example.com/t/*.csv.gz",+ None,+ "s3('https://b.example.com/t/*.csv.gz', 'key', 'secret', 'CSVWithNames', 'auto', 'gzip')",+ ),+ (+ "s3_with_structure",+ "https://b.example.com/t/*.csv.gz",+ "id String",+ "s3('https://b.example.com/t/*.csv.gz', 'key', 'secret', 'CSVWithNames', 'id String', 'gzip')",+ ),+ (+ "azure",+ "https://acct.blob.core.windows.net/c/t/*.csv.gz",+ "id String",+ "azureBlobStorage('https://acct.blob.core.windows.net', 'c', 't/*.csv.gz', 'key', 'secret', 'CSVWithNames', 'gzip', 'id String')",+ ),+ ]+ )+ def test_build_function_call_passes_an_explicit_codec(+ self, _name: str, url: str, structure: str | None, expected: str+ ) -> None:+ res = build_function_call(+ url, DataWarehouseTable.TableFormat.CSVWithNames, None, "key", "secret", structure, compression="gzip"+ )+ assert res == expected++ @parameterized.expand(+ [+ ("pinned", "https://b.example.com/t/export.csv.br", True),+ ("long_extension", "https://b.example.com/t/export.CSV.BROTLI", True),+ ("brace_alternative", "https://b.example.com/t/*.{csv,csv.br}", True),+ ("glob_after_the_extension", "https://b.example.com/t/*.br*", True),+ ("brace_alternative_across_folders", "https://b.example.com/{a/x.csv,b/y.br}", True),+ ("br_inside_a_brace_alternative", "https://b.example.com/t/{data.br,data.us}.csv", False),+ ("too_many_brace_expansions", "https://b.example.com/t/" + "{a,b}" * 40 + ".csv.br", True),+ ("lookalike_extension", "https://b.example.com/t/x.brand.csv", False),+ ("br_inside_the_name", "https://b.example.com/t/x.br.csv", False),+ ("br_host", "https://files.example.com.br/t/x.csv", False),+ ]+ )+ def test_build_function_call_rejects_brotli_files(self, _name: str, url: str, rejected: bool) -> None:+ if rejected:+ with self.assertRaisesMessage(ExposedHogQLError, "Brotli"):+ build_function_call(url, DataWarehouseTable.TableFormat.CSVWithNames, None, "key", "secret")+ else:+ build_function_call(url, DataWarehouseTable.TableFormat.CSVWithNames, None, "key", "secret")++ @parameterized.expand([(DataWarehouseTable.TableFormat.Delta,), (DataWarehouseTable.TableFormat.DeltaS3Wrapper,)])+ def test_build_function_call_reads_a_delta_table_in_a_br_folder(self, table_format: str) -> None:+ build_function_call("https://b.example.com/lake/table.br", table_format, None, "key", "secret")++class TestS3Table(BaseTest):def _events_from_sql(self) -> str:if settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA:diff --git a/products/warehouse_sources/backend/models/table.py b/products/warehouse_sources/backend/models/table.pyindex c491bf417e9f..674a0218a469 100644--- a/products/warehouse_sources/backend/models/table.py+++ b/products/warehouse_sources/backend/models/table.py@@ -31,6 +31,7 @@ from posthog.hogql.database.models import DatabaseField, FieldOrTable, StructDatfrom posthog.hogql.database.s3_table import (DataWarehouseTable as HogQLDataWarehouseTable,build_function_call,+ compression_for_url_pattern,)from posthog.hogql.escape_sql import (backquote_clickhouse_identifier,@@ -747,11 +748,30 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Delet"""return read_format != DataWarehouseTableFormat.Delta or self.credential_id is None+ def _chdb_compression(self, read_format: str) -> str | None:+ compression = compression_for_url_pattern(self.url_pattern)+ # A wrong codec fails a Parquet read, which falls back to the cluster, but a CSV or JSON read+ # can infer a wrong schema instead.+ if compression is None and read_format == DataWarehouseTableFormat.Parquet:+ return "none"+ return compression+def _describe_with_chdb(- self, s3_table_func: str, placeholder_context: HogQLContext, describe_settings: dict[str, str | int]+ self, read_format: str, compression: str, describe_settings: dict[str, str | int]) -> list[tuple[str, ...]] | None:"""The DESCRIBE rows read by the embedded engine, or None when it cannot answer andClickHouse has to."""+ placeholder_context = HogQLContext(team_id=self.team.pk)+ s3_table_func = build_function_call(+ url=self.url_pattern,+ queryable_folder=self.queryable_folder,+ format=read_format,+ access_key=self.credential.access_key if self.credential else None,+ access_secret=self.credential.access_secret if self.credential else None,+ context=placeholder_context,+ table_size_mib=0,+ compression=compression,+ )try:quoted_placeholders = {k: escape_param_clickhouse(v) for k, v in placeholder_context.values.items()}# chdb doesn't support parameterized queries@@ -782,9 +802,10 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Deletcontext=placeholder_context,table_size_mib=0, # Use the non-cluster s3 table function for chdb)+ chdb_compression = self._chdb_compression(describe_format)# chdb hangs in CI during tests- if not TEST and self._chdb_may_read(describe_format):- result = self._describe_with_chdb(s3_table_func, placeholder_context, describe_settings)+ if not TEST and chdb_compression is not None and self._chdb_may_read(describe_format):+ result = self._describe_with_chdb(describe_format, chdb_compression, describe_settings)if result is None:tag_queries(@@ -892,7 +913,7 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Delet"""return self.size_in_s3_mib is None or self.size_in_s3_mib < S3_CLUSTER_TABLE_SIZE_MIB- def _count_with_chdb(self) -> int | None:+ def _count_with_chdb(self, compression: str) -> int | None:"""The row count read by the embedded engine, or None when it cannot answer and ClickHousehas to."""placeholder_context = HogQLContext(team_id=self.team.pk)@@ -904,6 +925,7 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Deletaccess_secret=self.credential.access_secret if self.credential else None,context=placeholder_context,table_size_mib=0, # Use the non-cluster s3 table function for chdb+ compression=compression,)try:quoted_placeholders = {k: escape_param_clickhouse(v) for k, v in placeholder_context.values.items()}@@ -922,9 +944,15 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Deletreturn int(rows[0][0]) if rows else Nonedef get_count(self, safe_expose_ch_error=True) -> int:+ chdb_compression = self._chdb_compression(self.format)# chdb hangs in CI during tests- if not TEST and self._count_fits_the_chdb_budget() and self._chdb_may_read(self.format):- chdb_count = self._count_with_chdb()+ if (+ not TEST+ and chdb_compression is not None+ and self._count_fits_the_chdb_budget()+ and self._chdb_may_read(self.format)+ ):+ chdb_count = self._count_with_chdb(chdb_compression)if chdb_count is not None:return chdb_countdiff --git a/products/warehouse_sources/backend/tests/test_table.py b/products/warehouse_sources/backend/tests/test_table.pyindex c5aea52e2b97..df8b293fb814 100644--- a/products/warehouse_sources/backend/tests/test_table.py+++ b/products/warehouse_sources/backend/tests/test_table.py@@ -424,6 +424,44 @@ class TestChdbEngineChoiceByFormat:assert mock_run_chdb.called is uses_chdbassert mock_sync_execute.called is not uses_chdb+ @pytest.mark.parametrize(+ "table_format,file_pattern,chdb_codec",+ [+ (DataWarehouseTable.TableFormat.Parquet, "*.parquet", "none"),+ (DataWarehouseTable.TableFormat.CSVWithNames, "*.csv.gz", "gzip"),+ (DataWarehouseTable.TableFormat.Parquet, "*", "none"),+ (DataWarehouseTable.TableFormat.CSVWithNames, "*", None),+ (DataWarehouseTable.TableFormat.JSON, "**", None),+ ],+ )+ def test_chdb_reads_with_the_codec_the_url_pattern_names(+ self, table_format: str, file_pattern: str, chdb_codec: str | None+ ) -> None:+ table = self._table(table_format, has_credential=True)+ table.url_pattern = f"https://bucket.example.com/t/{file_pattern}"+ with (+ patch("products.warehouse_sources.backend.models.table.TEST", False),+ patch(+ "products.warehouse_sources.backend.models.table.run_chdb_query",+ side_effect=lambda query: "7\n" if "count()" in query else "id,String\n",+ ) as mock_run_chdb,+ patch(+ "products.warehouse_sources.backend.models.table.sync_execute",+ side_effect=lambda query, **_: [(7,)] if "count()" in query else [("id", "String")],+ ) as mock_sync_execute,+ ):+ assert table.get_columns()["id"]["clickhouse"] == "String"+ assert table.get_count() == 7++ if chdb_codec is None:+ assert not mock_run_chdb.called+ assert mock_sync_execute.call_count == 2+ else:+ chdb_queries = [call.args[0] for call in mock_run_chdb.call_args_list]+ assert len(chdb_queries) == 2+ assert all(query.endswith(f"'auto', '{chdb_codec}')") for query in chdb_queries), chdb_queries+ assert not mock_sync_execute.called+def _test_certificate(name: str,