:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
refactor(hogql): split s3 table function renderers (#110489)
posthog[bot]6c275dbd49e2
diff --git a/posthog/hogql/database/s3_table.py b/posthog/hogql/database/s3_table.pyindex d6dfa87f5153..11bb04781e9e 100644--- a/posthog/hogql/database/s3_table.py+++ b/posthog/hogql/database/s3_table.py@@ -128,6 +128,120 @@ def parse_duckdb_s3_source(url: str) -> DuckDBS3Source | None:)+class _FunctionCallParams:+ def __init__(self, context: Optional[HogQLContext]) -> None:+ self._context = context+ self._raw_params: dict[str, str] = {}++ def add(self, value: str, is_sensitive: bool = True) -> str:+ if self._context is not None:+ if is_sensitive:+ return self._context.add_sensitive_value(value)+ return self._context.add_value(value)++ param_name = f"value_{len(self._raw_params)}"+ self._raw_params[param_name] = value+ return f"%({param_name})s"++ def finish(self, expr: str) -> str:+ if self._context is not None:+ return f"{expr})"++ return f"{substitute_params(expr, self._raw_params)})"+++def _s3_function(table_size_mib: Optional[float]) -> str:+ if table_size_mib is not None and table_size_mib >= 1024: # 1 GiB+ return "s3Cluster('posthog', "+ return "s3("+++def _queryable_folder_url(url: str, queryable_folder: str) -> str:+ # Hack: Remove the last directory from the URL and add the queryable folder instead+ # TODO(Gilbert09): Fix this: simplify logic around how we construct the S3 and+ # http urls and make all url generation going through a single place+ parsed = urlparse(url)+ new_path = str(PurePosixPath(parsed.path).parent) + "/"+ new_url = urlunparse(parsed._replace(path=new_path))+ return new_url + queryable_folder + "/**.parquet"+++def _url_table_function(+ params: _FunctionCallParams,+ function: str,+ url: str,+ format: Optional[str],+ structure: Optional[str],+ access_key: Optional[str],+ access_secret: Optional[str],+ compression: Optional[str] = None,+) -> str:+ """Renders `s3`, `s3Cluster` and `deltaLake` calls, which share one argument order.++ A `format` of None renders the literal 'Parquet' instead of a parameter.+ """+ escaped_url = params.add(url)+ format_arg = params.add(format, False) if format is not None else "'Parquet'"+ escaped_structure = params.add(structure, False) if structure else None++ expr = f"{function}{escaped_url}"++ if access_key and access_secret:+ escaped_access_key = params.add(access_key)+ escaped_access_secret = params.add(access_secret)+ expr += f", {escaped_access_key}, {escaped_access_secret}"++ expr += f", {format_arg}"++ if escaped_structure:+ expr += f", {escaped_structure}"+ elif compression:+ # s3() only reads the codec after a structure argument.+ expr += ", 'auto'"++ if compression:+ expr += f", {params.add(compression, False)}"++ return params.finish(expr)+++def _azure_blob_storage_call(+ params: _FunctionCallParams,+ url: str,+ format: str,+ structure: Optional[str],+ access_key: Optional[str],+ access_secret: Optional[str],+ compression: Optional[str] = None,+) -> str:+ azure_match = _AZURE_BLOB_ACCOUNT_URL_RE.match(url)+ if azure_match is None:+ raise ExposedHogQLError("Can't parse Azure blob storage URL")+ container_name, separator, blob_name = url[azure_match.end() :].partition("/")+ if not container_name or not separator:+ raise ExposedHogQLError("Can't parse Azure blob storage URL")++ storage_account_url = params.add(azure_match.group(1))+ container = params.add(container_name)+ blob_path = params.add(blob_name)++ if not access_key or not access_secret:+ raise ExposedHogQLError("Azure blob storage has no access key or secret")++ escaped_access_key = params.add(access_key)+ escaped_access_secret = params.add(access_secret)+ escaped_format = params.add(format, False)+ escaped_compression = params.add(compression, False) if compression else "'auto'"++ expr = f"azureBlobStorage({storage_account_url}, {container}, {blob_path}, {escaped_access_key}, {escaped_access_secret}, {escaped_format}, {escaped_compression}"++ if structure:+ escaped_structure = params.add(structure, False)+ expr += f", {escaped_structure}"++ return params.finish(expr)++def compression_for_url_pattern(url: str) -> str | None:"""None when the pattern can match files with different extensions."""_, dot, extension = url.rsplit("/", 1)[-1].rpartition(".")@@ -174,148 +288,33 @@ def build_function_call(access_key = settings.DATAWAREHOUSE_LOCAL_ACCESS_KEYaccess_secret = settings.DATAWAREHOUSE_LOCAL_ACCESS_SECRET- use_s3_cluster = False- if table_size_mib is not None and table_size_mib >= 1024: # 1 GiB- use_s3_cluster = True-# If a table has a queryable url set, then use that directlyif queryable_folder and format == "DeltaS3Wrapper":- # Hack: Remove the last directory from the URL and add the queryable folder instead- # TODO(Gilbert09): Fix this: simplify logic around how we construct the S3 and- # http urls and make all url generation going through a single place- parsed = urlparse(url)- new_path = str(PurePosixPath(parsed.path).parent) + "/"- new_url = urlunparse(parsed._replace(path=new_path))- url = new_url + queryable_folder + "/**.parquet"+ url = _queryable_folder_url(url, queryable_folder)format = "Parquet"- raw_params: dict[str, str] = {}-- def add_param(value: str, is_sensitive: bool = True) -> str:- if context is not None:- if is_sensitive:- return context.add_sensitive_value(value)- return context.add_value(value)-- param_name = f"value_{len(raw_params.items())}"- raw_params[param_name] = value- return f"%({param_name})s"-- def return_expr(expr: str) -> str:- if context is not None:- return f"{expr})"-- return f"{substitute_params(expr, raw_params)})"+ params = _FunctionCallParams(context)- # DeltaS3Wrapper formatif format == "DeltaS3Wrapper":- if url.endswith("/"):- escaped_url = add_param(f"{url[:-1]}__query/**.parquet")- else:- escaped_url = add_param(f"{url}__query/**.parquet")-- if structure:- escaped_structure = add_param(structure, False)-- if use_s3_cluster:- expr = f"s3Cluster('posthog', {escaped_url}"- else:- expr = f"s3({escaped_url}"-- if access_key and access_secret:- escaped_access_key = add_param(access_key)- escaped_access_secret = add_param(access_secret)-- expr += f", {escaped_access_key}, {escaped_access_secret}"-- expr += ", 'Parquet'"-- if structure:- expr += f", {escaped_structure}"-- return return_expr(expr)+ query_url = f"{url.removesuffix('/')}__query/**.parquet"+ return _url_table_function(+ params, _s3_function(table_size_mib), query_url, None, structure, access_key, access_secret+ )- # Delta formatif format == "Delta":- escaped_url = add_param(url)- if structure:- escaped_structure = add_param(structure, False)-- expr = f"deltaLake({escaped_url}"-- if access_key and access_secret:- escaped_access_key = add_param(access_key)- escaped_access_secret = add_param(access_secret)-- expr += f", {escaped_access_key}, {escaped_access_secret}"-- expr += ", 'Parquet'"-- if structure:- expr += f", {escaped_structure}"-- return return_expr(expr)+ return _url_table_function(params, "deltaLake(", url, None, structure, access_key, access_secret)if _reads_brotli_files(url):raise ExposedHogQLError("PostHog can't read Brotli-compressed (.br) files. Compress the files with gzip or zstd instead.")- # Azure- if azure_match := _AZURE_BLOB_ACCOUNT_URL_RE.match(url):- container_name, separator, blob_name = url[azure_match.end() :].partition("/")- if not container_name or not separator:- raise ExposedHogQLError("Can't parse Azure blob storage URL")-- storage_account_url = add_param(azure_match.group(1))- container = add_param(container_name)- blob_path = add_param(blob_name)+ if is_azure_blob_url(url):+ return _azure_blob_storage_call(params, url, format, structure, access_key, access_secret, compression)- if not access_key or not access_secret:- raise ExposedHogQLError("Azure blob storage has no access key or secret")-- escaped_access_key = add_param(access_key)- escaped_access_secret = add_param(access_secret)- escaped_format = add_param(format, False)- escaped_compression = add_param(compression, False) if compression else "'auto'"-- expr = f"azureBlobStorage({storage_account_url}, {container}, {blob_path}, {escaped_access_key}, {escaped_access_secret}, {escaped_format}, {escaped_compression}"-- if structure:- escaped_structure = add_param(structure, False)- expr += f", {escaped_structure}"-- return return_expr(expr)-- # S3- escaped_url = add_param(url)- escaped_format = add_param(format, False)- if structure:- escaped_structure = add_param(structure, False)-- if use_s3_cluster:- expr = f"s3Cluster('posthog', {escaped_url}"- else:- expr = f"s3({escaped_url}"-- if access_key and access_secret:- escaped_access_key = add_param(access_key)- escaped_access_secret = add_param(access_secret)-- expr += f", {escaped_access_key}, {escaped_access_secret}"-- expr += f", {escaped_format}"-- if structure:- expr += f", {escaped_structure}"- elif compression:- # s3() only reads the codec after a structure argument.- expr += ", 'auto'"-- if compression:- expr += f", {add_param(compression, False)}"-- return return_expr(expr)+ return _url_table_function(+ params, _s3_function(table_size_mib), url, format, structure, access_key, access_secret, compression+ )class S3Table(FunctionCallTable):diff --git a/posthog/hogql/database/test/test_s3_table.py b/posthog/hogql/database/test/test_s3_table.pyindex 304815c36819..8015a87da036 100644--- a/posthog/hogql/database/test/test_s3_table.py+++ b/posthog/hogql/database/test/test_s3_table.py@@ -99,6 +99,10 @@ class TestS3Compression(SimpleTestCase):def test_build_function_call_reads_a_delta_table_in_a_br_folder(self, table_format: str) -> None:build_function_call("https://b.example.com/lake/table.br", table_format, None, "key", "secret")+ def test_empty_format_is_preserved(self) -> None:+ result = build_function_call("http://url.com", "", access_key="key", access_secret="secret")+ assert result == "s3('http://url.com', 'key', 'secret', '')"+class TestS3Table(BaseTest):def _events_from_sql(self) -> str: