Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

refactor(hogql): split s3 table function renderers (#110489)

posthog[bot]6c275dbd49e2
diff --git a/posthog/hogql/database/s3_table.py b/posthog/hogql/database/s3_table.py
index d6dfa87f5153..11bb04781e9e 100644
--- a/posthog/hogql/database/s3_table.py
+++ b/posthog/hogql/database/s3_table.py
@@ -128,6 +128,120 @@ def parse_duckdb_s3_source(url: str) -> DuckDBS3Source | None:
)
+class _FunctionCallParams:
+ def __init__(self, context: Optional[HogQLContext]) -> None:
+ self._context = context
+ self._raw_params: dict[str, str] = {}
+
+ def add(self, value: str, is_sensitive: bool = True) -> str:
+ if self._context is not None:
+ if is_sensitive:
+ return self._context.add_sensitive_value(value)
+ return self._context.add_value(value)
+
+ param_name = f"value_{len(self._raw_params)}"
+ self._raw_params[param_name] = value
+ return f"%({param_name})s"
+
+ def finish(self, expr: str) -> str:
+ if self._context is not None:
+ return f"{expr})"
+
+ return f"{substitute_params(expr, self._raw_params)})"
+
+
+def _s3_function(table_size_mib: Optional[float]) -> str:
+ if table_size_mib is not None and table_size_mib >= 1024: # 1 GiB
+ return "s3Cluster('posthog', "
+ return "s3("
+
+
+def _queryable_folder_url(url: str, queryable_folder: str) -> str:
+ # Hack: Remove the last directory from the URL and add the queryable folder instead
+ # TODO(Gilbert09): Fix this: simplify logic around how we construct the S3 and
+ # http urls and make all url generation going through a single place
+ parsed = urlparse(url)
+ new_path = str(PurePosixPath(parsed.path).parent) + "/"
+ new_url = urlunparse(parsed._replace(path=new_path))
+ return new_url + queryable_folder + "/**.parquet"
+
+
+def _url_table_function(
+ params: _FunctionCallParams,
+ function: str,
+ url: str,
+ format: Optional[str],
+ structure: Optional[str],
+ access_key: Optional[str],
+ access_secret: Optional[str],
+ compression: Optional[str] = None,
+) -> str:
+ """Renders `s3`, `s3Cluster` and `deltaLake` calls, which share one argument order.
+
+ A `format` of None renders the literal 'Parquet' instead of a parameter.
+ """
+ escaped_url = params.add(url)
+ format_arg = params.add(format, False) if format is not None else "'Parquet'"
+ escaped_structure = params.add(structure, False) if structure else None
+
+ expr = f"{function}{escaped_url}"
+
+ if access_key and access_secret:
+ escaped_access_key = params.add(access_key)
+ escaped_access_secret = params.add(access_secret)
+ expr += f", {escaped_access_key}, {escaped_access_secret}"
+
+ expr += f", {format_arg}"
+
+ if escaped_structure:
+ expr += f", {escaped_structure}"
+ elif compression:
+ # s3() only reads the codec after a structure argument.
+ expr += ", 'auto'"
+
+ if compression:
+ expr += f", {params.add(compression, False)}"
+
+ return params.finish(expr)
+
+
+def _azure_blob_storage_call(
+ params: _FunctionCallParams,
+ url: str,
+ format: str,
+ structure: Optional[str],
+ access_key: Optional[str],
+ access_secret: Optional[str],
+ compression: Optional[str] = None,
+) -> str:
+ azure_match = _AZURE_BLOB_ACCOUNT_URL_RE.match(url)
+ if azure_match is None:
+ raise ExposedHogQLError("Can't parse Azure blob storage URL")
+ container_name, separator, blob_name = url[azure_match.end() :].partition("/")
+ if not container_name or not separator:
+ raise ExposedHogQLError("Can't parse Azure blob storage URL")
+
+ storage_account_url = params.add(azure_match.group(1))
+ container = params.add(container_name)
+ blob_path = params.add(blob_name)
+
+ if not access_key or not access_secret:
+ raise ExposedHogQLError("Azure blob storage has no access key or secret")
+
+ escaped_access_key = params.add(access_key)
+ escaped_access_secret = params.add(access_secret)
+ escaped_format = params.add(format, False)
+ escaped_compression = params.add(compression, False) if compression else "'auto'"
+
+ expr = f"azureBlobStorage({storage_account_url}, {container}, {blob_path}, {escaped_access_key}, {escaped_access_secret}, {escaped_format}, {escaped_compression}"
+
+ if structure:
+ escaped_structure = params.add(structure, False)
+ expr += f", {escaped_structure}"
+
+ return params.finish(expr)
+
+
def compression_for_url_pattern(url: str) -> str | None:
"""None when the pattern can match files with different extensions."""
_, dot, extension = url.rsplit("/", 1)[-1].rpartition(".")
@@ -174,148 +288,33 @@ def build_function_call(
access_key = settings.DATAWAREHOUSE_LOCAL_ACCESS_KEY
access_secret = settings.DATAWAREHOUSE_LOCAL_ACCESS_SECRET
- use_s3_cluster = False
- if table_size_mib is not None and table_size_mib >= 1024: # 1 GiB
- use_s3_cluster = True
-
# If a table has a queryable url set, then use that directly
if queryable_folder and format == "DeltaS3Wrapper":
- # Hack: Remove the last directory from the URL and add the queryable folder instead
- # TODO(Gilbert09): Fix this: simplify logic around how we construct the S3 and
- # http urls and make all url generation going through a single place
- parsed = urlparse(url)
- new_path = str(PurePosixPath(parsed.path).parent) + "/"
- new_url = urlunparse(parsed._replace(path=new_path))
- url = new_url + queryable_folder + "/**.parquet"
+ url = _queryable_folder_url(url, queryable_folder)
format = "Parquet"
- raw_params: dict[str, str] = {}
-
- def add_param(value: str, is_sensitive: bool = True) -> str:
- if context is not None:
- if is_sensitive:
- return context.add_sensitive_value(value)
- return context.add_value(value)
-
- param_name = f"value_{len(raw_params.items())}"
- raw_params[param_name] = value
- return f"%({param_name})s"
-
- def return_expr(expr: str) -> str:
- if context is not None:
- return f"{expr})"
-
- return f"{substitute_params(expr, raw_params)})"
+ params = _FunctionCallParams(context)
- # DeltaS3Wrapper format
if format == "DeltaS3Wrapper":
- if url.endswith("/"):
- escaped_url = add_param(f"{url[:-1]}__query/**.parquet")
- else:
- escaped_url = add_param(f"{url}__query/**.parquet")
-
- if structure:
- escaped_structure = add_param(structure, False)
-
- if use_s3_cluster:
- expr = f"s3Cluster('posthog', {escaped_url}"
- else:
- expr = f"s3({escaped_url}"
-
- if access_key and access_secret:
- escaped_access_key = add_param(access_key)
- escaped_access_secret = add_param(access_secret)
-
- expr += f", {escaped_access_key}, {escaped_access_secret}"
-
- expr += ", 'Parquet'"
-
- if structure:
- expr += f", {escaped_structure}"
-
- return return_expr(expr)
+ query_url = f"{url.removesuffix('/')}__query/**.parquet"
+ return _url_table_function(
+ params, _s3_function(table_size_mib), query_url, None, structure, access_key, access_secret
+ )
- # Delta format
if format == "Delta":
- escaped_url = add_param(url)
- if structure:
- escaped_structure = add_param(structure, False)
-
- expr = f"deltaLake({escaped_url}"
-
- if access_key and access_secret:
- escaped_access_key = add_param(access_key)
- escaped_access_secret = add_param(access_secret)
-
- expr += f", {escaped_access_key}, {escaped_access_secret}"
-
- expr += ", 'Parquet'"
-
- if structure:
- expr += f", {escaped_structure}"
-
- return return_expr(expr)
+ return _url_table_function(params, "deltaLake(", url, None, structure, access_key, access_secret)
if _reads_brotli_files(url):
raise ExposedHogQLError(
"PostHog can't read Brotli-compressed (.br) files. Compress the files with gzip or zstd instead."
)
- # Azure
- if azure_match := _AZURE_BLOB_ACCOUNT_URL_RE.match(url):
- container_name, separator, blob_name = url[azure_match.end() :].partition("/")
- if not container_name or not separator:
- raise ExposedHogQLError("Can't parse Azure blob storage URL")
-
- storage_account_url = add_param(azure_match.group(1))
- container = add_param(container_name)
- blob_path = add_param(blob_name)
+ if is_azure_blob_url(url):
+ return _azure_blob_storage_call(params, url, format, structure, access_key, access_secret, compression)
- if not access_key or not access_secret:
- raise ExposedHogQLError("Azure blob storage has no access key or secret")
-
- escaped_access_key = add_param(access_key)
- escaped_access_secret = add_param(access_secret)
- escaped_format = add_param(format, False)
- escaped_compression = add_param(compression, False) if compression else "'auto'"
-
- expr = f"azureBlobStorage({storage_account_url}, {container}, {blob_path}, {escaped_access_key}, {escaped_access_secret}, {escaped_format}, {escaped_compression}"
-
- if structure:
- escaped_structure = add_param(structure, False)
- expr += f", {escaped_structure}"
-
- return return_expr(expr)
-
- # S3
- escaped_url = add_param(url)
- escaped_format = add_param(format, False)
- if structure:
- escaped_structure = add_param(structure, False)
-
- if use_s3_cluster:
- expr = f"s3Cluster('posthog', {escaped_url}"
- else:
- expr = f"s3({escaped_url}"
-
- if access_key and access_secret:
- escaped_access_key = add_param(access_key)
- escaped_access_secret = add_param(access_secret)
-
- expr += f", {escaped_access_key}, {escaped_access_secret}"
-
- expr += f", {escaped_format}"
-
- if structure:
- expr += f", {escaped_structure}"
- elif compression:
- # s3() only reads the codec after a structure argument.
- expr += ", 'auto'"
-
- if compression:
- expr += f", {add_param(compression, False)}"
-
- return return_expr(expr)
+ return _url_table_function(
+ params, _s3_function(table_size_mib), url, format, structure, access_key, access_secret, compression
+ )
class S3Table(FunctionCallTable):
diff --git a/posthog/hogql/database/test/test_s3_table.py b/posthog/hogql/database/test/test_s3_table.py
index 304815c36819..8015a87da036 100644
--- a/posthog/hogql/database/test/test_s3_table.py
+++ b/posthog/hogql/database/test/test_s3_table.py
@@ -99,6 +99,10 @@ class TestS3Compression(SimpleTestCase):
def test_build_function_call_reads_a_delta_table_in_a_br_folder(self, table_format: str) -> None:
build_function_call("https://b.example.com/lake/table.br", table_format, None, "key", "secret")
+ def test_empty_format_is_preserved(self) -> None:
+ result = build_function_call("http://url.com", "", access_key="key", access_secret="secret")
+ assert result == "s3('http://url.com', 'key', 'secret', '')"
+
class TestS3Table(BaseTest):
def _events_from_sql(self) -> str: