Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): tighten azure blob url detection and routing (#122823)

Felipe Rosa de Almeidaf152e3b06b3a
diff --git a/posthog/hogql/database/s3_table.py b/posthog/hogql/database/s3_table.py
index 66a042084135..68b4aef2e0a9 100644
--- a/posthog/hogql/database/s3_table.py
+++ b/posthog/hogql/database/s3_table.py
@@ -19,6 +19,7 @@ _AWS_S3_ENDPOINT_RE = re.compile(r"s3(?:[.-][a-z0-9-]+)*\.amazonaws\.com(?:\.cn)
_AWS_S3_VIRTUAL_HOST_RE = re.compile(r"(?P<bucket>.+)\.(?P<endpoint>s3(?:[.-][a-z0-9-]+)*\.amazonaws\.com(?:\.cn)?)")
_AWS_REGION_RE = re.compile(r"(?:^|[.-])(?P<region>[a-z]{2,4}(?:-[a-z0-9]+)+-\d)(?:\.|$)")
_AZURE_BLOB_HOST_SUFFIX = ".blob.core.windows.net"
+_AZURE_BLOB_ACCOUNT_URL_RE = re.compile(r"(https://[A-Za-z0-9-]+\.blob\.core\.windows\.net)/")
_FORMAT_LABELS = {
"CSV": "CSV",
"CSVWithNames": "CSV with headers",
@@ -154,6 +155,10 @@ def _reads_brotli_files(url: str) -> bool:
)
+def is_azure_blob_url(url: str) -> bool:
+ return _AZURE_BLOB_ACCOUNT_URL_RE.match(url) is not None
+
+
def build_function_call(
url: str,
format: str,
@@ -257,18 +262,14 @@ def build_function_call(
)
# Azure
- if re.match(r"^https:\/\/.+\.blob\.core\.windows\.net\/", url):
- regex_result = re.search(r"(https:\/\/.+\.blob\.core\.windows\.net)\/(.+?)\/(.*)", url)
- if regex_result is None:
- raise ExposedHogQLError("Can't parse Azure blob storage URL")
-
- groups = regex_result.groups()
- if len(groups) < 3:
+ if azure_match := _AZURE_BLOB_ACCOUNT_URL_RE.match(url):
+ container_name, separator, blob_name = url[azure_match.end() :].partition("/")
+ if not container_name or not separator:
raise ExposedHogQLError("Can't parse Azure blob storage URL")
- storage_account_url = add_param(groups[0])
- container = add_param(groups[1])
- blob_path = add_param(groups[2])
+ storage_account_url = add_param(azure_match.group(1))
+ container = add_param(container_name)
+ blob_path = add_param(blob_name)
if not access_key or not access_secret:
raise ExposedHogQLError("Azure blob storage has no access key or secret")
diff --git a/posthog/hogql/database/test/test_s3_table.py b/posthog/hogql/database/test/test_s3_table.py
index 8f7956211fb6..304815c36819 100644
--- a/posthog/hogql/database/test/test_s3_table.py
+++ b/posthog/hogql/database/test/test_s3_table.py
@@ -459,6 +459,19 @@ class TestS3Table(BaseTest):
== "azureBlobStorage('https://tomposthogtest.blob.core.windows.net', 'somecontainer', 'path/to/file.parquet', 'tomposthogtest', 'blah', 'Parquet', 'auto')"
)
+ @parameterized.expand(
+ [
+ ("host_in_path", "https://attacker.example/x.blob.core.windows.net/c/f.csv"),
+ ("host_in_userinfo", "https://[email protected]/c/f.csv"),
+ ("suffix_subdomain", "https://x.blob.core.windows.net.attacker.example/c/f.csv"),
+ ("backslash_authority", "https://attacker.example\\@x.blob.core.windows.net/c/f.csv"),
+ ]
+ )
+ def test_s3_build_function_call_azure_requires_azure_host(self, _name: str, url: str):
+ res = build_function_call(url, DataWarehouseTable.TableFormat.CSV, None, "key", "secret", None, None)
+
+ assert res.startswith("s3(")
+
def test_s3_build_function_call_azure_with_context(self):
self._init_database()
diff --git a/products/warehouse_sources/backend/models/table.py b/products/warehouse_sources/backend/models/table.py
index 5bc8bb2efdea..e065a60e013b 100644
--- a/products/warehouse_sources/backend/models/table.py
+++ b/products/warehouse_sources/backend/models/table.py
@@ -32,6 +32,7 @@ from posthog.hogql.database.s3_table import (
DataWarehouseTable as HogQLDataWarehouseTable,
build_function_call,
compression_for_url_pattern,
+ is_azure_blob_url,
)
from posthog.hogql.escape_sql import (
backquote_clickhouse_identifier,
@@ -291,7 +292,7 @@ _CHDB_SUBPROCESS_PROXY_ENV_VARS = ("HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY")
# redirects by default, so a URL that passes validation can send chdb to an internal address. The
# ClickHouse cluster checks every hop against remote_url_allow_hosts. chdb has no such list, so it
# follows no redirect, and a URL that needs one fails over to the cluster.
-_CHDB_GUARD_SETTINGS: dict[str, str | int] = {"s3_max_redirects": 0}
+_CHDB_GUARD_SETTINGS: dict[str, str | int] = {"s3_max_redirects": 0, "s3_retry_attempts": 0}
def run_chdb_query(query: str, timeout: float = CHDB_QUERY_TIMEOUT_SECONDS) -> str:
@@ -769,6 +770,9 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Delet
URL the team supplied, so its Delta reads go to the cluster, which checks every hop against
remote_url_allow_hosts. A table without a credential reads a URL that PostHog built.
"""
+ # chdb's Azure client ignores the egress proxy and connects directly.
+ if is_azure_blob_url(self.url_pattern):
+ return False
return read_format != DataWarehouseTableFormat.Delta or self.credential_id is None
def _chdb_compression(self, read_format: str) -> str | None:
diff --git a/products/warehouse_sources/backend/tests/test_table.py b/products/warehouse_sources/backend/tests/test_table.py
index 3ea501642881..9dafae3b2e4c 100644
--- a/products/warehouse_sources/backend/tests/test_table.py
+++ b/products/warehouse_sources/backend/tests/test_table.py
@@ -364,15 +364,34 @@ class TestGetCountEngineChoice:
class TestChdbEngineChoiceByFormat:
- def _table(self, table_format: str, has_credential: bool) -> DataWarehouseTable:
+ def _table(
+ self, table_format: str, has_credential: bool, url_pattern: str = "https://bucket.example.com/t/"
+ ) -> DataWarehouseTable:
return DataWarehouseTable(
name="t",
format=table_format,
team=Team(id=1),
- url_pattern="https://bucket.example.com/t/",
+ url_pattern=url_pattern,
credential=DataWarehouseCredential(access_key="key", access_secret="secret") if has_credential else None,
)
+ @pytest.mark.parametrize("method,clickhouse_result", [("get_columns", [("id", "String")]), ("get_count", [(7,)])])
+ def test_azure_reads_run_on_the_cluster(self, method: str, clickhouse_result: list[tuple]) -> None:
+ table = self._table(
+ DataWarehouseTable.TableFormat.Parquet, True, "https://acct.blob.core.windows.net/container/t/*.parquet"
+ )
+ with (
+ patch("products.warehouse_sources.backend.models.table.TEST", False),
+ patch("products.warehouse_sources.backend.models.table.run_chdb_query") as mock_run_chdb,
+ patch(
+ "products.warehouse_sources.backend.models.table.sync_execute", return_value=clickhouse_result
+ ) as mock_sync_execute,
+ ):
+ getattr(table, method)()
+
+ assert not mock_run_chdb.called
+ assert "azureBlobStorage(" in mock_sync_execute.call_args.args[0]
+
@pytest.mark.parametrize(
"table_format,has_credential,uses_chdb",
[
@@ -520,7 +539,7 @@ class TestRunChdbQuery:
assert child_env["AWS_REGION"] == "us-east-1"
assert child_env["AWS_EC2_METADATA_DISABLED"] == "true"
assert child_env["https_proxy"] == "http://web-1:[email protected]:4750/"
- assert mock_run.call_args.kwargs["input"] == "SET s3_max_redirects = 0; SELECT 1"
+ assert mock_run.call_args.kwargs["input"] == "SET s3_max_redirects = 0; SET s3_retry_attempts = 0; SELECT 1"
def test_read_rejects_a_certificate_issued_for_another_host(
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path