Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): route chdb reads through the egress proxy (#121773)

Felipe Rosa de Almeidade1a405bbc2c
diff --git a/products/warehouse_sources/backend/models/table.py b/products/warehouse_sources/backend/models/table.py
index 4f8cd4d20cb8..298e75b413c2 100644
--- a/products/warehouse_sources/backend/models/table.py
+++ b/products/warehouse_sources/backend/models/table.py
@@ -272,6 +272,10 @@ _CHDB_SUBPROCESS_EXCLUDED_ENV_VARS = frozenset({"LD_PRELOAD", "MALLOC_CONF"})
# credentials when a read carries none.
_CHDB_SUBPROCESS_FORCED_ENV_VARS = {"AWS_EC2_METADATA_DISABLED": "true"}
+# The deployment sets the egress proxy in the uppercase variables, and chdb reads only the lowercase
+# ones. Without a lowercase copy, chdb connects directly and its reads skip the proxy's address checks.
+_CHDB_SUBPROCESS_PROXY_ENV_VARS = ("HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY")
+
# A table's URL is customer input, and URL validation only sees the first host. The S3 client follows
# redirects by default, so a URL that passes validation can send chdb to an internal address. The
# ClickHouse cluster checks every hop against remote_url_allow_hosts. chdb has no such list, so it
@@ -292,6 +296,7 @@ def run_chdb_query(query: str, timeout: float = CHDB_QUERY_TIMEOUT_SECONDS) -> s
timeout=timeout,
env={
**{name: value for name, value in os.environ.items() if name not in _CHDB_SUBPROCESS_EXCLUDED_ENV_VARS},
+ **{name.lower(): os.environ[name] for name in _CHDB_SUBPROCESS_PROXY_ENV_VARS if name in os.environ},
**_CHDB_SUBPROCESS_FORCED_ENV_VARS,
},
)
diff --git a/products/warehouse_sources/backend/tests/test_table.py b/products/warehouse_sources/backend/tests/test_table.py
index c1dafe2646c2..58118c920925 100644
--- a/products/warehouse_sources/backend/tests/test_table.py
+++ b/products/warehouse_sources/backend/tests/test_table.py
@@ -428,6 +428,8 @@ class TestRunChdbQuery:
monkeypatch.setenv("MALLOC_CONF", "background_thread:false")
monkeypatch.setenv("AWS_REGION", "us-east-1")
monkeypatch.setenv("AWS_EC2_METADATA_DISABLED", "false")
+ monkeypatch.setenv("HTTPS_PROXY", "http://web-1:[email protected]:4750/")
+ monkeypatch.delenv("https_proxy", raising=False)
completed = subprocess.CompletedProcess(args=[], returncode=0, stdout="1\n", stderr="")
with patch(
"products.warehouse_sources.backend.models.table.subprocess.run", return_value=completed
@@ -439,6 +441,7 @@ class TestRunChdbQuery:
assert "MALLOC_CONF" not in child_env
assert child_env["AWS_REGION"] == "us-east-1"
assert child_env["AWS_EC2_METADATA_DISABLED"] == "true"
+ assert child_env["https_proxy"] == "http://web-1:[email protected]:4750/"
assert mock_run.call_args.kwargs["input"] == "SET s3_max_redirects = 0; SELECT 1"
@pytest.mark.parametrize(