Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): stop chdb following s3 redirects (#120958)

Tom Owersda3681c9c15b
diff --git a/products/warehouse_sources/backend/models/table.py b/products/warehouse_sources/backend/models/table.py
index 73eef75c43bc..8a551d84a1a7 100644
--- a/products/warehouse_sources/backend/models/table.py
+++ b/products/warehouse_sources/backend/models/table.py
@@ -265,6 +265,17 @@ sys.stdout.write(str(result))
# that preloaded jemalloc.
_CHDB_SUBPROCESS_EXCLUDED_ENV_VARS = frozenset({"LD_PRELOAD", "MALLOC_CONF"})
+# The pod authenticates to AWS through its service account token and never needs the instance
+# metadata service. Turning it off keeps the AWS SDK inside chdb from asking that service for
+# credentials when a read carries none.
+_CHDB_SUBPROCESS_FORCED_ENV_VARS = {"AWS_EC2_METADATA_DISABLED": "true"}
+
+# A table's URL is customer input, and URL validation only sees the first host. The S3 client follows
+# redirects by default, so a URL that passes validation can send chdb to an internal address. The
+# ClickHouse cluster checks every hop against remote_url_allow_hosts. chdb has no such list, so it
+# follows no redirect, and a URL that needs one fails over to the cluster.
+_CHDB_GUARD_SETTINGS: dict[str, str | int] = {"s3_max_redirects": 0}
+
def run_chdb_query(query: str, timeout: float = CHDB_QUERY_TIMEOUT_SECONDS) -> str:
# The query is passed over stdin because it embeds S3 credentials — argv is world-readable.
@@ -273,11 +284,14 @@ def run_chdb_query(query: str, timeout: float = CHDB_QUERY_TIMEOUT_SECONDS) -> s
try:
process = subprocess.run(
[sys.executable, "-c", _CHDB_SUBPROCESS_SCRIPT],
- input=query,
+ input=chdb_set_statements(_CHDB_GUARD_SETTINGS) + query,
capture_output=True,
text=True,
timeout=timeout,
- env={name: value for name, value in os.environ.items() if name not in _CHDB_SUBPROCESS_EXCLUDED_ENV_VARS},
+ env={
+ **{name: value for name, value in os.environ.items() if name not in _CHDB_SUBPROCESS_EXCLUDED_ENV_VARS},
+ **_CHDB_SUBPROCESS_FORCED_ENV_VARS,
+ },
)
except subprocess.TimeoutExpired:
raise RuntimeError(f"chdb query timed out after {timeout}s")
@@ -637,6 +651,9 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Delet
# a dataset that outgrows the budget reports the size of the customer's table rather than
# a fault. get_count keeps the largest tables off chdb for the same reason.
"chdb query timed out after",
+ # chdb follows no redirect (see _CHDB_GUARD_SETTINGS), so a URL that needs one, such as a
+ # path-style URL on the wrong region, is read by the cluster, which checks every hop.
+ "too many redirects",
)
# chdb 4 links delta-kernel only in its Linux wheels, so macOS dev boxes have no deltaLake().
# On Linux the same error means the engine lost Delta support and has to reach error tracking.
@@ -690,6 +707,33 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Delet
settings["format_csv_allow_double_quotes"] = 1 if self.csv_allow_double_quotes else 0
return settings
+ def _chdb_may_read(self, read_format: str) -> bool:
+ """Whether the embedded engine may run this read, given the format it reads the URL with.
+
+ deltaLake() reads through delta-kernel's own HTTP client, which s3_max_redirects does not
+ govern, so chdb cannot stop it from following a redirect. A table with a credential reads a
+ URL the team supplied, so its Delta reads go to the cluster, which checks every hop against
+ remote_url_allow_hosts. A table without a credential reads a URL that PostHog built.
+ """
+ return read_format != DataWarehouseTableFormat.Delta or self.credential_id is None
+
+ def _describe_with_chdb(
+ self, s3_table_func: str, placeholder_context: HogQLContext, describe_settings: dict[str, str | int]
+ ) -> list[tuple[str, ...]] | None:
+ """The DESCRIBE rows read by the embedded engine, or None when it cannot answer and
+ ClickHouse has to."""
+ try:
+ quoted_placeholders = {k: escape_param_clickhouse(v) for k, v in placeholder_context.values.items()}
+ # chdb doesn't support parameterized queries
+ chdb_query = f"{chdb_set_statements(describe_settings)}DESCRIBE TABLE {s3_table_func}" % quoted_placeholders
+ chdb_result = run_chdb_query(chdb_query)
+ except Exception as chdb_error:
+ self._capture_unexpected_chdb_error(chdb_error)
+ return None
+
+ reader = csv.reader(StringIO(chdb_result))
+ return [tuple(row) for row in reader]
+
def get_columns(
self,
safe_expose_ch_error: bool = True,
@@ -697,31 +741,22 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Delet
result: list[tuple[str, ...]] | None = None
describe_settings = self._describe_settings()
placeholder_context = HogQLContext(team_id=self.team.pk)
+ # Use deltaLake() to get table schema for evolved tables
+ describe_format = DataWarehouseTableFormat.Delta if self.format == "DeltaS3Wrapper" else self.format
s3_table_func = build_function_call(
url=self.url_pattern,
queryable_folder=self.queryable_folder,
- format="Delta" # Use deltaLake() to get table schema for evolved tables
- if self.format == "DeltaS3Wrapper"
- else self.format,
+ format=describe_format,
access_key=self.credential.access_key if self.credential else None,
access_secret=self.credential.access_secret if self.credential else None,
context=placeholder_context,
table_size_mib=0, # Use the non-cluster s3 table function for chdb
)
- try:
- # chdb hangs in CI during tests
- if TEST:
- raise Exception()
-
- quoted_placeholders = {k: escape_param_clickhouse(v) for k, v in placeholder_context.values.items()}
- # chdb doesn't support parameterized queries
- chdb_query = f"{chdb_set_statements(describe_settings)}DESCRIBE TABLE {s3_table_func}" % quoted_placeholders
- chdb_result = run_chdb_query(chdb_query)
- reader = csv.reader(StringIO(chdb_result))
- result = [tuple(row) for row in reader]
- except Exception as chdb_error:
- self._capture_unexpected_chdb_error(chdb_error)
+ # chdb hangs in CI during tests
+ if not TEST and self._chdb_may_read(describe_format):
+ result = self._describe_with_chdb(s3_table_func, placeholder_context, describe_settings)
+ if result is None:
tag_queries(
team_id=self.team.pk,
table_id=self.id,
@@ -858,7 +893,7 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Delet
def get_count(self, safe_expose_ch_error=True) -> int:
# chdb hangs in CI during tests
- if not TEST and self._count_fits_the_chdb_budget():
+ if not TEST and self._count_fits_the_chdb_budget() and self._chdb_may_read(self.format):
chdb_count = self._count_with_chdb()
if chdb_count is not None:
return chdb_count
diff --git a/products/warehouse_sources/backend/tests/test_table.py b/products/warehouse_sources/backend/tests/test_table.py
index b2b961dea4cf..c1dafe2646c2 100644
--- a/products/warehouse_sources/backend/tests/test_table.py
+++ b/products/warehouse_sources/backend/tests/test_table.py
@@ -346,6 +346,75 @@ class TestGetCountEngineChoice:
assert "s3Cluster(" in mock_sync_execute.call_args.args[0]
+class TestChdbEngineChoiceByFormat:
+ def _table(self, table_format: str, has_credential: bool) -> DataWarehouseTable:
+ return DataWarehouseTable(
+ name="t",
+ format=table_format,
+ team=Team(id=1),
+ url_pattern="https://bucket.example.com/t/",
+ credential=DataWarehouseCredential(access_key="key", access_secret="secret") if has_credential else None,
+ )
+
+ @pytest.mark.parametrize(
+ "table_format,has_credential,uses_chdb",
+ [
+ (DataWarehouseTable.TableFormat.Delta, True, False),
+ (DataWarehouseTable.TableFormat.DeltaS3Wrapper, True, False),
+ (DataWarehouseTable.TableFormat.Delta, False, True),
+ (DataWarehouseTable.TableFormat.DeltaS3Wrapper, False, True),
+ (DataWarehouseTable.TableFormat.Parquet, True, True),
+ (DataWarehouseTable.TableFormat.CSVWithNames, True, True),
+ (DataWarehouseTable.TableFormat.JSON, True, True),
+ ],
+ )
+ def test_get_columns_reads_customer_delta_urls_on_the_cluster(
+ self, table_format: str, has_credential: bool, uses_chdb: bool
+ ) -> None:
+ with (
+ patch("products.warehouse_sources.backend.models.table.TEST", False),
+ patch(
+ "products.warehouse_sources.backend.models.table.run_chdb_query", return_value="id,String\n"
+ ) as mock_run_chdb,
+ patch(
+ "products.warehouse_sources.backend.models.table.sync_execute", return_value=[("id", "String")]
+ ) as mock_sync_execute,
+ ):
+ columns = self._table(table_format, has_credential).get_columns()
+
+ assert columns["id"]["clickhouse"] == "String"
+ assert mock_run_chdb.called is uses_chdb
+ assert mock_sync_execute.called is not uses_chdb
+
+ @pytest.mark.parametrize(
+ "table_format,has_credential,uses_chdb",
+ [
+ (DataWarehouseTable.TableFormat.Delta, True, False),
+ (DataWarehouseTable.TableFormat.Delta, False, True),
+ # A count of a DeltaS3Wrapper table reads its parquet files through s3(), not deltaLake().
+ (DataWarehouseTable.TableFormat.DeltaS3Wrapper, True, True),
+ (DataWarehouseTable.TableFormat.Parquet, True, True),
+ ],
+ )
+ def test_get_count_reads_customer_delta_urls_on_the_cluster(
+ self, table_format: str, has_credential: bool, uses_chdb: bool
+ ) -> None:
+ with (
+ patch("products.warehouse_sources.backend.models.table.TEST", False),
+ patch(
+ "products.warehouse_sources.backend.models.table.run_chdb_query", return_value="7\n"
+ ) as mock_run_chdb,
+ patch(
+ "products.warehouse_sources.backend.models.table.sync_execute", return_value=[(7,)]
+ ) as mock_sync_execute,
+ ):
+ count = self._table(table_format, has_credential).get_count()
+
+ assert count == 7
+ assert mock_run_chdb.called is uses_chdb
+ assert mock_sync_execute.called is not uses_chdb
+
+
class TestRunChdbQuery:
def test_hung_query_is_killed_and_raises_instead_of_blocking(self) -> None:
# Real subprocess: chdb import alone exceeds the timeout, so this exercises the
@@ -354,10 +423,11 @@ class TestRunChdbQuery:
with pytest.raises(RuntimeError, match="timed out"):
run_chdb_query("SELECT sleep(2)", timeout=0.5)
- def test_child_process_does_not_inherit_a_preloaded_allocator(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ def test_child_process_runs_with_hardened_environment_and_settings(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("LD_PRELOAD", "/usr/lib/aarch64-linux-gnu/libjemalloc.so.2")
monkeypatch.setenv("MALLOC_CONF", "background_thread:false")
monkeypatch.setenv("AWS_REGION", "us-east-1")
+ monkeypatch.setenv("AWS_EC2_METADATA_DISABLED", "false")
completed = subprocess.CompletedProcess(args=[], returncode=0, stdout="1\n", stderr="")
with patch(
"products.warehouse_sources.backend.models.table.subprocess.run", return_value=completed
@@ -368,6 +438,8 @@ class TestRunChdbQuery:
assert "LD_PRELOAD" not in child_env
assert "MALLOC_CONF" not in child_env
assert child_env["AWS_REGION"] == "us-east-1"
+ assert child_env["AWS_EC2_METADATA_DISABLED"] == "true"
+ assert mock_run.call_args.kwargs["input"] == "SET s3_max_redirects = 0; SELECT 1"
@pytest.mark.parametrize(
"stderr",
@@ -378,6 +450,8 @@ class TestRunChdbQuery:
# wording, so a ClickHouse rephrasing has to fail the test rather than pass silently.
"Code: 48. DB::Exception: Reading from files with different schema is not possible. "
"The table has 23 columns, which is different from 24 columns in the file. (NOT_IMPLEMENTED)",
+ "Code: 579. DB::Exception: Too many redirects while trying to access "
+ "https://s3.amazonaws.com/bucket/table/. (TOO_MANY_REDIRECTS)",
],
)
def test_suppressed_chdb_error_classification_survives_subprocess_boundary(self, stderr: str) -> None: