:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse-sources): stop chdb following s3 redirects (#120958)
Tom Owersda3681c9c15b
diff --git a/products/warehouse_sources/backend/models/table.py b/products/warehouse_sources/backend/models/table.pyindex 73eef75c43bc..8a551d84a1a7 100644--- a/products/warehouse_sources/backend/models/table.py+++ b/products/warehouse_sources/backend/models/table.py@@ -265,6 +265,17 @@ sys.stdout.write(str(result))# that preloaded jemalloc._CHDB_SUBPROCESS_EXCLUDED_ENV_VARS = frozenset({"LD_PRELOAD", "MALLOC_CONF"})+# The pod authenticates to AWS through its service account token and never needs the instance+# metadata service. Turning it off keeps the AWS SDK inside chdb from asking that service for+# credentials when a read carries none.+_CHDB_SUBPROCESS_FORCED_ENV_VARS = {"AWS_EC2_METADATA_DISABLED": "true"}++# A table's URL is customer input, and URL validation only sees the first host. The S3 client follows+# redirects by default, so a URL that passes validation can send chdb to an internal address. The+# ClickHouse cluster checks every hop against remote_url_allow_hosts. chdb has no such list, so it+# follows no redirect, and a URL that needs one fails over to the cluster.+_CHDB_GUARD_SETTINGS: dict[str, str | int] = {"s3_max_redirects": 0}+def run_chdb_query(query: str, timeout: float = CHDB_QUERY_TIMEOUT_SECONDS) -> str:# The query is passed over stdin because it embeds S3 credentials — argv is world-readable.@@ -273,11 +284,14 @@ def run_chdb_query(query: str, timeout: float = CHDB_QUERY_TIMEOUT_SECONDS) -> stry:process = subprocess.run([sys.executable, "-c", _CHDB_SUBPROCESS_SCRIPT],- input=query,+ input=chdb_set_statements(_CHDB_GUARD_SETTINGS) + query,capture_output=True,text=True,timeout=timeout,- env={name: value for name, value in os.environ.items() if name not in _CHDB_SUBPROCESS_EXCLUDED_ENV_VARS},+ env={+ **{name: value for name, value in os.environ.items() if name not in _CHDB_SUBPROCESS_EXCLUDED_ENV_VARS},+ **_CHDB_SUBPROCESS_FORCED_ENV_VARS,+ },)except subprocess.TimeoutExpired:raise RuntimeError(f"chdb query timed out after {timeout}s")@@ -637,6 +651,9 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Delet# a dataset that outgrows the budget reports the size of the customer's table rather than# a fault. get_count keeps the largest tables off chdb for the same reason."chdb query timed out after",+ # chdb follows no redirect (see _CHDB_GUARD_SETTINGS), so a URL that needs one, such as a+ # path-style URL on the wrong region, is read by the cluster, which checks every hop.+ "too many redirects",)# chdb 4 links delta-kernel only in its Linux wheels, so macOS dev boxes have no deltaLake().# On Linux the same error means the engine lost Delta support and has to reach error tracking.@@ -690,6 +707,33 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Deletsettings["format_csv_allow_double_quotes"] = 1 if self.csv_allow_double_quotes else 0return settings+ def _chdb_may_read(self, read_format: str) -> bool:+ """Whether the embedded engine may run this read, given the format it reads the URL with.++ deltaLake() reads through delta-kernel's own HTTP client, which s3_max_redirects does not+ govern, so chdb cannot stop it from following a redirect. A table with a credential reads a+ URL the team supplied, so its Delta reads go to the cluster, which checks every hop against+ remote_url_allow_hosts. A table without a credential reads a URL that PostHog built.+ """+ return read_format != DataWarehouseTableFormat.Delta or self.credential_id is None++ def _describe_with_chdb(+ self, s3_table_func: str, placeholder_context: HogQLContext, describe_settings: dict[str, str | int]+ ) -> list[tuple[str, ...]] | None:+ """The DESCRIBE rows read by the embedded engine, or None when it cannot answer and+ ClickHouse has to."""+ try:+ quoted_placeholders = {k: escape_param_clickhouse(v) for k, v in placeholder_context.values.items()}+ # chdb doesn't support parameterized queries+ chdb_query = f"{chdb_set_statements(describe_settings)}DESCRIBE TABLE {s3_table_func}" % quoted_placeholders+ chdb_result = run_chdb_query(chdb_query)+ except Exception as chdb_error:+ self._capture_unexpected_chdb_error(chdb_error)+ return None++ reader = csv.reader(StringIO(chdb_result))+ return [tuple(row) for row in reader]+def get_columns(self,safe_expose_ch_error: bool = True,@@ -697,31 +741,22 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Deletresult: list[tuple[str, ...]] | None = Nonedescribe_settings = self._describe_settings()placeholder_context = HogQLContext(team_id=self.team.pk)+ # Use deltaLake() to get table schema for evolved tables+ describe_format = DataWarehouseTableFormat.Delta if self.format == "DeltaS3Wrapper" else self.formats3_table_func = build_function_call(url=self.url_pattern,queryable_folder=self.queryable_folder,- format="Delta" # Use deltaLake() to get table schema for evolved tables- if self.format == "DeltaS3Wrapper"- else self.format,+ format=describe_format,access_key=self.credential.access_key if self.credential else None,access_secret=self.credential.access_secret if self.credential else None,context=placeholder_context,table_size_mib=0, # Use the non-cluster s3 table function for chdb)- try:- # chdb hangs in CI during tests- if TEST:- raise Exception()-- quoted_placeholders = {k: escape_param_clickhouse(v) for k, v in placeholder_context.values.items()}- # chdb doesn't support parameterized queries- chdb_query = f"{chdb_set_statements(describe_settings)}DESCRIBE TABLE {s3_table_func}" % quoted_placeholders- chdb_result = run_chdb_query(chdb_query)- reader = csv.reader(StringIO(chdb_result))- result = [tuple(row) for row in reader]- except Exception as chdb_error:- self._capture_unexpected_chdb_error(chdb_error)+ # chdb hangs in CI during tests+ if not TEST and self._chdb_may_read(describe_format):+ result = self._describe_with_chdb(s3_table_func, placeholder_context, describe_settings)+ if result is None:tag_queries(team_id=self.team.pk,table_id=self.id,@@ -858,7 +893,7 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Deletdef get_count(self, safe_expose_ch_error=True) -> int:# chdb hangs in CI during tests- if not TEST and self._count_fits_the_chdb_budget():+ if not TEST and self._count_fits_the_chdb_budget() and self._chdb_may_read(self.format):chdb_count = self._count_with_chdb()if chdb_count is not None:return chdb_countdiff --git a/products/warehouse_sources/backend/tests/test_table.py b/products/warehouse_sources/backend/tests/test_table.pyindex b2b961dea4cf..c1dafe2646c2 100644--- a/products/warehouse_sources/backend/tests/test_table.py+++ b/products/warehouse_sources/backend/tests/test_table.py@@ -346,6 +346,75 @@ class TestGetCountEngineChoice:assert "s3Cluster(" in mock_sync_execute.call_args.args[0]+class TestChdbEngineChoiceByFormat:+ def _table(self, table_format: str, has_credential: bool) -> DataWarehouseTable:+ return DataWarehouseTable(+ name="t",+ format=table_format,+ team=Team(id=1),+ url_pattern="https://bucket.example.com/t/",+ credential=DataWarehouseCredential(access_key="key", access_secret="secret") if has_credential else None,+ )++ @pytest.mark.parametrize(+ "table_format,has_credential,uses_chdb",+ [+ (DataWarehouseTable.TableFormat.Delta, True, False),+ (DataWarehouseTable.TableFormat.DeltaS3Wrapper, True, False),+ (DataWarehouseTable.TableFormat.Delta, False, True),+ (DataWarehouseTable.TableFormat.DeltaS3Wrapper, False, True),+ (DataWarehouseTable.TableFormat.Parquet, True, True),+ (DataWarehouseTable.TableFormat.CSVWithNames, True, True),+ (DataWarehouseTable.TableFormat.JSON, True, True),+ ],+ )+ def test_get_columns_reads_customer_delta_urls_on_the_cluster(+ self, table_format: str, has_credential: bool, uses_chdb: bool+ ) -> None:+ with (+ patch("products.warehouse_sources.backend.models.table.TEST", False),+ patch(+ "products.warehouse_sources.backend.models.table.run_chdb_query", return_value="id,String\n"+ ) as mock_run_chdb,+ patch(+ "products.warehouse_sources.backend.models.table.sync_execute", return_value=[("id", "String")]+ ) as mock_sync_execute,+ ):+ columns = self._table(table_format, has_credential).get_columns()++ assert columns["id"]["clickhouse"] == "String"+ assert mock_run_chdb.called is uses_chdb+ assert mock_sync_execute.called is not uses_chdb++ @pytest.mark.parametrize(+ "table_format,has_credential,uses_chdb",+ [+ (DataWarehouseTable.TableFormat.Delta, True, False),+ (DataWarehouseTable.TableFormat.Delta, False, True),+ # A count of a DeltaS3Wrapper table reads its parquet files through s3(), not deltaLake().+ (DataWarehouseTable.TableFormat.DeltaS3Wrapper, True, True),+ (DataWarehouseTable.TableFormat.Parquet, True, True),+ ],+ )+ def test_get_count_reads_customer_delta_urls_on_the_cluster(+ self, table_format: str, has_credential: bool, uses_chdb: bool+ ) -> None:+ with (+ patch("products.warehouse_sources.backend.models.table.TEST", False),+ patch(+ "products.warehouse_sources.backend.models.table.run_chdb_query", return_value="7\n"+ ) as mock_run_chdb,+ patch(+ "products.warehouse_sources.backend.models.table.sync_execute", return_value=[(7,)]+ ) as mock_sync_execute,+ ):+ count = self._table(table_format, has_credential).get_count()++ assert count == 7+ assert mock_run_chdb.called is uses_chdb+ assert mock_sync_execute.called is not uses_chdb++class TestRunChdbQuery:def test_hung_query_is_killed_and_raises_instead_of_blocking(self) -> None:# Real subprocess: chdb import alone exceeds the timeout, so this exercises the@@ -354,10 +423,11 @@ class TestRunChdbQuery:with pytest.raises(RuntimeError, match="timed out"):run_chdb_query("SELECT sleep(2)", timeout=0.5)- def test_child_process_does_not_inherit_a_preloaded_allocator(self, monkeypatch: pytest.MonkeyPatch) -> None:+ def test_child_process_runs_with_hardened_environment_and_settings(self, monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setenv("LD_PRELOAD", "/usr/lib/aarch64-linux-gnu/libjemalloc.so.2")monkeypatch.setenv("MALLOC_CONF", "background_thread:false")monkeypatch.setenv("AWS_REGION", "us-east-1")+ monkeypatch.setenv("AWS_EC2_METADATA_DISABLED", "false")completed = subprocess.CompletedProcess(args=[], returncode=0, stdout="1\n", stderr="")with patch("products.warehouse_sources.backend.models.table.subprocess.run", return_value=completed@@ -368,6 +438,8 @@ class TestRunChdbQuery:assert "LD_PRELOAD" not in child_envassert "MALLOC_CONF" not in child_envassert child_env["AWS_REGION"] == "us-east-1"+ assert child_env["AWS_EC2_METADATA_DISABLED"] == "true"+ assert mock_run.call_args.kwargs["input"] == "SET s3_max_redirects = 0; SELECT 1"@pytest.mark.parametrize("stderr",@@ -378,6 +450,8 @@ class TestRunChdbQuery:# wording, so a ClickHouse rephrasing has to fail the test rather than pass silently."Code: 48. DB::Exception: Reading from files with different schema is not possible. ""The table has 23 columns, which is different from 24 columns in the file. (NOT_IMPLEMENTED)",+ "Code: 579. DB::Exception: Too many redirects while trying to access "+ "https://s3.amazonaws.com/bucket/table/. (TOO_MANY_REDIRECTS)",],)def test_suppressed_chdb_error_classification_survives_subprocess_boundary(self, stderr: str) -> None: