:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse): aggregate table statistics with arrow compute (#126178)
estefania7928d7d6c8ec
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.pyindex daf743373921..fe1050499f88 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py@@ -64,6 +64,8 @@ MAX_RECOMPUTE_INTERVAL = timedelta(days=int(os.getenv("WAREHOUSE_STATS_MAX_RECOM# Each commit since the last computation is one small object-store read; past this many the full# Add-action scan is the cheaper path again.MAX_INCREMENTAL_COMMITS = int(os.getenv("WAREHOUSE_STATS_MAX_INCREMENTAL_COMMITS", "200"))+# Files x columns cap for the Add-action scan: Arrow holds about 12 bytes per cell, so the default is ~300 MB.+MAX_ADD_ACTION_CELLS = int(os.getenv("WAREHOUSE_STATS_MAX_ADD_ACTION_CELLS", "25000000"))# Product-analytics events — query these to track statistics volume, columns profiled, skips, and errors.EVENT_STARTED = "data warehouse table statistics started"@@ -116,6 +118,21 @@ class _ColumnStat:has_min_max: bool+def _arrow_extreme(values: Any, which: str) -> Any:+ """The smallest or largest non-null value of one Arrow column as a Python value, or None."""+ import pyarrow as pa # noqa: PLC0415 — heavy dep kept off this module's flag-check import path+ import pyarrow.compute as pc # noqa: PLC0415++ try:+ return pc.min_max(values)[which].as_py()+ except pa.ArrowNotImplementedError:+ # Arrow has no ordering for some types (list, struct), but Python can still compare their values.+ non_null = [v for v in values.to_pylist() if v is not None]+ if not non_null:+ return None+ return min(non_null) if which == "min" else max(non_null)++def _aggregate_add_action_stats(add_actions: Any, columns: dict[str, Any]) -> tuple[int, dict[str, _ColumnStat]]:"""Aggregate the Delta log's per-file stats into per-column whole-table stats.@@ -124,29 +141,30 @@ def _aggregate_add_action_stats(add_actions: Any, columns: dict[str, Any]) -> tuReturns `(row_count, {column_name: _ColumnStat})`. Columns with no log stats get `has_min_max=False`."""import pyarrow as pa # noqa: PLC0415 — heavy dep kept off this module's flag-check import path+ import pyarrow.compute as pc # noqa: PLC0415- # deltalake>=1.x returns an arro3 RecordBatch (no `to_pydict`); pa.table() normalizes both that- # (via the Arrow C interface) and an already-pyarrow input to a pyarrow Table we can read as a dict.- data = pa.table(add_actions).to_pydict()- row_count = sum(n for n in data.get("num_records", []) if n is not None)+ # deltalake>=1.x hands back an arro3 RecordBatch; the Arrow aggregates below avoid one Python object per file x column.+ data = add_actions if isinstance(add_actions, pa.Table | pa.RecordBatch) else pa.table(add_actions)+ present = set(data.schema.names)+ row_count = (pc.sum(data.column("num_records")).as_py() or 0) if "num_records" in present else 0result: dict[str, _ColumnStat] = {}for name, definition in (columns or {}).items():null_key, min_key, max_key = f"null_count.{name}", f"min.{name}", f"max.{name}"# `null_count = None` (unknown) when the log carried no per-file null counts — either the key is- # absent or every file's value is None. Mirrors the min/max guard below; a real zero stays 0.- null_values = [v for v in data.get(null_key, []) if v is not None] if null_key in data else []- null_count = sum(null_values) if null_values else None+ # absent or every file's value is None. `pc.sum` skips nulls and returns null when none remain;+ # a real zero stays 0.+ null_count = pc.sum(data.column(null_key)).as_py() if null_key in present else None- mins = [v for v in data.get(min_key, []) if v is not None] if min_key in data else []- maxs = [v for v in data.get(max_key, []) if v is not None] if max_key in data else []+ min_raw = _arrow_extreme(data.column(min_key), "min") if min_key in present else None+ max_raw = _arrow_extreme(data.column(max_key), "max") if max_key in present else None# A source string column can carry a NUL (0x00) byte; min_value/max_value land in a Postgres# text column, which rejects it outright. Stripping it is consistent with treating these# bounds as approximate (delta-rs already truncates long strings here).- min_value = str(min(mins)).replace("\x00", "") if mins else None- max_value = str(max(maxs)).replace("\x00", "") if maxs else None- has_min_max = bool(mins or maxs)+ min_value = str(min_raw).replace("\x00", "") if min_raw is not None else None+ max_value = str(max_raw).replace("\x00", "") if max_raw is not None else None+ has_min_max = min_raw is not None or max_raw is not Noneresult[name] = _ColumnStat(column_type=clean_type(_column_type(definition)) or "unknown",@@ -607,6 +625,7 @@ def compute_table_statistics_sync(team_id: int, schema_id: uuid.UUID) -> dict[stdelta_version=delta_version,log=log,)+ file_count: int | None = Noneif folded is not None:row_count, stats_by_column = foldedbasis = "incremental"@@ -620,6 +639,16 @@ def compute_table_statistics_sync(team_id: int, schema_id: uuid.UUID) -> dict[stemit_completed("skipped", reason="no_delta_table")return {"status": "skipped", "reason": "no_delta_table"}delta_version = delta_table.version()+ file_count = len(delta_table.file_uris())+ if file_count * len(columns) > MAX_ADD_ACTION_CELLS:+ emit_completed(+ "skipped",+ reason="too_many_files",+ file_count=file_count,+ target_columns=len(columns),+ delta_version=delta_version,+ )+ return {"status": "skipped", "reason": "too_many_files"}add_actions = delta_table.get_add_actions(flatten=True)if add_actions.num_rows == 0:emit_completed("skipped", reason="no_files")@@ -642,7 +671,14 @@ def compute_table_statistics_sync(team_id: int, schema_id: uuid.UUID) -> dict[stfor column_name, stat in stats_by_column.items():_upsert_statistics(team, table, column_name, row_count, stat, delta_version, basis)- log.info("warehouse_statistics.done", columns=len(stats_by_column), row_count=row_count, basis=basis)+ log.info(+ "warehouse_statistics.done",+ columns=len(stats_by_column),+ row_count=row_count,+ basis=basis,+ file_count=file_count,+ target_columns=len(columns),+ )emit_completed("done", columns=len(stats_by_column), row_count=row_count, delta_version=delta_version, basis=basis)return {"status": "done", "columns": len(stats_by_column), "row_count": row_count, "basis": basis}diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.pyindex 9b7c5fec1cd7..12e701e74d10 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py@@ -3,6 +3,7 @@ import uuidimport datetime as dtimport tempfileimport threading+import dataclassesfrom decimal import Decimalfrom typing import Any@@ -79,6 +80,37 @@ class TestAggregateAddActionStats:assert stats["amount"].null_count == 3assert stats["amount"].min_value == "3"+ def test_mixed_type_columns_aggregate_independently(self) -> None:+ utc = dt.UTC+ add_actions = pa.table(+ {+ "path": ["a.parquet", "b.parquet", "c.parquet"],+ "size_bytes": [100, 200, 300],+ "num_records": [10, None, 5],+ "null_count.id": [0, 0, 0],+ "min.id": [7, 1, 4],+ "max.id": [9, 30, 4],+ "null_count.name": [1, None, 2],+ "min.name": ["éclair", "apple", None],+ "max.name": ["zebra", "émile", None],+ "null_count.created_at": [None, None, None],+ "min.created_at": [dt.datetime(2024, 3, 1, tzinfo=utc), None, dt.datetime(2024, 1, 1, tzinfo=utc)],+ "max.created_at": [dt.datetime(2024, 3, 2, tzinfo=utc), None, dt.datetime(2025, 1, 1, tzinfo=utc)],+ "min.unregistered": [1, 2, 3],+ }+ )+ columns = {"id": "Int64", "name": "String", "created_at": "DateTime64", "payload": "String"}++ row_count, stats = _aggregate_add_action_stats(add_actions, columns)++ assert row_count == 15+ assert {name: dataclasses.astuple(stat) for name, stat in stats.items()} == {+ "id": ("Int64", 0, "1", "30", True),+ "name": ("String", 3, "apple", "émile", True),+ "created_at": ("DateTime64", None, "2024-01-01 00:00:00+00:00", "2025-01-01 00:00:00+00:00", True),+ "payload": ("String", None, None, None, False),+ }+def test_column_without_log_stats_marks_has_min_max_false(self) -> None:# A column present in the table but with no min/max/null in the Delta log (e.g. nested type).add_actions = pa.table({"num_records": [5]})@@ -135,6 +167,7 @@ class TestAggregateAddActionStats:# text column, which rejects it outright (DataError: "PostgreSQL text fields cannot# contain NUL (0x00) bytes"). It must be stripped here, before the DB write.("ab\x00c", "z\x00", "abc", "z"),+ ([1, 2], [3], "[1, 2]", "[3]"),],)def test_min_max_coerced_to_string(self, min_val, max_val, expected_min, expected_max) -> None:@@ -1078,6 +1111,23 @@ class TestComputeTableStatisticsSync:result = compute_table_statistics_sync(team.id, schema.id)assert result == {"status": "skipped", "reason": "no_files"}+ @parameterized.expand([("at_the_cap", 2, "done"), ("over_the_cap", 3, "skipped")])+ def test_add_action_scan_is_capped_by_files_times_columns(+ self, _name: str, file_count: int, expected_status: str+ ) -> None:+ team = self._team()+ schema, table, _ = self._schema_table_job(team, columns={"amount": "Int64", "currency": "String"})+ helper = self._mock_delta(pa.table({"num_records": [1] * file_count}))+ helper.get_delta_table.return_value.file_uris.return_value = [f"f{i}.parquet" for i in range(file_count)]+ with patch(DELTA_HELPER_PATH, return_value=helper), patch.object(comp, "MAX_ADD_ACTION_CELLS", 4):+ result = compute_table_statistics_sync(team.id, schema.id)++ assert result["status"] == expected_status+ if expected_status == "skipped":+ assert result == {"status": "skipped", "reason": "too_many_files"}+ helper.get_delta_table.return_value.get_add_actions.assert_not_called()+ assert not WarehouseColumnStatistics.objects.for_team(team.id).filter(table_id=table.id).exists()+@pytest.mark.django_db(transaction=True)class TestComputeTableStatisticsActivity: