Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse): aggregate table statistics with arrow compute (#126178)

estefania7928d7d6c8ec
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py
index daf743373921..fe1050499f88 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py
@@ -64,6 +64,8 @@ MAX_RECOMPUTE_INTERVAL = timedelta(days=int(os.getenv("WAREHOUSE_STATS_MAX_RECOM
# Each commit since the last computation is one small object-store read; past this many the full
# Add-action scan is the cheaper path again.
MAX_INCREMENTAL_COMMITS = int(os.getenv("WAREHOUSE_STATS_MAX_INCREMENTAL_COMMITS", "200"))
+# Files x columns cap for the Add-action scan: Arrow holds about 12 bytes per cell, so the default is ~300 MB.
+MAX_ADD_ACTION_CELLS = int(os.getenv("WAREHOUSE_STATS_MAX_ADD_ACTION_CELLS", "25000000"))
# Product-analytics events — query these to track statistics volume, columns profiled, skips, and errors.
EVENT_STARTED = "data warehouse table statistics started"
@@ -116,6 +118,21 @@ class _ColumnStat:
has_min_max: bool
+def _arrow_extreme(values: Any, which: str) -> Any:
+ """The smallest or largest non-null value of one Arrow column as a Python value, or None."""
+ import pyarrow as pa # noqa: PLC0415 — heavy dep kept off this module's flag-check import path
+ import pyarrow.compute as pc # noqa: PLC0415
+
+ try:
+ return pc.min_max(values)[which].as_py()
+ except pa.ArrowNotImplementedError:
+ # Arrow has no ordering for some types (list, struct), but Python can still compare their values.
+ non_null = [v for v in values.to_pylist() if v is not None]
+ if not non_null:
+ return None
+ return min(non_null) if which == "min" else max(non_null)
+
+
def _aggregate_add_action_stats(add_actions: Any, columns: dict[str, Any]) -> tuple[int, dict[str, _ColumnStat]]:
"""Aggregate the Delta log's per-file stats into per-column whole-table stats.
@@ -124,29 +141,30 @@ def _aggregate_add_action_stats(add_actions: Any, columns: dict[str, Any]) -> tu
Returns `(row_count, {column_name: _ColumnStat})`. Columns with no log stats get `has_min_max=False`.
"""
import pyarrow as pa # noqa: PLC0415 — heavy dep kept off this module's flag-check import path
+ import pyarrow.compute as pc # noqa: PLC0415
- # deltalake>=1.x returns an arro3 RecordBatch (no `to_pydict`); pa.table() normalizes both that
- # (via the Arrow C interface) and an already-pyarrow input to a pyarrow Table we can read as a dict.
- data = pa.table(add_actions).to_pydict()
- row_count = sum(n for n in data.get("num_records", []) if n is not None)
+ # deltalake>=1.x hands back an arro3 RecordBatch; the Arrow aggregates below avoid one Python object per file x column.
+ data = add_actions if isinstance(add_actions, pa.Table | pa.RecordBatch) else pa.table(add_actions)
+ present = set(data.schema.names)
+ row_count = (pc.sum(data.column("num_records")).as_py() or 0) if "num_records" in present else 0
result: dict[str, _ColumnStat] = {}
for name, definition in (columns or {}).items():
null_key, min_key, max_key = f"null_count.{name}", f"min.{name}", f"max.{name}"
# `null_count = None` (unknown) when the log carried no per-file null counts — either the key is
- # absent or every file's value is None. Mirrors the min/max guard below; a real zero stays 0.
- null_values = [v for v in data.get(null_key, []) if v is not None] if null_key in data else []
- null_count = sum(null_values) if null_values else None
+ # absent or every file's value is None. `pc.sum` skips nulls and returns null when none remain;
+ # a real zero stays 0.
+ null_count = pc.sum(data.column(null_key)).as_py() if null_key in present else None
- mins = [v for v in data.get(min_key, []) if v is not None] if min_key in data else []
- maxs = [v for v in data.get(max_key, []) if v is not None] if max_key in data else []
+ min_raw = _arrow_extreme(data.column(min_key), "min") if min_key in present else None
+ max_raw = _arrow_extreme(data.column(max_key), "max") if max_key in present else None
# A source string column can carry a NUL (0x00) byte; min_value/max_value land in a Postgres
# text column, which rejects it outright. Stripping it is consistent with treating these
# bounds as approximate (delta-rs already truncates long strings here).
- min_value = str(min(mins)).replace("\x00", "") if mins else None
- max_value = str(max(maxs)).replace("\x00", "") if maxs else None
- has_min_max = bool(mins or maxs)
+ min_value = str(min_raw).replace("\x00", "") if min_raw is not None else None
+ max_value = str(max_raw).replace("\x00", "") if max_raw is not None else None
+ has_min_max = min_raw is not None or max_raw is not None
result[name] = _ColumnStat(
column_type=clean_type(_column_type(definition)) or "unknown",
@@ -607,6 +625,7 @@ def compute_table_statistics_sync(team_id: int, schema_id: uuid.UUID) -> dict[st
delta_version=delta_version,
log=log,
)
+ file_count: int | None = None
if folded is not None:
row_count, stats_by_column = folded
basis = "incremental"
@@ -620,6 +639,16 @@ def compute_table_statistics_sync(team_id: int, schema_id: uuid.UUID) -> dict[st
emit_completed("skipped", reason="no_delta_table")
return {"status": "skipped", "reason": "no_delta_table"}
delta_version = delta_table.version()
+ file_count = len(delta_table.file_uris())
+ if file_count * len(columns) > MAX_ADD_ACTION_CELLS:
+ emit_completed(
+ "skipped",
+ reason="too_many_files",
+ file_count=file_count,
+ target_columns=len(columns),
+ delta_version=delta_version,
+ )
+ return {"status": "skipped", "reason": "too_many_files"}
add_actions = delta_table.get_add_actions(flatten=True)
if add_actions.num_rows == 0:
emit_completed("skipped", reason="no_files")
@@ -642,7 +671,14 @@ def compute_table_statistics_sync(team_id: int, schema_id: uuid.UUID) -> dict[st
for column_name, stat in stats_by_column.items():
_upsert_statistics(team, table, column_name, row_count, stat, delta_version, basis)
- log.info("warehouse_statistics.done", columns=len(stats_by_column), row_count=row_count, basis=basis)
+ log.info(
+ "warehouse_statistics.done",
+ columns=len(stats_by_column),
+ row_count=row_count,
+ basis=basis,
+ file_count=file_count,
+ target_columns=len(columns),
+ )
emit_completed("done", columns=len(stats_by_column), row_count=row_count, delta_version=delta_version, basis=basis)
return {"status": "done", "columns": len(stats_by_column), "row_count": row_count, "basis": basis}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py
index 9b7c5fec1cd7..12e701e74d10 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py
@@ -3,6 +3,7 @@ import uuid
import datetime as dt
import tempfile
import threading
+import dataclasses
from decimal import Decimal
from typing import Any
@@ -79,6 +80,37 @@ class TestAggregateAddActionStats:
assert stats["amount"].null_count == 3
assert stats["amount"].min_value == "3"
+ def test_mixed_type_columns_aggregate_independently(self) -> None:
+ utc = dt.UTC
+ add_actions = pa.table(
+ {
+ "path": ["a.parquet", "b.parquet", "c.parquet"],
+ "size_bytes": [100, 200, 300],
+ "num_records": [10, None, 5],
+ "null_count.id": [0, 0, 0],
+ "min.id": [7, 1, 4],
+ "max.id": [9, 30, 4],
+ "null_count.name": [1, None, 2],
+ "min.name": ["éclair", "apple", None],
+ "max.name": ["zebra", "émile", None],
+ "null_count.created_at": [None, None, None],
+ "min.created_at": [dt.datetime(2024, 3, 1, tzinfo=utc), None, dt.datetime(2024, 1, 1, tzinfo=utc)],
+ "max.created_at": [dt.datetime(2024, 3, 2, tzinfo=utc), None, dt.datetime(2025, 1, 1, tzinfo=utc)],
+ "min.unregistered": [1, 2, 3],
+ }
+ )
+ columns = {"id": "Int64", "name": "String", "created_at": "DateTime64", "payload": "String"}
+
+ row_count, stats = _aggregate_add_action_stats(add_actions, columns)
+
+ assert row_count == 15
+ assert {name: dataclasses.astuple(stat) for name, stat in stats.items()} == {
+ "id": ("Int64", 0, "1", "30", True),
+ "name": ("String", 3, "apple", "émile", True),
+ "created_at": ("DateTime64", None, "2024-01-01 00:00:00+00:00", "2025-01-01 00:00:00+00:00", True),
+ "payload": ("String", None, None, None, False),
+ }
+
def test_column_without_log_stats_marks_has_min_max_false(self) -> None:
# A column present in the table but with no min/max/null in the Delta log (e.g. nested type).
add_actions = pa.table({"num_records": [5]})
@@ -135,6 +167,7 @@ class TestAggregateAddActionStats:
# text column, which rejects it outright (DataError: "PostgreSQL text fields cannot
# contain NUL (0x00) bytes"). It must be stripped here, before the DB write.
("ab\x00c", "z\x00", "abc", "z"),
+ ([1, 2], [3], "[1, 2]", "[3]"),
],
)
def test_min_max_coerced_to_string(self, min_val, max_val, expected_min, expected_max) -> None:
@@ -1078,6 +1111,23 @@ class TestComputeTableStatisticsSync:
result = compute_table_statistics_sync(team.id, schema.id)
assert result == {"status": "skipped", "reason": "no_files"}
+ @parameterized.expand([("at_the_cap", 2, "done"), ("over_the_cap", 3, "skipped")])
+ def test_add_action_scan_is_capped_by_files_times_columns(
+ self, _name: str, file_count: int, expected_status: str
+ ) -> None:
+ team = self._team()
+ schema, table, _ = self._schema_table_job(team, columns={"amount": "Int64", "currency": "String"})
+ helper = self._mock_delta(pa.table({"num_records": [1] * file_count}))
+ helper.get_delta_table.return_value.file_uris.return_value = [f"f{i}.parquet" for i in range(file_count)]
+ with patch(DELTA_HELPER_PATH, return_value=helper), patch.object(comp, "MAX_ADD_ACTION_CELLS", 4):
+ result = compute_table_statistics_sync(team.id, schema.id)
+
+ assert result["status"] == expected_status
+ if expected_status == "skipped":
+ assert result == {"status": "skipped", "reason": "too_many_files"}
+ helper.get_delta_table.return_value.get_add_actions.assert_not_called()
+ assert not WarehouseColumnStatistics.objects.for_team(team.id).filter(table_id=table.id).exists()
+
@pytest.mark.django_db(transaction=True)
class TestComputeTableStatisticsActivity: