:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): check every commit before folding table statistics (#120844)
Tom Owersb4456839261f
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.pyindex 4c1f9bd67146..daf743373921 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py@@ -363,13 +363,17 @@ class _FoldState:if isinstance(null_count, int) and not isinstance(null_count, bool):self.null_count = (self.null_count or 0) + null_countmin_value = stats.get("minValues", {}).get(name)- if min_value is not None:- parsed = _parse_log_value(self.delta_type, min_value)- self.min_value = parsed if self.min_value is None else min(self.min_value, parsed)max_value = stats.get("maxValues", {}).get(name)+ self.include(+ _parse_log_value(self.delta_type, min_value) if min_value is not None else None,+ _parse_log_value(self.delta_type, max_value) if max_value is not None else None,+ )++ def include(self, min_value: Any, max_value: Any) -> None:+ if min_value is not None:+ self.min_value = min_value if self.min_value is None else min(self.min_value, min_value)if max_value is not None:- parsed = _parse_log_value(self.delta_type, max_value)- self.max_value = parsed if self.max_value is None else max(self.max_value, parsed)+ self.max_value = max_value if self.max_value is None else max(self.max_value, max_value)def _fold_commit_stats(@@ -411,17 +415,16 @@ def _fold_commit_stats(# scan re-derives them instead.if not isinstance(delta_type, str) and (stored.min_value is not None or stored.max_value is not None):return None- states[name] = _FoldState(- delta_type=delta_type,- null_count=stored.null_count,- min_value=_parse_stored_value(delta_type, stored.min_value) if stored.min_value is not None else None,- max_value=_parse_stored_value(delta_type, stored.max_value) if stored.max_value is not None else None,- )+ states[name] = _FoldState(delta_type=delta_type, null_count=stored.null_count, min_value=None, max_value=None)if len(row_counts) != 1:return Nonerow_count = row_counts.pop()- for version in range(base_version + 1, delta_version + 1):+ # Stats and stored bounds parse under the current schema, but an older file or the stored row+ # carries its bounds in the type of its own time. Newest first, a commit that changed the schema+ # stops the fold before anything older is parsed; every aggregate here ignores order.+ for version in range(delta_version, base_version, -1):+ added_stats: list[str] = []for action in read_commit_actions(version):kinds = set(action) - _NEUTRAL_ACTIONSif not kinds:@@ -431,7 +434,9 @@ def _fold_commit_stats(add = action["add"]if not add.get("dataChange", True) or not add.get("stats"):return None- stats = json.loads(add["stats"], parse_float=Decimal)+ added_stats.append(add["stats"])+ for raw_stats in added_stats:+ stats = json.loads(raw_stats, parse_float=Decimal)num_records = stats.get("numRecords")if not isinstance(num_records, int) or isinstance(num_records, bool):return None@@ -439,6 +444,13 @@ def _fold_commit_stats(for name, state in states.items():state.fold(stats, name)+ for name, state in states.items():+ stored = existing[name]+ state.include(+ _parse_stored_value(state.delta_type, stored.min_value) if stored.min_value is not None else None,+ _parse_stored_value(state.delta_type, stored.max_value) if stored.max_value is not None else None,+ )+return row_count, {name: _ColumnStat(column_type=clean_type(_column_type(columns[name])) or "unknown",diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.pyindex 423bd3d3df83..9b7c5fec1cd7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py@@ -565,6 +565,8 @@ class TestComputeTableStatisticsSync:[("a_file_was_removed", "remove", 8, dt.timedelta(days=2), {"amount": "long"}, True),("the_schema_changed", "metadata", 8, dt.timedelta(days=2), {"amount": "long"}, True),+ ("a_later_commit_made_it_a_string", "to_string", 9, dt.timedelta(days=2), {"amount": "string"}, True),+ ("a_later_commit_made_it_a_long", "to_long", 9, dt.timedelta(days=2), {"amount": "long"}, True),("a_file_carries_no_stats", "no_stats", 8, dt.timedelta(days=2), {"amount": "long"}, True),("a_column_was_added", "append", 8, dt.timedelta(days=2), {"amount": "long", "total": "long"}, False),(@@ -591,9 +593,19 @@ class TestComputeTableStatisticsSync:schema, table, _ = self._schema_table_job(team, columns={name: {"clickhouse": "Nullable(Int64)"} for name in columns})- self._stored(team, table, age=age)+ if commit_kind == "to_long":+ self._stored(team, table, age=age, min_value="apple", max_value="pear")+ else:+ self._stored(team, table, age=age)commit: list[dict[str, Any]]- if commit_kind == "remove":+ commits: dict[int, list[dict[str, Any]]] = {}+ if commit_kind == "to_string":+ commit = [{"metaData": {"schemaString": "{}"}}, self._add(1, amount=("a", "z", 0))]+ commits[8] = [self._add(10, amount=(1, 60, 0))]+ elif commit_kind == "to_long":+ commit = [{"metaData": {"schemaString": "{}"}}, self._add(1, amount=(1, 60, 0))]+ commits[8] = [self._add(10, amount=("banana", "plum", 0))]+ elif commit_kind == "remove":commit = [self._add(1, amount=(1, 1, 0)), {"remove": {"path": "old.parquet", "dataChange": True}}]elif commit_kind == "metadata":commit = [{"metaData": {"schemaString": "{}"}}, self._add(1, amount=(1, 1, 0))]@@ -611,7 +623,9 @@ class TestComputeTableStatisticsSync:)helper = self._mock_delta(add_actions, version=table_version, delta_types=columns)with (- patch.object(comp, "_read_commit_actions", return_value=commit) as mock_read,+ patch.object(+ comp, "_read_commit_actions", side_effect=lambda _uri, _options, version: commits.get(version, commit)+ ) as mock_read,patch.object(comp, "capture_exception") as mock_capture,patch(DELTA_HELPER_PATH, return_value=helper),):@@ -627,17 +641,17 @@ class TestComputeTableStatisticsSync:assert stat.computed_for_delta_version == table_versiondef test_an_unparseable_stored_value_falls_back_without_leaking_it(self) -> None:- # A stored bound written under an earlier column type (say string) cannot be parsed under a- # changed type (say long): `int("not-a-number")` raises `ValueError` with the raw text in its- # message. The fold must fall back to a full scan, and neither the capture nor the log line it- # reports on may repeat that text.+ # A stored bound that does not parse under the column's type, with no schema change in the+ # commit range to explain it: `int("not-a-number")` raises `ValueError` with the raw text in+ # its message. The fold must fall back to a full scan, and neither the capture nor the log+ # line it reports on may repeat that text.team = self._team()schema, table, _ = self._schema_table_job(team)self._stored(team, table, min_value="not-a-number", max_value="not-a-number-either")add_actions = pa.table({"num_records": [99], "null_count.amount": [0], "min.amount": [1], "max.amount": [1]})helper = self._mock_delta(add_actions, version=8, delta_types={"amount": "long"})with (- patch.object(comp, "_read_commit_actions", return_value=[self._add(1, amount=(1, 1, 0))]) as mock_read,+ patch.object(comp, "_read_commit_actions", return_value=[self._add(1, amount=(1, 1, 0))]),patch.object(comp, "capture_exception") as mock_capture,patch(DELTA_HELPER_PATH, return_value=helper),):@@ -645,7 +659,6 @@ class TestComputeTableStatisticsSync:assert result["status"] == "done"assert result["basis"] == "full"- assert not mock_read.calledmock_capture.assert_called_once()reported = str(mock_capture.call_args[0][0])assert "not-a-number" not in reported