Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): check every commit before folding table statistics (#120844)

Tom Owersb4456839261f
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py
index 4c1f9bd67146..daf743373921 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py
@@ -363,13 +363,17 @@ class _FoldState:
if isinstance(null_count, int) and not isinstance(null_count, bool):
self.null_count = (self.null_count or 0) + null_count
min_value = stats.get("minValues", {}).get(name)
- if min_value is not None:
- parsed = _parse_log_value(self.delta_type, min_value)
- self.min_value = parsed if self.min_value is None else min(self.min_value, parsed)
max_value = stats.get("maxValues", {}).get(name)
+ self.include(
+ _parse_log_value(self.delta_type, min_value) if min_value is not None else None,
+ _parse_log_value(self.delta_type, max_value) if max_value is not None else None,
+ )
+
+ def include(self, min_value: Any, max_value: Any) -> None:
+ if min_value is not None:
+ self.min_value = min_value if self.min_value is None else min(self.min_value, min_value)
if max_value is not None:
- parsed = _parse_log_value(self.delta_type, max_value)
- self.max_value = parsed if self.max_value is None else max(self.max_value, parsed)
+ self.max_value = max_value if self.max_value is None else max(self.max_value, max_value)
def _fold_commit_stats(
@@ -411,17 +415,16 @@ def _fold_commit_stats(
# scan re-derives them instead.
if not isinstance(delta_type, str) and (stored.min_value is not None or stored.max_value is not None):
return None
- states[name] = _FoldState(
- delta_type=delta_type,
- null_count=stored.null_count,
- min_value=_parse_stored_value(delta_type, stored.min_value) if stored.min_value is not None else None,
- max_value=_parse_stored_value(delta_type, stored.max_value) if stored.max_value is not None else None,
- )
+ states[name] = _FoldState(delta_type=delta_type, null_count=stored.null_count, min_value=None, max_value=None)
if len(row_counts) != 1:
return None
row_count = row_counts.pop()
- for version in range(base_version + 1, delta_version + 1):
+ # Stats and stored bounds parse under the current schema, but an older file or the stored row
+ # carries its bounds in the type of its own time. Newest first, a commit that changed the schema
+ # stops the fold before anything older is parsed; every aggregate here ignores order.
+ for version in range(delta_version, base_version, -1):
+ added_stats: list[str] = []
for action in read_commit_actions(version):
kinds = set(action) - _NEUTRAL_ACTIONS
if not kinds:
@@ -431,7 +434,9 @@ def _fold_commit_stats(
add = action["add"]
if not add.get("dataChange", True) or not add.get("stats"):
return None
- stats = json.loads(add["stats"], parse_float=Decimal)
+ added_stats.append(add["stats"])
+ for raw_stats in added_stats:
+ stats = json.loads(raw_stats, parse_float=Decimal)
num_records = stats.get("numRecords")
if not isinstance(num_records, int) or isinstance(num_records, bool):
return None
@@ -439,6 +444,13 @@ def _fold_commit_stats(
for name, state in states.items():
state.fold(stats, name)
+ for name, state in states.items():
+ stored = existing[name]
+ state.include(
+ _parse_stored_value(state.delta_type, stored.min_value) if stored.min_value is not None else None,
+ _parse_stored_value(state.delta_type, stored.max_value) if stored.max_value is not None else None,
+ )
+
return row_count, {
name: _ColumnStat(
column_type=clean_type(_column_type(columns[name])) or "unknown",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py
index 423bd3d3df83..9b7c5fec1cd7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py
@@ -565,6 +565,8 @@ class TestComputeTableStatisticsSync:
[
("a_file_was_removed", "remove", 8, dt.timedelta(days=2), {"amount": "long"}, True),
("the_schema_changed", "metadata", 8, dt.timedelta(days=2), {"amount": "long"}, True),
+ ("a_later_commit_made_it_a_string", "to_string", 9, dt.timedelta(days=2), {"amount": "string"}, True),
+ ("a_later_commit_made_it_a_long", "to_long", 9, dt.timedelta(days=2), {"amount": "long"}, True),
("a_file_carries_no_stats", "no_stats", 8, dt.timedelta(days=2), {"amount": "long"}, True),
("a_column_was_added", "append", 8, dt.timedelta(days=2), {"amount": "long", "total": "long"}, False),
(
@@ -591,9 +593,19 @@ class TestComputeTableStatisticsSync:
schema, table, _ = self._schema_table_job(
team, columns={name: {"clickhouse": "Nullable(Int64)"} for name in columns}
)
- self._stored(team, table, age=age)
+ if commit_kind == "to_long":
+ self._stored(team, table, age=age, min_value="apple", max_value="pear")
+ else:
+ self._stored(team, table, age=age)
commit: list[dict[str, Any]]
- if commit_kind == "remove":
+ commits: dict[int, list[dict[str, Any]]] = {}
+ if commit_kind == "to_string":
+ commit = [{"metaData": {"schemaString": "{}"}}, self._add(1, amount=("a", "z", 0))]
+ commits[8] = [self._add(10, amount=(1, 60, 0))]
+ elif commit_kind == "to_long":
+ commit = [{"metaData": {"schemaString": "{}"}}, self._add(1, amount=(1, 60, 0))]
+ commits[8] = [self._add(10, amount=("banana", "plum", 0))]
+ elif commit_kind == "remove":
commit = [self._add(1, amount=(1, 1, 0)), {"remove": {"path": "old.parquet", "dataChange": True}}]
elif commit_kind == "metadata":
commit = [{"metaData": {"schemaString": "{}"}}, self._add(1, amount=(1, 1, 0))]
@@ -611,7 +623,9 @@ class TestComputeTableStatisticsSync:
)
helper = self._mock_delta(add_actions, version=table_version, delta_types=columns)
with (
- patch.object(comp, "_read_commit_actions", return_value=commit) as mock_read,
+ patch.object(
+ comp, "_read_commit_actions", side_effect=lambda _uri, _options, version: commits.get(version, commit)
+ ) as mock_read,
patch.object(comp, "capture_exception") as mock_capture,
patch(DELTA_HELPER_PATH, return_value=helper),
):
@@ -627,17 +641,17 @@ class TestComputeTableStatisticsSync:
assert stat.computed_for_delta_version == table_version
def test_an_unparseable_stored_value_falls_back_without_leaking_it(self) -> None:
- # A stored bound written under an earlier column type (say string) cannot be parsed under a
- # changed type (say long): `int("not-a-number")` raises `ValueError` with the raw text in its
- # message. The fold must fall back to a full scan, and neither the capture nor the log line it
- # reports on may repeat that text.
+ # A stored bound that does not parse under the column's type, with no schema change in the
+ # commit range to explain it: `int("not-a-number")` raises `ValueError` with the raw text in
+ # its message. The fold must fall back to a full scan, and neither the capture nor the log
+ # line it reports on may repeat that text.
team = self._team()
schema, table, _ = self._schema_table_job(team)
self._stored(team, table, min_value="not-a-number", max_value="not-a-number-either")
add_actions = pa.table({"num_records": [99], "null_count.amount": [0], "min.amount": [1], "max.amount": [1]})
helper = self._mock_delta(add_actions, version=8, delta_types={"amount": "long"})
with (
- patch.object(comp, "_read_commit_actions", return_value=[self._add(1, amount=(1, 1, 0))]) as mock_read,
+ patch.object(comp, "_read_commit_actions", return_value=[self._add(1, amount=(1, 1, 0))]),
patch.object(comp, "capture_exception") as mock_capture,
patch(DELTA_HELPER_PATH, return_value=helper),
):
@@ -645,7 +659,6 @@ class TestComputeTableStatisticsSync:
assert result["status"] == "done"
assert result["basis"] == "full"
- assert not mock_read.called
mock_capture.assert_called_once()
reported = str(mock_capture.call_args[0][0])
assert "not-a-number" not in reported