:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-modeling): verify incremental unique keys by join, raise cap to 1 GiB (#123697)
Jovan Sakovicc3dd04873ff0
diff --git a/posthog/temporal/data_modeling/activities/incremental_write.py b/posthog/temporal/data_modeling/activities/incremental_write.pyindex f66d03826783..1aab64af0f05 100644--- a/posthog/temporal/data_modeling/activities/incremental_write.py+++ b/posthog/temporal/data_modeling/activities/incremental_write.py@@ -29,8 +29,13 @@ import pyarrow.compute as pc# Exact duplicate detection has to remember every distinct key a run has produced. The cap bounds# that state so a huge materialization cannot OOM the worker; hitting it fails the run loudly-# rather than quietly skipping verification.-MAX_UNIQUE_KEY_CHECK_BYTES = 256 * 1024 * 1024+# rather than quietly skipping verification. Peak memory of the check is about this state plus one+# batch, and a worker runs many activities at once, so the cap has to stay a small share of a pod.+MAX_UNIQUE_KEY_CHECK_BYTES = 1024 * 1024 * 1024++# A materialization batch is about 100 MB, so a narrow key-only table sends ~12M rows at once, and the+# group-by hash table for that many rows dwarfs the stored keys. Checking in slices keeps that bounded.+UNIQUE_KEY_CHECK_SLICE_ROWS = 1_000_000class IncrementalWriteError(Exception):@@ -103,12 +108,19 @@ class UniqueKeyTracker:it into an update, silently — so the distinct keys seen so far are carried across ``check``calls. Only the key columns of the group_by output are retained, so memory grows with distinctkeys, not rows, and the byte cap turns "too big to verify" into a loud failure.++ Each call groups only the new batch, then probes the stored keys with a hash join whose build+ side is that small grouped batch. The stored keys are kept as a list of per-batch chunks and are+ never merged into one buffer or inserted into a hash table again. Each call only looks them up in+ the batch's small table, which is much cheaper than regrouping everything seen so far."""def __init__(self, unique_key: tuple[str, ...], max_bytes: int = MAX_UNIQUE_KEY_CHECK_BYTES) -> None:self._columns = list(unique_key)self._max_bytes = max_bytes- self._seen: pa.Table | None = None+ self._seen_chunks: list[pa.Table] = []+ self._seen_bytes = 0+ self._seen_rows = 0def check(self, batch: pa.RecordBatch) -> None:"""Fail if ``batch`` has a missing, null, or duplicate key — including a key already seen@@ -121,25 +133,36 @@ class UniqueKeyTracker:f"Update the incremental configuration to match the query's columns.")assert_no_null_keys(batch, tuple(self._columns))+ for offset in range(0, max(batch.num_rows, 1), UNIQUE_KEY_CHECK_SLICE_ROWS):+ self._check_slice(batch.slice(offset, UNIQUE_KEY_CHECK_SLICE_ROWS))+ def _check_slice(self, batch: pa.RecordBatch) -> None:keys = pa.Table.from_batches([batch]).select(self._columns)- combined = keys if self._seen is None else pa.concat_tables([self._seen, keys])- grouped = combined.group_by(self._columns).aggregate([([], "count_all")])- if grouped.num_rows != combined.num_rows:- worst = pc.max(grouped.column("count_all")).as_py()+ grouped = keys.group_by(self._columns).aggregate([([], "count_all")])+ worst = pc.max(grouped.column("count_all")).as_py() if grouped.num_rows != keys.num_rows else 1+ if self._seen_chunks:+ # Acero builds the hash table on the right input, so the grouped batch goes on the right+ # and the stored keys are only streamed through it. Every stored key is distinct, so a+ # key found there occurs once more than its count in this batch.+ already_seen = pa.concat_tables(self._seen_chunks).join(grouped, keys=self._columns, join_type="right semi")+ if already_seen.num_rows:+ worst = max(worst, pc.max(already_seen.column("count_all")).as_py() + 1)+ if worst > 1:raise IncrementalWriteError(f"The unique key ({', '.join(self._columns)}) does not identify a single row: one key "f"appears {worst} times in this run's result. Add the columns that tell those rows apart.")- seen = grouped.select(self._columns)- if seen.nbytes > self._max_bytes:+ distinct = grouped.select(self._columns).combine_chunks()+ self._seen_bytes += distinct.nbytes+ self._seen_rows += distinct.num_rows+ if self._seen_bytes > self._max_bytes:raise IncrementalWriteError(- f"This query returns too many unique keys to verify ({seen.num_rows} so far). "+ f"This query returns too many unique keys to verify ({self._seen_rows} so far). "f"Reduce the rows the query returns, or turn off incremental materialization for "f"this view to fall back to a full refresh.")- self._seen = seen+ self._seen_chunks.append(distinct)def max_of(batch: pa.RecordBatch, column: str, running: Any = None) -> Any:diff --git a/posthog/temporal/tests/data_modeling/test_incremental_write.py b/posthog/temporal/tests/data_modeling/test_incremental_write.pyindex 676f9910adfc..8a7759ea0640 100644--- a/posthog/temporal/tests/data_modeling/test_incremental_write.py+++ b/posthog/temporal/tests/data_modeling/test_incremental_write.py@@ -10,9 +10,12 @@ from unittest.mock import MagicMock, patchfrom django.test import override_settings+import numpy as npimport pyarrow as pa+import pyarrow.compute as pcfrom posthog.temporal.data_modeling.activities.incremental_write import (+ MAX_UNIQUE_KEY_CHECK_BYTES,IncrementalWriteError,UniqueKeyTracker,upsert_batch,@@ -29,12 +32,27 @@ def _batch(ids: list[int | None], names: list[str | None]) -> pa.RecordBatch:@pytest.mark.parametrize("batches,match",[- pytest.param([([1, 1], ["a", "b"])], "does not identify a single row", id="duplicate_within_a_batch"),+ pytest.param([([1, 1], ["a", "b"])], "appears 2 times", id="duplicate_within_a_batch"),pytest.param([([1], ["a"]), ([2], ["b"]), ([1], ["c"])],- "does not identify a single row",+ "appears 2 times",id="duplicate_across_batches",),+ pytest.param(+ [([1], ["a"]), ([1, 1, 2], ["b", "c", "d"])],+ "appears 3 times",+ id="seen_key_repeated_within_a_later_batch",+ ),+ pytest.param(+ [([1], ["a"]), ([1] * 5, ["b"] * 5)],+ "appears 6 times",+ id="seen_key_five_times_in_a_later_batch",+ ),+ pytest.param(+ [([9], ["a"]), ([9] + [5] * 5, ["b"] * 6)],+ "appears 5 times",+ id="within_batch_count_outweighs_a_seen_key",+ ),pytest.param([([1, None], ["a", "b"])], "is null", id="null_key"),],)@@ -46,6 +64,14 @@ def test_single_column_violations_are_detected(batches: list[tuple], match: str)tracker.check(_batch(ids, names))+def test_a_duplicate_split_across_slices_of_one_batch_is_detected() -> None:+ tracker = UniqueKeyTracker(("id",))++ with patch("posthog.temporal.data_modeling.activities.incremental_write.UNIQUE_KEY_CHECK_SLICE_ROWS", 2):+ with pytest.raises(IncrementalWriteError, match="appears 2 times"):+ tracker.check(_batch([1, 2, 3, 1], ["a", "b", "c", "d"]))++def test_distinct_keys_across_batches_pass() -> None:tracker = UniqueKeyTracker(("id",))@@ -73,10 +99,19 @@ def test_a_missing_key_column_names_the_column() -> None:def test_exceeding_the_memory_cap_fails_loudly() -> None:"""The cap must fail the run, never silently stop checking."""- tracker = UniqueKeyTracker(("id",), max_bytes=8)+ tracker = UniqueKeyTracker(("id",), max_bytes=24)+ tracker.check(_batch([1, 2], ["a", "b"]))++ with pytest.raises(IncrementalWriteError, match=r"too many unique keys to verify \(4 so far\)"):+ tracker.check(_batch([3, 4], ["c", "d"]))+++def test_the_cap_admits_a_seven_million_key_uuid_run() -> None:+ sample = 10_000+ ids = pa.array(np.arange(sample, dtype=np.int64))+ uuid_sized = pa.table({"id": pc.utf8_lpad(pc.cast(ids, pa.string()), width=36, padding="0")})- with pytest.raises(IncrementalWriteError, match="too many unique keys"):- tracker.check(_batch([1, 2, 3, 4], ["a", "b", "c", "d"]))+ assert uuid_sized.nbytes * (7_000_000 // sample) <= MAX_UNIQUE_KEY_CHECK_BYTES@pytest.mark.parametrize(