Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-modeling): verify incremental unique keys by join, raise cap to 1 GiB (#123697)

Jovan Sakovicc3dd04873ff0
diff --git a/posthog/temporal/data_modeling/activities/incremental_write.py b/posthog/temporal/data_modeling/activities/incremental_write.py
index f66d03826783..1aab64af0f05 100644
--- a/posthog/temporal/data_modeling/activities/incremental_write.py
+++ b/posthog/temporal/data_modeling/activities/incremental_write.py
@@ -29,8 +29,13 @@ import pyarrow.compute as pc
# Exact duplicate detection has to remember every distinct key a run has produced. The cap bounds
# that state so a huge materialization cannot OOM the worker; hitting it fails the run loudly
-# rather than quietly skipping verification.
-MAX_UNIQUE_KEY_CHECK_BYTES = 256 * 1024 * 1024
+# rather than quietly skipping verification. Peak memory of the check is about this state plus one
+# batch, and a worker runs many activities at once, so the cap has to stay a small share of a pod.
+MAX_UNIQUE_KEY_CHECK_BYTES = 1024 * 1024 * 1024
+
+# A materialization batch is about 100 MB, so a narrow key-only table sends ~12M rows at once, and the
+# group-by hash table for that many rows dwarfs the stored keys. Checking in slices keeps that bounded.
+UNIQUE_KEY_CHECK_SLICE_ROWS = 1_000_000
class IncrementalWriteError(Exception):
@@ -103,12 +108,19 @@ class UniqueKeyTracker:
it into an update, silently — so the distinct keys seen so far are carried across ``check``
calls. Only the key columns of the group_by output are retained, so memory grows with distinct
keys, not rows, and the byte cap turns "too big to verify" into a loud failure.
+
+ Each call groups only the new batch, then probes the stored keys with a hash join whose build
+ side is that small grouped batch. The stored keys are kept as a list of per-batch chunks and are
+ never merged into one buffer or inserted into a hash table again. Each call only looks them up in
+ the batch's small table, which is much cheaper than regrouping everything seen so far.
"""
def __init__(self, unique_key: tuple[str, ...], max_bytes: int = MAX_UNIQUE_KEY_CHECK_BYTES) -> None:
self._columns = list(unique_key)
self._max_bytes = max_bytes
- self._seen: pa.Table | None = None
+ self._seen_chunks: list[pa.Table] = []
+ self._seen_bytes = 0
+ self._seen_rows = 0
def check(self, batch: pa.RecordBatch) -> None:
"""Fail if ``batch`` has a missing, null, or duplicate key — including a key already seen
@@ -121,25 +133,36 @@ class UniqueKeyTracker:
f"Update the incremental configuration to match the query's columns."
)
assert_no_null_keys(batch, tuple(self._columns))
+ for offset in range(0, max(batch.num_rows, 1), UNIQUE_KEY_CHECK_SLICE_ROWS):
+ self._check_slice(batch.slice(offset, UNIQUE_KEY_CHECK_SLICE_ROWS))
+ def _check_slice(self, batch: pa.RecordBatch) -> None:
keys = pa.Table.from_batches([batch]).select(self._columns)
- combined = keys if self._seen is None else pa.concat_tables([self._seen, keys])
- grouped = combined.group_by(self._columns).aggregate([([], "count_all")])
- if grouped.num_rows != combined.num_rows:
- worst = pc.max(grouped.column("count_all")).as_py()
+ grouped = keys.group_by(self._columns).aggregate([([], "count_all")])
+ worst = pc.max(grouped.column("count_all")).as_py() if grouped.num_rows != keys.num_rows else 1
+ if self._seen_chunks:
+ # Acero builds the hash table on the right input, so the grouped batch goes on the right
+ # and the stored keys are only streamed through it. Every stored key is distinct, so a
+ # key found there occurs once more than its count in this batch.
+ already_seen = pa.concat_tables(self._seen_chunks).join(grouped, keys=self._columns, join_type="right semi")
+ if already_seen.num_rows:
+ worst = max(worst, pc.max(already_seen.column("count_all")).as_py() + 1)
+ if worst > 1:
raise IncrementalWriteError(
f"The unique key ({', '.join(self._columns)}) does not identify a single row: one key "
f"appears {worst} times in this run's result. Add the columns that tell those rows apart."
)
- seen = grouped.select(self._columns)
- if seen.nbytes > self._max_bytes:
+ distinct = grouped.select(self._columns).combine_chunks()
+ self._seen_bytes += distinct.nbytes
+ self._seen_rows += distinct.num_rows
+ if self._seen_bytes > self._max_bytes:
raise IncrementalWriteError(
- f"This query returns too many unique keys to verify ({seen.num_rows} so far). "
+ f"This query returns too many unique keys to verify ({self._seen_rows} so far). "
f"Reduce the rows the query returns, or turn off incremental materialization for "
f"this view to fall back to a full refresh."
)
- self._seen = seen
+ self._seen_chunks.append(distinct)
def max_of(batch: pa.RecordBatch, column: str, running: Any = None) -> Any:
diff --git a/posthog/temporal/tests/data_modeling/test_incremental_write.py b/posthog/temporal/tests/data_modeling/test_incremental_write.py
index 676f9910adfc..8a7759ea0640 100644
--- a/posthog/temporal/tests/data_modeling/test_incremental_write.py
+++ b/posthog/temporal/tests/data_modeling/test_incremental_write.py
@@ -10,9 +10,12 @@ from unittest.mock import MagicMock, patch
from django.test import override_settings
+import numpy as np
import pyarrow as pa
+import pyarrow.compute as pc
from posthog.temporal.data_modeling.activities.incremental_write import (
+ MAX_UNIQUE_KEY_CHECK_BYTES,
IncrementalWriteError,
UniqueKeyTracker,
upsert_batch,
@@ -29,12 +32,27 @@ def _batch(ids: list[int | None], names: list[str | None]) -> pa.RecordBatch:
@pytest.mark.parametrize(
"batches,match",
[
- pytest.param([([1, 1], ["a", "b"])], "does not identify a single row", id="duplicate_within_a_batch"),
+ pytest.param([([1, 1], ["a", "b"])], "appears 2 times", id="duplicate_within_a_batch"),
pytest.param(
[([1], ["a"]), ([2], ["b"]), ([1], ["c"])],
- "does not identify a single row",
+ "appears 2 times",
id="duplicate_across_batches",
),
+ pytest.param(
+ [([1], ["a"]), ([1, 1, 2], ["b", "c", "d"])],
+ "appears 3 times",
+ id="seen_key_repeated_within_a_later_batch",
+ ),
+ pytest.param(
+ [([1], ["a"]), ([1] * 5, ["b"] * 5)],
+ "appears 6 times",
+ id="seen_key_five_times_in_a_later_batch",
+ ),
+ pytest.param(
+ [([9], ["a"]), ([9] + [5] * 5, ["b"] * 6)],
+ "appears 5 times",
+ id="within_batch_count_outweighs_a_seen_key",
+ ),
pytest.param([([1, None], ["a", "b"])], "is null", id="null_key"),
],
)
@@ -46,6 +64,14 @@ def test_single_column_violations_are_detected(batches: list[tuple], match: str)
tracker.check(_batch(ids, names))
+def test_a_duplicate_split_across_slices_of_one_batch_is_detected() -> None:
+ tracker = UniqueKeyTracker(("id",))
+
+ with patch("posthog.temporal.data_modeling.activities.incremental_write.UNIQUE_KEY_CHECK_SLICE_ROWS", 2):
+ with pytest.raises(IncrementalWriteError, match="appears 2 times"):
+ tracker.check(_batch([1, 2, 3, 1], ["a", "b", "c", "d"]))
+
+
def test_distinct_keys_across_batches_pass() -> None:
tracker = UniqueKeyTracker(("id",))
@@ -73,10 +99,19 @@ def test_a_missing_key_column_names_the_column() -> None:
def test_exceeding_the_memory_cap_fails_loudly() -> None:
"""The cap must fail the run, never silently stop checking."""
- tracker = UniqueKeyTracker(("id",), max_bytes=8)
+ tracker = UniqueKeyTracker(("id",), max_bytes=24)
+ tracker.check(_batch([1, 2], ["a", "b"]))
+
+ with pytest.raises(IncrementalWriteError, match=r"too many unique keys to verify \(4 so far\)"):
+ tracker.check(_batch([3, 4], ["c", "d"]))
+
+
+def test_the_cap_admits_a_seven_million_key_uuid_run() -> None:
+ sample = 10_000
+ ids = pa.array(np.arange(sample, dtype=np.int64))
+ uuid_sized = pa.table({"id": pc.utf8_lpad(pc.cast(ids, pa.string()), width=36, padding="0")})
- with pytest.raises(IncrementalWriteError, match="too many unique keys"):
- tracker.check(_batch([1, 2, 3, 4], ["a", "b", "c", "d"]))
+ assert uuid_sized.nbytes * (7_000_000 // sample) <= MAX_UNIQUE_KEY_CHECK_BYTES
@pytest.mark.parametrize(