:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-modeling): spill unique key checks to disk past a memory budget (#127415)
Raúl Negrón-Oteroee12cb3a85b5
diff --git a/posthog/temporal/data_modeling/activities/incremental_write.py b/posthog/temporal/data_modeling/activities/incremental_write.pyindex 1aab64af0f05..44bbc0c79e21 100644--- a/posthog/temporal/data_modeling/activities/incremental_write.py+++ b/posthog/temporal/data_modeling/activities/incremental_write.py@@ -20,6 +20,14 @@ column rather than as a Rust error or as quiet drift. The fourth is why the fullstays on delta-rs."""+import json+import shutil+import struct+import sqlite3+import threading+from collections.abc import Iterator+from pathlib import Path+from tempfile import TemporaryDirectoryfrom typing import Any, Optionalfrom django.conf import settings@@ -27,12 +35,19 @@ from django.conf import settingsimport pyarrow as paimport pyarrow.compute as pc-# Exact duplicate detection has to remember every distinct key a run has produced. The cap bounds-# that state so a huge materialization cannot OOM the worker; hitting it fails the run loudly-# rather than quietly skipping verification. Peak memory of the check is about this state plus one-# batch, and a worker runs many activities at once, so the cap has to stay a small share of a pod.+# Exact duplicate detection has to remember every distinct key a run has produced. Past this budget+# the keys move to a local SQLite index, so a huge materialization cannot OOM the worker. Peak memory+# of the in-memory check is about this state plus one slice, and a worker runs many activities at+# once, so the budget has to stay a small share of a pod.MAX_UNIQUE_KEY_CHECK_BYTES = 1024 * 1024 * 1024+# The spilled index shares the worker's temporary disk with every other activity on it. Past this+# size, or once that disk has less than the reserve free, the check fails the run loudly rather than+# filling the disk under its neighbors.+MAX_UNIQUE_KEY_SPILL_BYTES = 4 * 1024 * 1024 * 1024+MIN_FREE_SPILL_DISK_BYTES = 2 * 1024 * 1024 * 1024+_SPILL_FILE = "keys.sqlite3"+# A materialization batch is about 100 MB, so a narrow key-only table sends ~12M rows at once, and the# group-by hash table for that many rows dwarfs the stored keys. Checking in slices keeps that bounded.UNIQUE_KEY_CHECK_SLICE_ROWS = 1_000_000@@ -42,6 +57,10 @@ class IncrementalWriteError(Exception):"""A batch cannot be safely upserted. Carries a message meant for the run's error field."""+class _TrackerClosed(Exception):+ """The run closed the tracker while a check was still running on its thread."""++class SchemaDriftError(IncrementalWriteError):"""The query's output no longer matches the stored table.@@ -98,6 +117,12 @@ def assert_no_null_keys(batch: pa.RecordBatch, unique_key: tuple[str, ...]) -> N)+def _spill_key(row: tuple[Any, ...]) -> str:+ # Arrow compares float keys by their bits: NaN payloads stay apart, and so do 0.0 and -0.0.+ values = [struct.pack(">d", value).hex() if isinstance(value, float) else value for value in row]+ return json.dumps(values, default=str, ensure_ascii=False, separators=(",", ":"))++class UniqueKeyTracker:"""Exact unique-key enforcement across every batch of one materialization.@@ -106,13 +131,18 @@ class UniqueKeyTracker:twice, whereas here a duplicate means the config is wrong and quietly keeping one row wouldhide it. A per-batch check cannot see a duplicate split across batches — deltalite would turnit into an update, silently — so the distinct keys seen so far are carried across ``check``- calls. Only the key columns of the group_by output are retained, so memory grows with distinct- keys, not rows, and the byte cap turns "too big to verify" into a loud failure.-- Each call groups only the new batch, then probes the stored keys with a hash join whose build- side is that small grouped batch. The stored keys are kept as a list of per-batch chunks and are- never merged into one buffer or inserted into a hash table again. Each call only looks them up in- the batch's small table, which is much cheaper than regrouping everything seen so far.+ calls. Only key columns are retained, and a local SQLite index takes over when they exceed the+ memory budget. The index has its own disk budget, and hitting it fails the run loudly rather+ than quietly skipping verification.++ The caller must ``close`` the tracker when the run ends, including on cancellation. A check runs+ on an executor thread that cancelling the caller does not stop, so ``close`` also tells a running+ check to stop at its next chunk.++ In memory, each call groups only the new batch, then probes the stored keys with a hash join whose+ build side is that small grouped batch. The stored keys are kept as a list of per-batch chunks and+ are never merged into one buffer or inserted into a hash table again. Each call only looks them up+ in the batch's small table, which is much cheaper than regrouping everything seen so far."""def __init__(self, unique_key: tuple[str, ...], max_bytes: int = MAX_UNIQUE_KEY_CHECK_BYTES) -> None:@@ -120,7 +150,87 @@ class UniqueKeyTracker:self._max_bytes = max_bytesself._seen_chunks: list[pa.Table] = []self._seen_bytes = 0- self._seen_rows = 0+ self._spill: sqlite3.Connection | None = None+ self._spill_directory: TemporaryDirectory[str] | None = None+ self._running = threading.Lock()+ self._closed = threading.Event()++ def close(self) -> None:+ """Remove the spilled index. Safe to call from any thread: a running check stops at its next+ chunk and removes the index itself."""+ self._closed.set()+ if not self._running.acquire(blocking=False):+ return+ try:+ if self._spill is not None:+ self._spill.close()+ if self._spill_directory is not None:+ self._spill_directory.cleanup()+ finally:+ self._running.release()++ def _raise_if_closed(self) -> None:+ if self._closed.is_set():+ raise _TrackerClosed()++ def _duplicate_key_error(self, occurrences: int) -> IncrementalWriteError:+ return IncrementalWriteError(+ f"The unique key ({', '.join(self._columns)}) does not identify a single row: one key "+ f"appears {occurrences} times in this run's result. Add the columns that tell those rows apart."+ )++ def _raise_if_over_disk_budget(self) -> None:+ assert self._spill_directory is not None+ directory = Path(self._spill_directory.name)+ if (directory / _SPILL_FILE).stat().st_size > MAX_UNIQUE_KEY_SPILL_BYTES or (+ shutil.disk_usage(directory).free < MIN_FREE_SPILL_DISK_BYTES+ ):+ raise IncrementalWriteError(+ "This query returns too many unique keys to verify. Reduce the rows the query returns, or "+ "turn off incremental materialization for this view to fall back to a full refresh."+ )++ def _spill_keys(self, keys: pa.Table) -> Iterator[tuple[str]]:+ for batch in keys.to_batches(max_chunksize=8192):+ self._raise_if_closed()+ self._raise_if_over_disk_budget()+ for row in zip(*batch.to_pydict().values(), strict=True):+ yield (_spill_key(row),)++ def _store(self, distinct_keys: pa.Table) -> None:+ assert self._spill is not None+ try:+ with self._spill:+ self._spill.executemany("INSERT INTO unique_keys (key) VALUES (?)", self._spill_keys(distinct_keys))+ except sqlite3.IntegrityError as error:+ # The keys are distinct, so a rejected one appears here once and was stored once before.+ raise self._duplicate_key_error(2) from error++ def _check_disk(self, keys: pa.Table) -> None:+ assert self._spill is not None+ grouped = keys.group_by(self._columns).aggregate([([], "count_all")])+ if grouped.num_rows == keys.num_rows:+ self._store(keys)+ return+ # The run fails either way. Only a key at the slice's top count can appear once more than+ # that, if an earlier slice stored it.+ top = pc.max(grouped.column("count_all")).as_py()+ candidates = grouped.filter(pc.equal(grouped.column("count_all"), top)).select(self._columns)+ lookup = "SELECT 1 FROM unique_keys WHERE key = ?"+ stored = any(self._spill.execute(lookup, key).fetchone() for key in self._spill_keys(candidates))+ raise self._duplicate_key_error(top + stored)++ def _start_spill(self) -> None:+ self._spill_directory = TemporaryDirectory(prefix="posthog-materialization-keys-")+ # Each check is awaited before the next starts, but asyncio.to_thread may choose another thread.+ self._spill = sqlite3.connect(Path(self._spill_directory.name) / _SPILL_FILE, check_same_thread=False)+ # The index is discarded with the run, so journaling and fsync buy nothing.+ self._spill.execute("PRAGMA journal_mode=OFF")+ self._spill.execute("PRAGMA synchronous=OFF")+ self._spill.execute("CREATE TABLE unique_keys (key TEXT PRIMARY KEY) WITHOUT ROWID")+ for chunk in self._seen_chunks:+ self._store(chunk)+ self._seen_chunks = []def check(self, batch: pa.RecordBatch) -> None:"""Fail if ``batch`` has a missing, null, or duplicate key — including a key already seen@@ -133,11 +243,20 @@ class UniqueKeyTracker:f"Update the incremental configuration to match the query's columns.")assert_no_null_keys(batch, tuple(self._columns))- for offset in range(0, max(batch.num_rows, 1), UNIQUE_KEY_CHECK_SLICE_ROWS):- self._check_slice(batch.slice(offset, UNIQUE_KEY_CHECK_SLICE_ROWS))+ try:+ with self._running:+ for offset in range(0, max(batch.num_rows, 1), UNIQUE_KEY_CHECK_SLICE_ROWS):+ self._raise_if_closed()+ self._check_slice(batch.slice(offset, UNIQUE_KEY_CHECK_SLICE_ROWS))+ finally:+ if self._closed.is_set():+ self.close()def _check_slice(self, batch: pa.RecordBatch) -> None:keys = pa.Table.from_batches([batch]).select(self._columns)+ if self._spill is not None:+ self._check_disk(keys)+ returngrouped = keys.group_by(self._columns).aggregate([([], "count_all")])worst = pc.max(grouped.column("count_all")).as_py() if grouped.num_rows != keys.num_rows else 1if self._seen_chunks:@@ -148,21 +267,13 @@ class UniqueKeyTracker:if already_seen.num_rows:worst = max(worst, pc.max(already_seen.column("count_all")).as_py() + 1)if worst > 1:- raise IncrementalWriteError(- f"The unique key ({', '.join(self._columns)}) does not identify a single row: one key "- f"appears {worst} times in this run's result. Add the columns that tell those rows apart."- )+ raise self._duplicate_key_error(worst)distinct = grouped.select(self._columns).combine_chunks()+ self._seen_chunks.append(distinct)self._seen_bytes += distinct.nbytes- self._seen_rows += distinct.num_rowsif self._seen_bytes > self._max_bytes:- raise IncrementalWriteError(- f"This query returns too many unique keys to verify ({self._seen_rows} so far). "- f"Reduce the rows the query returns, or turn off incremental materialization for "- f"this view to fall back to a full refresh."- )- self._seen_chunks.append(distinct)+ self._start_spill()def max_of(batch: pa.RecordBatch, column: str, running: Any = None) -> Any:diff --git a/posthog/temporal/data_modeling/activities/materialize_view.py b/posthog/temporal/data_modeling/activities/materialize_view.pyindex 9d458131c4c9..9365e3021140 100644--- a/posthog/temporal/data_modeling/activities/materialize_view.py+++ b/posthog/temporal/data_modeling/activities/materialize_view.py@@ -1061,43 +1061,47 @@ async def _materialize_fully(# brief to_thread released between batches, never pins a worker thread for the whole# read — which is what starved the shared executor that heartbeats/db/logging use.batch_index = 0- async for batch, ch_types in hogql_table(hogql_query, objects.team, logger):- batch = _transform_unsupported_decimals(batch)- batch = _transform_date_and_datetimes(batch, ch_types)- batch = _transform_unsigned_integers(batch)- batch = _force_nullable(batch)+ try:+ async for batch, ch_types in hogql_table(hogql_query, objects.team, logger):+ batch = _transform_unsupported_decimals(batch)+ batch = _transform_date_and_datetimes(batch, ch_types)+ batch = _transform_unsigned_integers(batch)+ batch = _force_nullable(batch)+ if tracker is not None:+ await asyncio.to_thread(tracker.check, batch)+ await _stage_person_property_batch(person_property_sink, batch_index, batch, fatal=False)+ batch_index += 1+ if delta_table is None:+ pa_schema = batch.schema+ await asyncio.to_thread(+ deltalake.write_deltalake,+ table_or_uri=table_uri,+ data=batch,+ mode="overwrite",+ schema_mode="overwrite",+ storage_options=storage_options,+ )+ delta_table = deltalake.DeltaTable(table_uri, storage_options=storage_options)+ else:+ await asyncio.to_thread(+ deltalake.write_deltalake,+ table_or_uri=delta_table,+ data=batch,+ mode="append",+ schema_mode="merge",+ storage_options=storage_options,+ )+ # Staged only once the batch is committed: a run that announces rows it then fails to write+ # cannot take the announcement back.+ await cdp_sink.stage(batch)+ if plan.config is not None and plan.config.incremental_key in batch.schema.names:+ watermark = max_of(batch, plan.config.incremental_key, watermark)+ row_count = row_count + batch.num_rows+ objects.job.rows_materialized = row_count+ await database_sync_to_async_pool(objects.job.save)()+ finally:if tracker is not None:- await asyncio.to_thread(tracker.check, batch)- await _stage_person_property_batch(person_property_sink, batch_index, batch, fatal=False)- batch_index += 1- if delta_table is None:- pa_schema = batch.schema- await asyncio.to_thread(- deltalake.write_deltalake,- table_or_uri=table_uri,- data=batch,- mode="overwrite",- schema_mode="overwrite",- storage_options=storage_options,- )- delta_table = deltalake.DeltaTable(table_uri, storage_options=storage_options)- else:- await asyncio.to_thread(- deltalake.write_deltalake,- table_or_uri=delta_table,- data=batch,- mode="append",- schema_mode="merge",- storage_options=storage_options,- )- # Staged only once the batch is committed: a run that announces rows it then fails to write- # cannot take the announcement back.- await cdp_sink.stage(batch)- if plan.config is not None and plan.config.incremental_key in batch.schema.names:- watermark = max_of(batch, plan.config.incremental_key, watermark)- row_count = row_count + batch.num_rows- objects.job.rows_materialized = row_count- await database_sync_to_async_pool(objects.job.save)()+ tracker.close()await logger.ainfo(f"Finished writing to delta table. row_count={row_count}")file_uris: list[str] = []@@ -1197,6 +1201,8 @@ async def _materialize_incrementally(if isinstance(err, SchemaDriftError):await logger.awarning(f"Rebuilding after schema drift: {err}")raise+ finally:+ tracker.close()if watermark is not None:await database_sync_to_async_pool(set_incremental_state)(diff --git a/posthog/temporal/tests/data_modeling/test_incremental_write.py b/posthog/temporal/tests/data_modeling/test_incremental_write.pyindex 8a7759ea0640..020d214c7461 100644--- a/posthog/temporal/tests/data_modeling/test_incremental_write.py+++ b/posthog/temporal/tests/data_modeling/test_incremental_write.py@@ -5,24 +5,40 @@ detection logic itself, where the cheap edge cases live (composite keys, the bytcolumn missing from the output)."""+import json+import struct+import tempfile+import contextlib+from collections.abc import Sequence+from pathlib import Path+from typing import Any+import pytestfrom unittest.mock import MagicMock, patchfrom django.test import override_settings-import numpy as npimport pyarrow as pa-import pyarrow.compute as pcfrom posthog.temporal.data_modeling.activities.incremental_write import (MAX_UNIQUE_KEY_CHECK_BYTES,+ MAX_UNIQUE_KEY_SPILL_BYTES,IncrementalWriteError,UniqueKeyTracker,+ _TrackerClosed,upsert_batch,)+_MODULE = "posthog.temporal.data_modeling.activities.incremental_write"++_spill_modes = pytest.mark.parametrize("max_bytes", [1, MAX_UNIQUE_KEY_CHECK_BYTES], ids=["disk", "memory"])+++def _float_from_bits(hex_bits: str) -> float:+ return struct.unpack(">d", bytes.fromhex(hex_bits))[0]-def _batch(ids: list[int | None], names: list[str | None]) -> pa.RecordBatch:++def _batch(ids: Sequence[int | None], names: Sequence[str | None]) -> pa.RecordBatch:return pa.RecordBatch.from_arrays([pa.array(ids, type=pa.int64()), pa.array(names, type=pa.string())],names=["id", "name"],@@ -53,11 +69,17 @@ def _batch(ids: list[int | None], names: list[str | None]) -> pa.RecordBatch:"appears 5 times",id="within_batch_count_outweighs_a_seen_key",),+ pytest.param(+ [([1, 2], ["a", "b"]), ([1, 2, 2, 2], ["c", "d", "e", "f"])],+ "appears 4 times",+ id="most_repeated_key_was_also_seen",+ ),pytest.param([([1, None], ["a", "b"])], "is null", id="null_key"),],)-def test_single_column_violations_are_detected(batches: list[tuple], match: str) -> None:- tracker = UniqueKeyTracker(("id",))+@_spill_modes+def test_single_column_violations_are_detected(batches: list[tuple], match: str, max_bytes: int) -> None:+ tracker = UniqueKeyTracker(("id",), max_bytes=max_bytes)with pytest.raises(IncrementalWriteError, match=match):for ids, names in batches:@@ -79,9 +101,9 @@ def test_distinct_keys_across_batches_pass() -> None:tracker.check(_batch([3, 4], ["c", "d"]))-def test_composite_key_is_checked_as_a_tuple() -> None:- """Columns may repeat individually; only the full tuple repeating is a violation."""- tracker = UniqueKeyTracker(("id", "name"))+@_spill_modes+def test_composite_key_is_checked_as_a_tuple(max_bytes: int) -> None:+ tracker = UniqueKeyTracker(("id", "name"), max_bytes=max_bytes)tracker.check(_batch([1, 1], ["a", "b"]))tracker.check(_batch([2, 2], ["a", "b"]))@@ -90,6 +112,23 @@ def test_composite_key_is_checked_as_a_tuple() -> None:tracker.check(_batch([1], ["a"]))+ "value,other,same",+ [+ (-0.0, 0.0, False),+ (float("nan"), float("nan"), True),+ (_float_from_bits("7ff8000000000000"), _float_from_bits("7ff8000000000001"), False),+ ],+)+@_spill_modes+def test_float_keys_compare_alike_in_memory_and_on_disk(value: float, other: float, same: bool, max_bytes: int) -> None:+ tracker = UniqueKeyTracker(("key",), max_bytes=max_bytes)+ tracker.check(pa.record_batch([[value]], names=["key"]))+ with pytest.raises(IncrementalWriteError, match="appears 2 times") if same else contextlib.nullcontext():+ tracker.check(pa.record_batch([[other]], names=["key"]))+ tracker.close()++def test_a_missing_key_column_names_the_column() -> None:tracker = UniqueKeyTracker(("missing",))@@ -97,21 +136,57 @@ def test_a_missing_key_column_names_the_column() -> None:tracker.check(_batch([1], ["a"]))-def test_exceeding_the_memory_cap_fails_loudly() -> None:- """The cap must fail the run, never silently stop checking."""- tracker = UniqueKeyTracker(("id",), max_bytes=24)- tracker.check(_batch([1, 2], ["a", "b"]))+def test_spilling_preserves_keys_from_earlier_batches() -> None:+ tracker = UniqueKeyTracker(("id",), max_bytes=8)++ tracker.check(_batch([1], ["a"]))+ tracker.check(_batch([2, 3, 4], ["b", "c", "d"]))+ with pytest.raises(IncrementalWriteError, match="does not identify a single row"):+ tracker.check(_batch([1], ["a"]))+ tracker.close()+++ "spill_bytes,free_reserve",+ [pytest.param(1, 0, id="index_budget"), pytest.param(MAX_UNIQUE_KEY_SPILL_BYTES, 1 << 62, id="free_disk")],+)+def test_the_spilled_index_fails_loudly_past_its_disk_budget(spill_bytes: int, free_reserve: int) -> None:+ tracker = UniqueKeyTracker(("id",), max_bytes=1)+ with (+ patch(f"{_MODULE}.MAX_UNIQUE_KEY_SPILL_BYTES", spill_bytes),+ patch(f"{_MODULE}.MIN_FREE_SPILL_DISK_BYTES", free_reserve),+ pytest.raises(IncrementalWriteError, match="too many unique keys to verify"),+ ):+ tracker.check(_batch([1, 2], ["a", "b"]))+ tracker.close()+++def test_closing_removes_the_spilled_index(tmp_path: Path) -> None:+ with patch.object(tempfile, "tempdir", str(tmp_path)):+ tracker = UniqueKeyTracker(("id",), max_bytes=1)+ tracker.check(_batch([1], ["a"]))+ assert list(tmp_path.iterdir())++ tracker.close()++ assert not list(tmp_path.iterdir())+- with pytest.raises(IncrementalWriteError, match=r"too many unique keys to verify \(4 so far\)"):- tracker.check(_batch([3, 4], ["c", "d"]))+def test_closing_during_a_check_stops_it_and_removes_the_index(tmp_path: Path) -> None:+ with patch.object(tempfile, "tempdir", str(tmp_path)):+ tracker = UniqueKeyTracker(("id",), max_bytes=1)+ tracker.check(_batch([0], ["a"]))+ dumps = json.dumps+ def close_mid_insert(*args: Any, **kwargs: Any) -> str:+ tracker.close()+ return dumps(*args, **kwargs)-def test_the_cap_admits_a_seven_million_key_uuid_run() -> None:- sample = 10_000- ids = pa.array(np.arange(sample, dtype=np.int64))- uuid_sized = pa.table({"id": pc.utf8_lpad(pc.cast(ids, pa.string()), width=36, padding="0")})+ ids = list(range(1, 20_000))+ with patch(f"{_MODULE}.json.dumps", side_effect=close_mid_insert), pytest.raises(_TrackerClosed):+ tracker.check(_batch(ids, ["x"] * len(ids)))- assert uuid_sized.nbytes * (7_000_000 // sample) <= MAX_UNIQUE_KEY_CHECK_BYTES+ assert not list(tmp_path.iterdir())@pytest.mark.parametrize(