Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-modeling): spill unique key checks to disk past a memory budget (#127415)

Raúl Negrón-Oteroee12cb3a85b5
diff --git a/posthog/temporal/data_modeling/activities/incremental_write.py b/posthog/temporal/data_modeling/activities/incremental_write.py
index 1aab64af0f05..44bbc0c79e21 100644
--- a/posthog/temporal/data_modeling/activities/incremental_write.py
+++ b/posthog/temporal/data_modeling/activities/incremental_write.py
@@ -20,6 +20,14 @@ column rather than as a Rust error or as quiet drift. The fourth is why the full
stays on delta-rs.
"""
+import json
+import shutil
+import struct
+import sqlite3
+import threading
+from collections.abc import Iterator
+from pathlib import Path
+from tempfile import TemporaryDirectory
from typing import Any, Optional
from django.conf import settings
@@ -27,12 +35,19 @@ from django.conf import settings
import pyarrow as pa
import pyarrow.compute as pc
-# Exact duplicate detection has to remember every distinct key a run has produced. The cap bounds
-# that state so a huge materialization cannot OOM the worker; hitting it fails the run loudly
-# rather than quietly skipping verification. Peak memory of the check is about this state plus one
-# batch, and a worker runs many activities at once, so the cap has to stay a small share of a pod.
+# Exact duplicate detection has to remember every distinct key a run has produced. Past this budget
+# the keys move to a local SQLite index, so a huge materialization cannot OOM the worker. Peak memory
+# of the in-memory check is about this state plus one slice, and a worker runs many activities at
+# once, so the budget has to stay a small share of a pod.
MAX_UNIQUE_KEY_CHECK_BYTES = 1024 * 1024 * 1024
+# The spilled index shares the worker's temporary disk with every other activity on it. Past this
+# size, or once that disk has less than the reserve free, the check fails the run loudly rather than
+# filling the disk under its neighbors.
+MAX_UNIQUE_KEY_SPILL_BYTES = 4 * 1024 * 1024 * 1024
+MIN_FREE_SPILL_DISK_BYTES = 2 * 1024 * 1024 * 1024
+_SPILL_FILE = "keys.sqlite3"
+
# A materialization batch is about 100 MB, so a narrow key-only table sends ~12M rows at once, and the
# group-by hash table for that many rows dwarfs the stored keys. Checking in slices keeps that bounded.
UNIQUE_KEY_CHECK_SLICE_ROWS = 1_000_000
@@ -42,6 +57,10 @@ class IncrementalWriteError(Exception):
"""A batch cannot be safely upserted. Carries a message meant for the run's error field."""
+class _TrackerClosed(Exception):
+ """The run closed the tracker while a check was still running on its thread."""
+
+
class SchemaDriftError(IncrementalWriteError):
"""The query's output no longer matches the stored table.
@@ -98,6 +117,12 @@ def assert_no_null_keys(batch: pa.RecordBatch, unique_key: tuple[str, ...]) -> N
)
+def _spill_key(row: tuple[Any, ...]) -> str:
+ # Arrow compares float keys by their bits: NaN payloads stay apart, and so do 0.0 and -0.0.
+ values = [struct.pack(">d", value).hex() if isinstance(value, float) else value for value in row]
+ return json.dumps(values, default=str, ensure_ascii=False, separators=(",", ":"))
+
+
class UniqueKeyTracker:
"""Exact unique-key enforcement across every batch of one materialization.
@@ -106,13 +131,18 @@ class UniqueKeyTracker:
twice, whereas here a duplicate means the config is wrong and quietly keeping one row would
hide it. A per-batch check cannot see a duplicate split across batches — deltalite would turn
it into an update, silently — so the distinct keys seen so far are carried across ``check``
- calls. Only the key columns of the group_by output are retained, so memory grows with distinct
- keys, not rows, and the byte cap turns "too big to verify" into a loud failure.
-
- Each call groups only the new batch, then probes the stored keys with a hash join whose build
- side is that small grouped batch. The stored keys are kept as a list of per-batch chunks and are
- never merged into one buffer or inserted into a hash table again. Each call only looks them up in
- the batch's small table, which is much cheaper than regrouping everything seen so far.
+ calls. Only key columns are retained, and a local SQLite index takes over when they exceed the
+ memory budget. The index has its own disk budget, and hitting it fails the run loudly rather
+ than quietly skipping verification.
+
+ The caller must ``close`` the tracker when the run ends, including on cancellation. A check runs
+ on an executor thread that cancelling the caller does not stop, so ``close`` also tells a running
+ check to stop at its next chunk.
+
+ In memory, each call groups only the new batch, then probes the stored keys with a hash join whose
+ build side is that small grouped batch. The stored keys are kept as a list of per-batch chunks and
+ are never merged into one buffer or inserted into a hash table again. Each call only looks them up
+ in the batch's small table, which is much cheaper than regrouping everything seen so far.
"""
def __init__(self, unique_key: tuple[str, ...], max_bytes: int = MAX_UNIQUE_KEY_CHECK_BYTES) -> None:
@@ -120,7 +150,87 @@ class UniqueKeyTracker:
self._max_bytes = max_bytes
self._seen_chunks: list[pa.Table] = []
self._seen_bytes = 0
- self._seen_rows = 0
+ self._spill: sqlite3.Connection | None = None
+ self._spill_directory: TemporaryDirectory[str] | None = None
+ self._running = threading.Lock()
+ self._closed = threading.Event()
+
+ def close(self) -> None:
+ """Remove the spilled index. Safe to call from any thread: a running check stops at its next
+ chunk and removes the index itself."""
+ self._closed.set()
+ if not self._running.acquire(blocking=False):
+ return
+ try:
+ if self._spill is not None:
+ self._spill.close()
+ if self._spill_directory is not None:
+ self._spill_directory.cleanup()
+ finally:
+ self._running.release()
+
+ def _raise_if_closed(self) -> None:
+ if self._closed.is_set():
+ raise _TrackerClosed()
+
+ def _duplicate_key_error(self, occurrences: int) -> IncrementalWriteError:
+ return IncrementalWriteError(
+ f"The unique key ({', '.join(self._columns)}) does not identify a single row: one key "
+ f"appears {occurrences} times in this run's result. Add the columns that tell those rows apart."
+ )
+
+ def _raise_if_over_disk_budget(self) -> None:
+ assert self._spill_directory is not None
+ directory = Path(self._spill_directory.name)
+ if (directory / _SPILL_FILE).stat().st_size > MAX_UNIQUE_KEY_SPILL_BYTES or (
+ shutil.disk_usage(directory).free < MIN_FREE_SPILL_DISK_BYTES
+ ):
+ raise IncrementalWriteError(
+ "This query returns too many unique keys to verify. Reduce the rows the query returns, or "
+ "turn off incremental materialization for this view to fall back to a full refresh."
+ )
+
+ def _spill_keys(self, keys: pa.Table) -> Iterator[tuple[str]]:
+ for batch in keys.to_batches(max_chunksize=8192):
+ self._raise_if_closed()
+ self._raise_if_over_disk_budget()
+ for row in zip(*batch.to_pydict().values(), strict=True):
+ yield (_spill_key(row),)
+
+ def _store(self, distinct_keys: pa.Table) -> None:
+ assert self._spill is not None
+ try:
+ with self._spill:
+ self._spill.executemany("INSERT INTO unique_keys (key) VALUES (?)", self._spill_keys(distinct_keys))
+ except sqlite3.IntegrityError as error:
+ # The keys are distinct, so a rejected one appears here once and was stored once before.
+ raise self._duplicate_key_error(2) from error
+
+ def _check_disk(self, keys: pa.Table) -> None:
+ assert self._spill is not None
+ grouped = keys.group_by(self._columns).aggregate([([], "count_all")])
+ if grouped.num_rows == keys.num_rows:
+ self._store(keys)
+ return
+ # The run fails either way. Only a key at the slice's top count can appear once more than
+ # that, if an earlier slice stored it.
+ top = pc.max(grouped.column("count_all")).as_py()
+ candidates = grouped.filter(pc.equal(grouped.column("count_all"), top)).select(self._columns)
+ lookup = "SELECT 1 FROM unique_keys WHERE key = ?"
+ stored = any(self._spill.execute(lookup, key).fetchone() for key in self._spill_keys(candidates))
+ raise self._duplicate_key_error(top + stored)
+
+ def _start_spill(self) -> None:
+ self._spill_directory = TemporaryDirectory(prefix="posthog-materialization-keys-")
+ # Each check is awaited before the next starts, but asyncio.to_thread may choose another thread.
+ self._spill = sqlite3.connect(Path(self._spill_directory.name) / _SPILL_FILE, check_same_thread=False)
+ # The index is discarded with the run, so journaling and fsync buy nothing.
+ self._spill.execute("PRAGMA journal_mode=OFF")
+ self._spill.execute("PRAGMA synchronous=OFF")
+ self._spill.execute("CREATE TABLE unique_keys (key TEXT PRIMARY KEY) WITHOUT ROWID")
+ for chunk in self._seen_chunks:
+ self._store(chunk)
+ self._seen_chunks = []
def check(self, batch: pa.RecordBatch) -> None:
"""Fail if ``batch`` has a missing, null, or duplicate key — including a key already seen
@@ -133,11 +243,20 @@ class UniqueKeyTracker:
f"Update the incremental configuration to match the query's columns."
)
assert_no_null_keys(batch, tuple(self._columns))
- for offset in range(0, max(batch.num_rows, 1), UNIQUE_KEY_CHECK_SLICE_ROWS):
- self._check_slice(batch.slice(offset, UNIQUE_KEY_CHECK_SLICE_ROWS))
+ try:
+ with self._running:
+ for offset in range(0, max(batch.num_rows, 1), UNIQUE_KEY_CHECK_SLICE_ROWS):
+ self._raise_if_closed()
+ self._check_slice(batch.slice(offset, UNIQUE_KEY_CHECK_SLICE_ROWS))
+ finally:
+ if self._closed.is_set():
+ self.close()
def _check_slice(self, batch: pa.RecordBatch) -> None:
keys = pa.Table.from_batches([batch]).select(self._columns)
+ if self._spill is not None:
+ self._check_disk(keys)
+ return
grouped = keys.group_by(self._columns).aggregate([([], "count_all")])
worst = pc.max(grouped.column("count_all")).as_py() if grouped.num_rows != keys.num_rows else 1
if self._seen_chunks:
@@ -148,21 +267,13 @@ class UniqueKeyTracker:
if already_seen.num_rows:
worst = max(worst, pc.max(already_seen.column("count_all")).as_py() + 1)
if worst > 1:
- raise IncrementalWriteError(
- f"The unique key ({', '.join(self._columns)}) does not identify a single row: one key "
- f"appears {worst} times in this run's result. Add the columns that tell those rows apart."
- )
+ raise self._duplicate_key_error(worst)
distinct = grouped.select(self._columns).combine_chunks()
+ self._seen_chunks.append(distinct)
self._seen_bytes += distinct.nbytes
- self._seen_rows += distinct.num_rows
if self._seen_bytes > self._max_bytes:
- raise IncrementalWriteError(
- f"This query returns too many unique keys to verify ({self._seen_rows} so far). "
- f"Reduce the rows the query returns, or turn off incremental materialization for "
- f"this view to fall back to a full refresh."
- )
- self._seen_chunks.append(distinct)
+ self._start_spill()
def max_of(batch: pa.RecordBatch, column: str, running: Any = None) -> Any:
diff --git a/posthog/temporal/data_modeling/activities/materialize_view.py b/posthog/temporal/data_modeling/activities/materialize_view.py
index 9d458131c4c9..9365e3021140 100644
--- a/posthog/temporal/data_modeling/activities/materialize_view.py
+++ b/posthog/temporal/data_modeling/activities/materialize_view.py
@@ -1061,43 +1061,47 @@ async def _materialize_fully(
# brief to_thread released between batches, never pins a worker thread for the whole
# read — which is what starved the shared executor that heartbeats/db/logging use.
batch_index = 0
- async for batch, ch_types in hogql_table(hogql_query, objects.team, logger):
- batch = _transform_unsupported_decimals(batch)
- batch = _transform_date_and_datetimes(batch, ch_types)
- batch = _transform_unsigned_integers(batch)
- batch = _force_nullable(batch)
+ try:
+ async for batch, ch_types in hogql_table(hogql_query, objects.team, logger):
+ batch = _transform_unsupported_decimals(batch)
+ batch = _transform_date_and_datetimes(batch, ch_types)
+ batch = _transform_unsigned_integers(batch)
+ batch = _force_nullable(batch)
+ if tracker is not None:
+ await asyncio.to_thread(tracker.check, batch)
+ await _stage_person_property_batch(person_property_sink, batch_index, batch, fatal=False)
+ batch_index += 1
+ if delta_table is None:
+ pa_schema = batch.schema
+ await asyncio.to_thread(
+ deltalake.write_deltalake,
+ table_or_uri=table_uri,
+ data=batch,
+ mode="overwrite",
+ schema_mode="overwrite",
+ storage_options=storage_options,
+ )
+ delta_table = deltalake.DeltaTable(table_uri, storage_options=storage_options)
+ else:
+ await asyncio.to_thread(
+ deltalake.write_deltalake,
+ table_or_uri=delta_table,
+ data=batch,
+ mode="append",
+ schema_mode="merge",
+ storage_options=storage_options,
+ )
+ # Staged only once the batch is committed: a run that announces rows it then fails to write
+ # cannot take the announcement back.
+ await cdp_sink.stage(batch)
+ if plan.config is not None and plan.config.incremental_key in batch.schema.names:
+ watermark = max_of(batch, plan.config.incremental_key, watermark)
+ row_count = row_count + batch.num_rows
+ objects.job.rows_materialized = row_count
+ await database_sync_to_async_pool(objects.job.save)()
+ finally:
if tracker is not None:
- await asyncio.to_thread(tracker.check, batch)
- await _stage_person_property_batch(person_property_sink, batch_index, batch, fatal=False)
- batch_index += 1
- if delta_table is None:
- pa_schema = batch.schema
- await asyncio.to_thread(
- deltalake.write_deltalake,
- table_or_uri=table_uri,
- data=batch,
- mode="overwrite",
- schema_mode="overwrite",
- storage_options=storage_options,
- )
- delta_table = deltalake.DeltaTable(table_uri, storage_options=storage_options)
- else:
- await asyncio.to_thread(
- deltalake.write_deltalake,
- table_or_uri=delta_table,
- data=batch,
- mode="append",
- schema_mode="merge",
- storage_options=storage_options,
- )
- # Staged only once the batch is committed: a run that announces rows it then fails to write
- # cannot take the announcement back.
- await cdp_sink.stage(batch)
- if plan.config is not None and plan.config.incremental_key in batch.schema.names:
- watermark = max_of(batch, plan.config.incremental_key, watermark)
- row_count = row_count + batch.num_rows
- objects.job.rows_materialized = row_count
- await database_sync_to_async_pool(objects.job.save)()
+ tracker.close()
await logger.ainfo(f"Finished writing to delta table. row_count={row_count}")
file_uris: list[str] = []
@@ -1197,6 +1201,8 @@ async def _materialize_incrementally(
if isinstance(err, SchemaDriftError):
await logger.awarning(f"Rebuilding after schema drift: {err}")
raise
+ finally:
+ tracker.close()
if watermark is not None:
await database_sync_to_async_pool(set_incremental_state)(
diff --git a/posthog/temporal/tests/data_modeling/test_incremental_write.py b/posthog/temporal/tests/data_modeling/test_incremental_write.py
index 8a7759ea0640..020d214c7461 100644
--- a/posthog/temporal/tests/data_modeling/test_incremental_write.py
+++ b/posthog/temporal/tests/data_modeling/test_incremental_write.py
@@ -5,24 +5,40 @@ detection logic itself, where the cheap edge cases live (composite keys, the byt
column missing from the output).
"""
+import json
+import struct
+import tempfile
+import contextlib
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any
+
import pytest
from unittest.mock import MagicMock, patch
from django.test import override_settings
-import numpy as np
import pyarrow as pa
-import pyarrow.compute as pc
from posthog.temporal.data_modeling.activities.incremental_write import (
MAX_UNIQUE_KEY_CHECK_BYTES,
+ MAX_UNIQUE_KEY_SPILL_BYTES,
IncrementalWriteError,
UniqueKeyTracker,
+ _TrackerClosed,
upsert_batch,
)
+_MODULE = "posthog.temporal.data_modeling.activities.incremental_write"
+
+_spill_modes = pytest.mark.parametrize("max_bytes", [1, MAX_UNIQUE_KEY_CHECK_BYTES], ids=["disk", "memory"])
+
+
+def _float_from_bits(hex_bits: str) -> float:
+ return struct.unpack(">d", bytes.fromhex(hex_bits))[0]
-def _batch(ids: list[int | None], names: list[str | None]) -> pa.RecordBatch:
+
+def _batch(ids: Sequence[int | None], names: Sequence[str | None]) -> pa.RecordBatch:
return pa.RecordBatch.from_arrays(
[pa.array(ids, type=pa.int64()), pa.array(names, type=pa.string())],
names=["id", "name"],
@@ -53,11 +69,17 @@ def _batch(ids: list[int | None], names: list[str | None]) -> pa.RecordBatch:
"appears 5 times",
id="within_batch_count_outweighs_a_seen_key",
),
+ pytest.param(
+ [([1, 2], ["a", "b"]), ([1, 2, 2, 2], ["c", "d", "e", "f"])],
+ "appears 4 times",
+ id="most_repeated_key_was_also_seen",
+ ),
pytest.param([([1, None], ["a", "b"])], "is null", id="null_key"),
],
)
-def test_single_column_violations_are_detected(batches: list[tuple], match: str) -> None:
- tracker = UniqueKeyTracker(("id",))
+@_spill_modes
+def test_single_column_violations_are_detected(batches: list[tuple], match: str, max_bytes: int) -> None:
+ tracker = UniqueKeyTracker(("id",), max_bytes=max_bytes)
with pytest.raises(IncrementalWriteError, match=match):
for ids, names in batches:
@@ -79,9 +101,9 @@ def test_distinct_keys_across_batches_pass() -> None:
tracker.check(_batch([3, 4], ["c", "d"]))
-def test_composite_key_is_checked_as_a_tuple() -> None:
- """Columns may repeat individually; only the full tuple repeating is a violation."""
- tracker = UniqueKeyTracker(("id", "name"))
+@_spill_modes
+def test_composite_key_is_checked_as_a_tuple(max_bytes: int) -> None:
+ tracker = UniqueKeyTracker(("id", "name"), max_bytes=max_bytes)
tracker.check(_batch([1, 1], ["a", "b"]))
tracker.check(_batch([2, 2], ["a", "b"]))
@@ -90,6 +112,23 @@ def test_composite_key_is_checked_as_a_tuple() -> None:
tracker.check(_batch([1], ["a"]))
+ "value,other,same",
+ [
+ (-0.0, 0.0, False),
+ (float("nan"), float("nan"), True),
+ (_float_from_bits("7ff8000000000000"), _float_from_bits("7ff8000000000001"), False),
+ ],
+)
+@_spill_modes
+def test_float_keys_compare_alike_in_memory_and_on_disk(value: float, other: float, same: bool, max_bytes: int) -> None:
+ tracker = UniqueKeyTracker(("key",), max_bytes=max_bytes)
+ tracker.check(pa.record_batch([[value]], names=["key"]))
+ with pytest.raises(IncrementalWriteError, match="appears 2 times") if same else contextlib.nullcontext():
+ tracker.check(pa.record_batch([[other]], names=["key"]))
+ tracker.close()
+
+
def test_a_missing_key_column_names_the_column() -> None:
tracker = UniqueKeyTracker(("missing",))
@@ -97,21 +136,57 @@ def test_a_missing_key_column_names_the_column() -> None:
tracker.check(_batch([1], ["a"]))
-def test_exceeding_the_memory_cap_fails_loudly() -> None:
- """The cap must fail the run, never silently stop checking."""
- tracker = UniqueKeyTracker(("id",), max_bytes=24)
- tracker.check(_batch([1, 2], ["a", "b"]))
+def test_spilling_preserves_keys_from_earlier_batches() -> None:
+ tracker = UniqueKeyTracker(("id",), max_bytes=8)
+
+ tracker.check(_batch([1], ["a"]))
+ tracker.check(_batch([2, 3, 4], ["b", "c", "d"]))
+ with pytest.raises(IncrementalWriteError, match="does not identify a single row"):
+ tracker.check(_batch([1], ["a"]))
+ tracker.close()
+
+
+ "spill_bytes,free_reserve",
+ [pytest.param(1, 0, id="index_budget"), pytest.param(MAX_UNIQUE_KEY_SPILL_BYTES, 1 << 62, id="free_disk")],
+)
+def test_the_spilled_index_fails_loudly_past_its_disk_budget(spill_bytes: int, free_reserve: int) -> None:
+ tracker = UniqueKeyTracker(("id",), max_bytes=1)
+ with (
+ patch(f"{_MODULE}.MAX_UNIQUE_KEY_SPILL_BYTES", spill_bytes),
+ patch(f"{_MODULE}.MIN_FREE_SPILL_DISK_BYTES", free_reserve),
+ pytest.raises(IncrementalWriteError, match="too many unique keys to verify"),
+ ):
+ tracker.check(_batch([1, 2], ["a", "b"]))
+ tracker.close()
+
+
+def test_closing_removes_the_spilled_index(tmp_path: Path) -> None:
+ with patch.object(tempfile, "tempdir", str(tmp_path)):
+ tracker = UniqueKeyTracker(("id",), max_bytes=1)
+ tracker.check(_batch([1], ["a"]))
+ assert list(tmp_path.iterdir())
+
+ tracker.close()
+
+ assert not list(tmp_path.iterdir())
+
- with pytest.raises(IncrementalWriteError, match=r"too many unique keys to verify \(4 so far\)"):
- tracker.check(_batch([3, 4], ["c", "d"]))
+def test_closing_during_a_check_stops_it_and_removes_the_index(tmp_path: Path) -> None:
+ with patch.object(tempfile, "tempdir", str(tmp_path)):
+ tracker = UniqueKeyTracker(("id",), max_bytes=1)
+ tracker.check(_batch([0], ["a"]))
+ dumps = json.dumps
+ def close_mid_insert(*args: Any, **kwargs: Any) -> str:
+ tracker.close()
+ return dumps(*args, **kwargs)
-def test_the_cap_admits_a_seven_million_key_uuid_run() -> None:
- sample = 10_000
- ids = pa.array(np.arange(sample, dtype=np.int64))
- uuid_sized = pa.table({"id": pc.utf8_lpad(pc.cast(ids, pa.string()), width=36, padding="0")})
+ ids = list(range(1, 20_000))
+ with patch(f"{_MODULE}.json.dumps", side_effect=close_mid_insert), pytest.raises(_TrackerClosed):
+ tracker.check(_batch(ids, ["x"] * len(ids)))
- assert uuid_sized.nbytes * (7_000_000 // sample) <= MAX_UNIQUE_KEY_CHECK_BYTES
+ assert not list(tmp_path.iterdir())
@pytest.mark.parametrize(