Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse): stream staged rows in person property sync (#126177)

estefania38e2f194e1af
diff --git a/products/customer_analytics/backend/logic/account_property_sync.py b/products/customer_analytics/backend/logic/account_property_sync.py
index a5f9f5d9c67a..9678db4eb414 100644
--- a/products/customer_analytics/backend/logic/account_property_sync.py
+++ b/products/customer_analytics/backend/logic/account_property_sync.py
@@ -225,8 +225,13 @@ def _source_values(rows: list[dict[str, Any]], key_column: str, source_column: s
}
-def _decode_parquet_rows(data: bytes) -> list[dict[str, Any]]:
- return pq.read_table(io.BytesIO(data)).to_pylist()
+def _merge_snapshot_file(data: bytes, hashes: dict[str, str]) -> None:
+ for batch in _parquet_batches(data):
+ external_ids = batch.column("external_id").to_pylist()
+ value_hashes = batch.column("value_hash").to_pylist()
+ hashes.update(
+ (str(external_id), str(value_hash)) for external_id, value_hash in zip(external_ids, value_hashes)
+ )
def _encode_snapshot(hashes: dict[str, str]) -> bytes:
@@ -310,8 +315,7 @@ async def _merge_snapshot_files(s3_client: _SnapshotS3Client, file_keys: list[st
except FileNotFoundError:
complete = False
continue
- for row in await asyncio.to_thread(_decode_parquet_rows, data):
- hashes[str(row["external_id"])] = str(row["value_hash"])
+ await asyncio.to_thread(_merge_snapshot_file, data, hashes)
return _SnapshotMerge(hashes=hashes, complete=complete)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py
index 590835b52ccb..14eef7fddf87 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py
@@ -22,13 +22,14 @@ config models. The pure functions (bundle building, hashing, diff) are unit-test
S3/personhog/Kafka helpers are the boundaries and are mocked in the orchestration test.
"""
+import io
import json
import asyncio
import hashlib
import dataclasses
from datetime import date, datetime, time
from decimal import Decimal
-from typing import Any
+from typing import TYPE_CHECKING, Any
from django.conf import settings
@@ -46,7 +47,7 @@ from posthog.models.group_type_mapping import get_group_types_for_project
from posthog.models.person.util import get_persons_mapped_by_distinct_id
from posthog.sync import database_sync_to_async
-from products.data_warehouse.backend.facade.api import aget_s3_client
+from products.data_warehouse.backend.facade.api import aget_s3_client, get_s3_client
from products.warehouse_sources.backend.facade.person_property_provenance import stamp_person_property_provenance
from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
from products.warehouse_sources.backend.temporal.data_imports.external_product_hooks import (
@@ -63,6 +64,9 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.per
snapshot_prefix,
)
+if TYPE_CHECKING:
+ from s3fs import S3FileSystem
+
logger = structlog.get_logger(__name__)
EVENT_SOURCE = "customer_analytics_person_property_sync"
@@ -84,6 +88,15 @@ def _log_fields(binding: WarehouseBinding) -> dict[str, str]:
# would otherwise materialize gigabytes and get the worker OOM-killed.
_EXISTENCE_LOOKUP_CHUNK_SIZE = 1_000
+# Rows per streamed parquet/Delta batch. Batches are processed then discarded, so peak memory is one
+# batch plus the per-source {distinct_id: bundle} accumulator (bounded by distinct-person cardinality).
+ROW_BATCH_SIZE = 50_000
+
+# Sources folded per pass over the staged files. Every source in a pass keeps its own
+# {distinct_id: bundle} map, so a binding with many sources would otherwise hold one map per source
+# for every person at once. Each extra pass reads only its own sources' columns.
+SOURCES_PER_STAGED_PASS = 4
+
@frozen(frozen=False)
class PerSourceResult:
@@ -178,6 +191,20 @@ def select_changed(
return list(changed.items()), new_hashes
+def _accumulate_bundles(
+ accumulated: dict[str, dict[str, dict]], rows: list[dict], sources: list[PersonPropertySyncSource]
+) -> None:
+ """Fold one batch of rows into {source_id: {distinct_id: bundle}}, last write winning per distinct_id."""
+ for source in sources:
+ bucket = accumulated[str(source.source_id)]
+ for distinct_id, bundle in build_bundles(rows, source.key_column, source.column_property_map or {}):
+ bucket[distinct_id] = bundle
+
+
+def _wanted_columns(sources: list[PersonPropertySyncSource]) -> set[str]:
+ return {column for source in sources for column in (source.key_column, *(source.column_property_map or {}).keys())}
+
+
# --- S3 / Kafka / personhog boundaries (mocked in tests) ---------------------------------
@@ -189,9 +216,36 @@ def _snapshot_prefix(team_id: int, binding: WarehouseBinding, source_id: str) ->
return snapshot_prefix(team_id, binding, source_id)
-async def _read_staged_rows(team_id: int, binding: WarehouseBinding, job_id: str) -> list[dict]:
+@frozen
+class StagedBundleRead:
+ """What one job's staged files yielded for an incremental sync."""
+
+ bundles_by_source: dict[str, dict[str, dict]]
+ rows_read: int
+
+
+def _accumulate_staged_file(
+ s3_client: "S3FileSystem",
+ path: str,
+ sources: list[PersonPropertySyncSource],
+ accumulated: dict[str, dict[str, dict]],
+) -> int:
+ """Stream one staged parquet file batch by batch into ``accumulated``. Returns the rows read."""
+ rows_read = 0
+ with s3_client.open(path, "rb", cache_type="none") as file:
+ parquet = pq.ParquetFile(file, pre_buffer=False)
+ # Staged chunks can differ in columns after schema drift, and requesting an absent column raises.
+ columns = sorted(_wanted_columns(sources) & set(parquet.schema_arrow.names))
+ for batch in parquet.iter_batches(batch_size=ROW_BATCH_SIZE, columns=columns):
+ rows = batch.to_pylist()
+ rows_read += len(rows)
+ _accumulate_bundles(accumulated, rows, sources)
+ return rows_read
+
+
+async def _list_staged_files(team_id: int, binding: WarehouseBinding, job_id: str) -> list[str]:
+ """S3 keys of the job's staged files in write order. Empty when the run staged nothing."""
prefix = job_staged_prefix(team_id, binding, job_id)
- rows: list[dict] = []
async with aget_s3_client() as s3_client:
try:
listing = await s3_client._ls(f"s3://{prefix}/", detail=True)
@@ -199,24 +253,41 @@ async def _read_staged_rows(team_id: int, binding: WarehouseBinding, job_id: str
# No staged folder — the run staged nothing (common on a no-change incremental run).
logger.debug("person-property sync: no staged rows folder", team_id=team_id, **_log_fields(binding))
return []
- values = listing.values() if isinstance(listing, dict) else listing
- files = [f["Key"] for f in values if f.get("type") != "directory"]
- for file_path in files:
- data = await s3_client._cat_file(f"s3://{file_path}" if not file_path.startswith("s3://") else file_path)
- # Decode off the event loop: a large parquet chunk would otherwise block the async
- # activity long enough to starve the heartbeater and trip its timeout.
- rows.extend(await asyncio.to_thread(_decode_parquet_rows, data))
- return rows
-
+ values = listing.values() if isinstance(listing, dict) else listing
+ files = [f for f in values if f.get("type") != "directory"]
+ # Staged filenames sort in attempt and chunk order, so last-write-wins follows write order.
+ file_keys = sorted(f["Key"] for f in files)
+ logger.info(
+ "person-property sync: reading staged files",
+ team_id=team_id,
+ **_log_fields(binding),
+ job_id=job_id,
+ staged_files=len(file_keys),
+ staged_bytes=sum(f.get("size") or 0 for f in files),
+ )
+ return file_keys
-def _bytes_reader(data: bytes):
- import io # noqa: PLC0415 — local to keep the module import light
- return io.BytesIO(data)
+async def _read_staged_bundles(file_keys: list[str], sources: list[PersonPropertySyncSource]) -> StagedBundleRead:
+ """Stream the staged files into {source_id: {distinct_id: bundle}}, one file and one batch at a
+ time, so peak memory tracks distinct persons rather than staged row count."""
+ accumulated: dict[str, dict[str, dict]] = {str(source.source_id): {} for source in sources}
+ s3_sync_client = get_s3_client()
+ rows_read = 0
+ for key in file_keys:
+ # Read off the event loop so a large file can't starve the heartbeater.
+ rows_read += await asyncio.to_thread(
+ _accumulate_staged_file, s3_sync_client, _s3_uri(key), sources, accumulated
+ )
+ return StagedBundleRead(bundles_by_source=accumulated, rows_read=rows_read)
-def _decode_parquet_rows(data: bytes) -> list[dict]:
- return pq.read_table(_bytes_reader(data)).to_pylist()
+def _merge_snapshot_file(data: bytes, hashes: dict[str, str]) -> None:
+ parquet = pq.ParquetFile(io.BytesIO(data))
+ for batch in parquet.iter_batches(batch_size=ROW_BATCH_SIZE, columns=["distinct_id", "sent_hash"]):
+ distinct_ids = batch.column("distinct_id").to_pylist()
+ sent_hashes = batch.column("sent_hash").to_pylist()
+ hashes.update((str(distinct_id), str(sent_hash)) for distinct_id, sent_hash in zip(distinct_ids, sent_hashes))
def _snapshot_file_order(entry: dict) -> tuple:
@@ -261,8 +332,7 @@ async def _merge_snapshot_files(s3_client, file_keys: list[str]) -> dict[str, st
# after we listed the folder but before we read it. Its rows survive in that new file (or
# whichever one replaces it next run) — skip rather than fail the whole read.
continue
- for record in await asyncio.to_thread(_decode_parquet_rows, data):
- hashes[record["distinct_id"]] = record["sent_hash"]
+ await asyncio.to_thread(_merge_snapshot_file, data, hashes)
return hashes
@@ -518,6 +588,30 @@ async def _process_source_bundles(
return ps
+async def _process_accumulated_source(
+ *,
+ team_id: int,
+ team: Team,
+ binding: WarehouseBinding,
+ source: PersonPropertySyncSource,
+ bundles_by_source: dict[str, dict[str, dict]],
+ rows_read: int,
+ run_token: str,
+) -> PerSourceResult:
+ bundles = list(bundles_by_source.pop(str(source.source_id)).items())
+ return await _process_source_bundles(
+ team_id=team_id,
+ project_id=team.project_id,
+ binding=binding,
+ team_api_token=team.api_token,
+ team_uuid=str(team.uuid),
+ source=source,
+ bundles=bundles,
+ rows_read=rows_read,
+ run_token=run_token,
+ )
+
+
def _accumulate(result: SyncResult, ps: PerSourceResult) -> None:
result.per_source.append(ps)
result.changed += ps.changed
@@ -539,32 +633,32 @@ async def run_person_property_sync(*, team_id: int, binding: WarehouseBinding, j
return result
team = await database_sync_to_async(Team.objects.get, thread_sensitive=False)(id=team_id)
- rows = await _read_staged_rows(team_id, binding, job_id)
+ file_keys = await _list_staged_files(team_id, binding, job_id)
result.sources = len(sources)
- result.rows_read = len(rows)
- logger.info(
- "person-property sync: read staged rows",
- team_id=team_id,
- **_log_fields(binding),
- job_id=job_id,
- sources=len(sources),
- rows_read=len(rows),
- )
- for source in sources:
- bundles = await asyncio.to_thread(build_bundles, rows, source.key_column, source.column_property_map or {})
- ps = await _process_source_bundles(
+ for start in range(0, len(sources), SOURCES_PER_STAGED_PASS):
+ pass_sources = sources[start : start + SOURCES_PER_STAGED_PASS]
+ read = await _read_staged_bundles(file_keys, pass_sources)
+ result.rows_read = read.rows_read
+ logger.info(
+ "person-property sync: read staged rows",
team_id=team_id,
- project_id=team.project_id,
- binding=binding,
- team_api_token=team.api_token,
- team_uuid=str(team.uuid),
- source=source,
- bundles=bundles,
- rows_read=len(rows),
- run_token=job_id,
+ **_log_fields(binding),
+ job_id=job_id,
+ sources=len(pass_sources),
+ rows_read=read.rows_read,
)
- _accumulate(result, ps)
+ for source in pass_sources:
+ ps = await _process_accumulated_source(
+ team_id=team_id,
+ team=team,
+ binding=binding,
+ source=source,
+ bundles_by_source=read.bundles_by_source,
+ rows_read=read.rows_read,
+ run_token=job_id,
+ )
+ _accumulate(result, ps)
await _clear_staged(team_id, binding, job_id)
return result
@@ -572,10 +666,6 @@ async def run_person_property_sync(*, team_id: int, binding: WarehouseBinding, j
# --- backfill (full Delta read) ----------------------------------------------------------
-# Rows per streamed Delta batch. Batches are processed then discarded, so peak memory is one batch
-# plus the per-source {distinct_id: bundle} accumulator (bounded by distinct-person cardinality).
-BACKFILL_BATCH_SIZE = 50_000
-
# Backfill runs share one snapshot filename per source (a full-table read replaces the whole set),
# distinct from the incremental job-id filenames, so the two writers never collide.
BACKFILL_RUN_TOKEN = "backfill"
@@ -608,11 +698,8 @@ def _read_delta_bundles(
dataset = deltalake.DeltaTable(uri, storage_options=storage_options).to_pyarrow_dataset()
available = set(dataset.schema.names)
- wanted = {
- column for source in sources for column in (source.key_column, *(source.column_property_map or {}).keys())
- }
# Project only columns that exist — a misconfigured column drops out rather than erroring.
- project = sorted(wanted & available)
+ project = sorted(_wanted_columns(sources) & available)
# A source whose key column itself is missing produces zero bundles for the whole table, which is
# otherwise indistinguishable from a genuinely idle backfill. Report it back so the caller can fail
@@ -629,13 +716,10 @@ def _read_delta_bundles(
)
rows_read = 0
- for batch in dataset.to_batches(columns=project, batch_size=BACKFILL_BATCH_SIZE):
+ for batch in dataset.to_batches(columns=project, batch_size=ROW_BATCH_SIZE):
rows = batch.to_pylist()
rows_read += len(rows)
- for source in sources:
- bucket = accumulated[str(source.source_id)]
- for distinct_id, bundle in build_bundles(rows, source.key_column, source.column_property_map or {}):
- bucket[distinct_id] = bundle
+ _accumulate_bundles(accumulated, rows, sources)
return DeltaBundleRead(
bundles_by_source=accumulated,
rows_read=rows_read,
@@ -726,15 +810,12 @@ async def run_person_property_backfill(*, team_id: int, binding: WarehouseBindin
),
)
continue
- bundles = list(read.bundles_by_source[str(source.source_id)].items())
- ps = await _process_source_bundles(
+ ps = await _process_accumulated_source(
team_id=team_id,
- project_id=team.project_id,
+ team=team,
binding=binding,
- team_api_token=team.api_token,
- team_uuid=str(team.uuid),
source=source,
- bundles=bundles,
+ bundles_by_source=read.bundles_by_source,
rows_read=read.rows_read,
run_token=BACKFILL_RUN_TOKEN,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync_test.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync_test.py
index 2a3b4fa1c6aa..f15105acbfda 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync_test.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync_test.py
@@ -1,3 +1,4 @@
+import io
import json
import threading
from datetime import date, datetime
@@ -11,6 +12,8 @@ from django.db.models import Q
from django.test import TransactionTestCase
from django.test.utils import CaptureQueriesContext
+import pyarrow as pa
+import pyarrow.parquet as pq
from parameterized import parameterized
from posthog.models import Organization, PropertyDefinition, Team
@@ -156,10 +159,12 @@ class TestRunOrchestration:
order.append("produce")
return 2
+ fake = _FakeS3()
+ _stage(fake, "chunk_1_000000", rows)
with (
+ _fake_s3_patch(fake),
patch(f"{_MODULE}.person_property_sync_sources_for", return_value=[self._source()]),
patch(f"{_MODULE}.Team") as team_cls,
- patch(f"{_MODULE}._read_staged_rows", new=AsyncMock(return_value=rows)),
patch(f"{_MODULE}._read_snapshot_hashes", new=AsyncMock(return_value={})),
patch(f"{_MODULE}._filter_existing_ids", return_value={"a", "b"}) as existing,
patch(f"{_MODULE}._produce_intents", side_effect=_record_produce) as produce,
@@ -197,11 +202,55 @@ class TestRunOrchestration:
clear.assert_awaited_once()
assert order == ["reconcile", "produce", "snapshot"]
+ @parameterized.expand([("one_pass", 4), ("pass_per_source", 1)])
+ @pytest.mark.asyncio
+ async def test_streams_staged_files_into_one_projected_bundle_per_distinct_id(self, _name, sources_per_pass):
+ sources = [
+ PersonPropertySyncSource(
+ source_id="source-1",
+ definition_id="def-1",
+ key_column="distinct_id",
+ column_property_map={"plan": "plan_tier", "seats": "seat_count"},
+ ),
+ PersonPropertySyncSource(
+ source_id="source-2",
+ definition_id="def-2",
+ key_column="distinct_id",
+ column_property_map={"seats": "seat_count"},
+ ),
+ ]
+ fake = _FakeS3()
+ _stage(fake, "chunk_1_000001", [{"distinct_id": "a", "plan": "pro", "seats": 3, "notes": "unmapped"}])
+ _stage(fake, "chunk_1_000000", [{"distinct_id": "a", "plan": "free"}, {"distinct_id": "b", "plan": "pro"}])
+ with (
+ _fake_s3_patch(fake),
+ patch(f"{_MODULE}.SOURCES_PER_STAGED_PASS", sources_per_pass),
+ patch(f"{_MODULE}.person_property_sync_sources_for", return_value=sources),
+ patch(f"{_MODULE}.Team") as team_cls,
+ patch(f"{_MODULE}.build_bundles", wraps=pps.build_bundles) as build,
+ patch(f"{_MODULE}._read_snapshot_hashes", new=AsyncMock(return_value={})),
+ patch(f"{_MODULE}._filter_existing_ids", side_effect=lambda _team, _source, ids: set(ids)),
+ patch(f"{_MODULE}._produce_intents", return_value=2) as produce,
+ patch(f"{_MODULE}._write_snapshot_hashes", new=AsyncMock()),
+ patch(f"{_MODULE}._reconcile_property_definitions"),
+ patch(f"{_MODULE}._clear_staged", new=AsyncMock()),
+ ):
+ team_cls.objects.get.return_value = MagicMock(api_token="tok", project_id=7)
+ result = await pps.run_person_property_sync(team_id=1, binding=_SCHEMA, job_id="job-1")
+
+ produced = {call.args[2].source_id: sorted(call.args[3]) for call in produce.call_args_list}
+ assert produced == {
+ "source-1": [("a", {"plan_tier": "pro", "seat_count": 3}), ("b", {"plan_tier": "pro"})],
+ "source-2": [("a", {"seat_count": 3})],
+ }
+ assert result.rows_read == 3
+ assert all("notes" not in row for call in build.call_args_list for row in call.args[0])
+
@pytest.mark.asyncio
async def test_no_sources_is_a_noop(self):
with (
patch(f"{_MODULE}.person_property_sync_sources_for", return_value=None),
- patch(f"{_MODULE}._read_staged_rows", new=AsyncMock()) as read,
+ patch(f"{_MODULE}._read_staged_bundles", new=AsyncMock()) as read,
patch(f"{_MODULE}._clear_staged", new=AsyncMock()) as clear,
):
result = await pps.run_person_property_sync(team_id=1, binding=_SCHEMA, job_id="job-1")
@@ -464,7 +513,7 @@ class _FakeS3:
async def _ls(self, path, detail=True):
prefix = pps._s3_key(path).rstrip("/") + "/"
entries = [
- {"Key": key, "type": "file", "LastModified": self.times[key]}
+ {"Key": key, "type": "file", "LastModified": self.times[key], "size": len(self.store[key])}
for key in self.store
if key.startswith(prefix)
]
@@ -478,6 +527,12 @@ class _FakeS3:
raise FileNotFoundError(path)
return self.store[key]
+ def open(self, path, mode="rb", **_kwargs):
+ key = pps._s3_key(path)
+ if key not in self.store:
+ raise FileNotFoundError(path)
+ return io.BytesIO(self.store[key])
+
async def _pipe_file(self, path, data):
self._clock += 1
key = pps._s3_key(path)
@@ -498,7 +553,18 @@ def _fake_s3_patch(fake):
async def _cm():
yield fake
- return patch(f"{_MODULE}.aget_s3_client", lambda: _cm())
+ stack = contextlib.ExitStack()
+ stack.enter_context(patch(f"{_MODULE}.aget_s3_client", lambda: _cm()))
+ stack.enter_context(patch(f"{_MODULE}.get_s3_client", lambda: fake))
+ return stack
+
+
+def _stage(fake, name, rows, job_id="job-1"):
+ buffer = io.BytesIO()
+ pq.write_table(pa.Table.from_pylist(rows), buffer)
+ key = f"{pps._s3_key(pps.job_staged_prefix(1, _SCHEMA, job_id))}/{name}.parquet"
+ fake.store[key] = buffer.getvalue()
+ fake.times[key] = 0
class TestSnapshotCompaction:
@@ -591,11 +657,12 @@ class TestGroupTarget:
# If the group type can't be resolved (deleted/misconfigured), the consumer would DLQ every
# $groupidentify missing a group_type — so the source must be skipped, not produced.
team = MagicMock(api_token="tok", uuid="team-uuid")
- rows = [{"group_key": "acme", "plan": "pro"}]
+ fake = _FakeS3()
+ _stage(fake, "chunk_1_000000", [{"group_key": "acme", "plan": "pro"}])
with (
+ _fake_s3_patch(fake),
patch(f"{_MODULE}.person_property_sync_sources_for", return_value=[self._group_source()]),
patch(f"{_MODULE}.Team") as team_cls,
- patch(f"{_MODULE}._read_staged_rows", new=AsyncMock(return_value=rows)),
patch(f"{_MODULE}._read_snapshot_hashes", new=AsyncMock(return_value={})),
patch(f"{_MODULE}._filter_existing_ids", return_value={"acme"}),
patch(f"{_MODULE}._group_type_name", return_value=None),
@@ -618,12 +685,13 @@ class TestGroupTarget:
# a child environment has no rows under its own team id. Resolving by team id found nothing
# there, which skipped every group source on that team and produced no intents at all.
team = MagicMock(api_token="tok", uuid="team-uuid", project_id=77)
- rows = [{"group_key": "acme", "plan": "pro"}]
by_project = {77: [{"group_type_index": 0, "group_type": "organization"}]}
+ fake = _FakeS3()
+ _stage(fake, "chunk_1_000000", [{"group_key": "acme", "plan": "pro"}])
with (
+ _fake_s3_patch(fake),
patch(f"{_MODULE}.person_property_sync_sources_for", return_value=[self._group_source()]),
patch(f"{_MODULE}.Team") as team_cls,
- patch(f"{_MODULE}._read_staged_rows", new=AsyncMock(return_value=rows)),
patch(f"{_MODULE}._read_snapshot_hashes", new=AsyncMock(return_value={})),
patch(f"{_MODULE}._filter_existing_ids", return_value={"acme"}),
patch(