:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse): stream staged rows in person property sync (#126177)
estefania38e2f194e1af
diff --git a/products/customer_analytics/backend/logic/account_property_sync.py b/products/customer_analytics/backend/logic/account_property_sync.pyindex a5f9f5d9c67a..9678db4eb414 100644--- a/products/customer_analytics/backend/logic/account_property_sync.py+++ b/products/customer_analytics/backend/logic/account_property_sync.py@@ -225,8 +225,13 @@ def _source_values(rows: list[dict[str, Any]], key_column: str, source_column: s}-def _decode_parquet_rows(data: bytes) -> list[dict[str, Any]]:- return pq.read_table(io.BytesIO(data)).to_pylist()+def _merge_snapshot_file(data: bytes, hashes: dict[str, str]) -> None:+ for batch in _parquet_batches(data):+ external_ids = batch.column("external_id").to_pylist()+ value_hashes = batch.column("value_hash").to_pylist()+ hashes.update(+ (str(external_id), str(value_hash)) for external_id, value_hash in zip(external_ids, value_hashes)+ )def _encode_snapshot(hashes: dict[str, str]) -> bytes:@@ -310,8 +315,7 @@ async def _merge_snapshot_files(s3_client: _SnapshotS3Client, file_keys: list[stexcept FileNotFoundError:complete = Falsecontinue- for row in await asyncio.to_thread(_decode_parquet_rows, data):- hashes[str(row["external_id"])] = str(row["value_hash"])+ await asyncio.to_thread(_merge_snapshot_file, data, hashes)return _SnapshotMerge(hashes=hashes, complete=complete)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.pyindex 590835b52ccb..14eef7fddf87 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py@@ -22,13 +22,14 @@ config models. The pure functions (bundle building, hashing, diff) are unit-testS3/personhog/Kafka helpers are the boundaries and are mocked in the orchestration test."""+import ioimport jsonimport asyncioimport hashlibimport dataclassesfrom datetime import date, datetime, timefrom decimal import Decimal-from typing import Any+from typing import TYPE_CHECKING, Anyfrom django.conf import settings@@ -46,7 +47,7 @@ from posthog.models.group_type_mapping import get_group_types_for_projectfrom posthog.models.person.util import get_persons_mapped_by_distinct_idfrom posthog.sync import database_sync_to_async-from products.data_warehouse.backend.facade.api import aget_s3_client+from products.data_warehouse.backend.facade.api import aget_s3_client, get_s3_clientfrom products.warehouse_sources.backend.facade.person_property_provenance import stamp_person_property_provenancefrom products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchemafrom products.warehouse_sources.backend.temporal.data_imports.external_product_hooks import (@@ -63,6 +64,9 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.persnapshot_prefix,)+if TYPE_CHECKING:+ from s3fs import S3FileSystem+logger = structlog.get_logger(__name__)EVENT_SOURCE = "customer_analytics_person_property_sync"@@ -84,6 +88,15 @@ def _log_fields(binding: WarehouseBinding) -> dict[str, str]:# would otherwise materialize gigabytes and get the worker OOM-killed._EXISTENCE_LOOKUP_CHUNK_SIZE = 1_000+# Rows per streamed parquet/Delta batch. Batches are processed then discarded, so peak memory is one+# batch plus the per-source {distinct_id: bundle} accumulator (bounded by distinct-person cardinality).+ROW_BATCH_SIZE = 50_000++# Sources folded per pass over the staged files. Every source in a pass keeps its own+# {distinct_id: bundle} map, so a binding with many sources would otherwise hold one map per source+# for every person at once. Each extra pass reads only its own sources' columns.+SOURCES_PER_STAGED_PASS = 4+@frozen(frozen=False)class PerSourceResult:@@ -178,6 +191,20 @@ def select_changed(return list(changed.items()), new_hashes+def _accumulate_bundles(+ accumulated: dict[str, dict[str, dict]], rows: list[dict], sources: list[PersonPropertySyncSource]+) -> None:+ """Fold one batch of rows into {source_id: {distinct_id: bundle}}, last write winning per distinct_id."""+ for source in sources:+ bucket = accumulated[str(source.source_id)]+ for distinct_id, bundle in build_bundles(rows, source.key_column, source.column_property_map or {}):+ bucket[distinct_id] = bundle+++def _wanted_columns(sources: list[PersonPropertySyncSource]) -> set[str]:+ return {column for source in sources for column in (source.key_column, *(source.column_property_map or {}).keys())}++# --- S3 / Kafka / personhog boundaries (mocked in tests) ---------------------------------@@ -189,9 +216,36 @@ def _snapshot_prefix(team_id: int, binding: WarehouseBinding, source_id: str) ->return snapshot_prefix(team_id, binding, source_id)-async def _read_staged_rows(team_id: int, binding: WarehouseBinding, job_id: str) -> list[dict]:+@frozen+class StagedBundleRead:+ """What one job's staged files yielded for an incremental sync."""++ bundles_by_source: dict[str, dict[str, dict]]+ rows_read: int+++def _accumulate_staged_file(+ s3_client: "S3FileSystem",+ path: str,+ sources: list[PersonPropertySyncSource],+ accumulated: dict[str, dict[str, dict]],+) -> int:+ """Stream one staged parquet file batch by batch into ``accumulated``. Returns the rows read."""+ rows_read = 0+ with s3_client.open(path, "rb", cache_type="none") as file:+ parquet = pq.ParquetFile(file, pre_buffer=False)+ # Staged chunks can differ in columns after schema drift, and requesting an absent column raises.+ columns = sorted(_wanted_columns(sources) & set(parquet.schema_arrow.names))+ for batch in parquet.iter_batches(batch_size=ROW_BATCH_SIZE, columns=columns):+ rows = batch.to_pylist()+ rows_read += len(rows)+ _accumulate_bundles(accumulated, rows, sources)+ return rows_read+++async def _list_staged_files(team_id: int, binding: WarehouseBinding, job_id: str) -> list[str]:+ """S3 keys of the job's staged files in write order. Empty when the run staged nothing."""prefix = job_staged_prefix(team_id, binding, job_id)- rows: list[dict] = []async with aget_s3_client() as s3_client:try:listing = await s3_client._ls(f"s3://{prefix}/", detail=True)@@ -199,24 +253,41 @@ async def _read_staged_rows(team_id: int, binding: WarehouseBinding, job_id: str# No staged folder — the run staged nothing (common on a no-change incremental run).logger.debug("person-property sync: no staged rows folder", team_id=team_id, **_log_fields(binding))return []- values = listing.values() if isinstance(listing, dict) else listing- files = [f["Key"] for f in values if f.get("type") != "directory"]- for file_path in files:- data = await s3_client._cat_file(f"s3://{file_path}" if not file_path.startswith("s3://") else file_path)- # Decode off the event loop: a large parquet chunk would otherwise block the async- # activity long enough to starve the heartbeater and trip its timeout.- rows.extend(await asyncio.to_thread(_decode_parquet_rows, data))- return rows-+ values = listing.values() if isinstance(listing, dict) else listing+ files = [f for f in values if f.get("type") != "directory"]+ # Staged filenames sort in attempt and chunk order, so last-write-wins follows write order.+ file_keys = sorted(f["Key"] for f in files)+ logger.info(+ "person-property sync: reading staged files",+ team_id=team_id,+ **_log_fields(binding),+ job_id=job_id,+ staged_files=len(file_keys),+ staged_bytes=sum(f.get("size") or 0 for f in files),+ )+ return file_keys-def _bytes_reader(data: bytes):- import io # noqa: PLC0415 — local to keep the module import light- return io.BytesIO(data)+async def _read_staged_bundles(file_keys: list[str], sources: list[PersonPropertySyncSource]) -> StagedBundleRead:+ """Stream the staged files into {source_id: {distinct_id: bundle}}, one file and one batch at a+ time, so peak memory tracks distinct persons rather than staged row count."""+ accumulated: dict[str, dict[str, dict]] = {str(source.source_id): {} for source in sources}+ s3_sync_client = get_s3_client()+ rows_read = 0+ for key in file_keys:+ # Read off the event loop so a large file can't starve the heartbeater.+ rows_read += await asyncio.to_thread(+ _accumulate_staged_file, s3_sync_client, _s3_uri(key), sources, accumulated+ )+ return StagedBundleRead(bundles_by_source=accumulated, rows_read=rows_read)-def _decode_parquet_rows(data: bytes) -> list[dict]:- return pq.read_table(_bytes_reader(data)).to_pylist()+def _merge_snapshot_file(data: bytes, hashes: dict[str, str]) -> None:+ parquet = pq.ParquetFile(io.BytesIO(data))+ for batch in parquet.iter_batches(batch_size=ROW_BATCH_SIZE, columns=["distinct_id", "sent_hash"]):+ distinct_ids = batch.column("distinct_id").to_pylist()+ sent_hashes = batch.column("sent_hash").to_pylist()+ hashes.update((str(distinct_id), str(sent_hash)) for distinct_id, sent_hash in zip(distinct_ids, sent_hashes))def _snapshot_file_order(entry: dict) -> tuple:@@ -261,8 +332,7 @@ async def _merge_snapshot_files(s3_client, file_keys: list[str]) -> dict[str, st# after we listed the folder but before we read it. Its rows survive in that new file (or# whichever one replaces it next run) — skip rather than fail the whole read.continue- for record in await asyncio.to_thread(_decode_parquet_rows, data):- hashes[record["distinct_id"]] = record["sent_hash"]+ await asyncio.to_thread(_merge_snapshot_file, data, hashes)return hashes@@ -518,6 +588,30 @@ async def _process_source_bundles(return ps+async def _process_accumulated_source(+ *,+ team_id: int,+ team: Team,+ binding: WarehouseBinding,+ source: PersonPropertySyncSource,+ bundles_by_source: dict[str, dict[str, dict]],+ rows_read: int,+ run_token: str,+) -> PerSourceResult:+ bundles = list(bundles_by_source.pop(str(source.source_id)).items())+ return await _process_source_bundles(+ team_id=team_id,+ project_id=team.project_id,+ binding=binding,+ team_api_token=team.api_token,+ team_uuid=str(team.uuid),+ source=source,+ bundles=bundles,+ rows_read=rows_read,+ run_token=run_token,+ )++def _accumulate(result: SyncResult, ps: PerSourceResult) -> None:result.per_source.append(ps)result.changed += ps.changed@@ -539,32 +633,32 @@ async def run_person_property_sync(*, team_id: int, binding: WarehouseBinding, jreturn resultteam = await database_sync_to_async(Team.objects.get, thread_sensitive=False)(id=team_id)- rows = await _read_staged_rows(team_id, binding, job_id)+ file_keys = await _list_staged_files(team_id, binding, job_id)result.sources = len(sources)- result.rows_read = len(rows)- logger.info(- "person-property sync: read staged rows",- team_id=team_id,- **_log_fields(binding),- job_id=job_id,- sources=len(sources),- rows_read=len(rows),- )- for source in sources:- bundles = await asyncio.to_thread(build_bundles, rows, source.key_column, source.column_property_map or {})- ps = await _process_source_bundles(+ for start in range(0, len(sources), SOURCES_PER_STAGED_PASS):+ pass_sources = sources[start : start + SOURCES_PER_STAGED_PASS]+ read = await _read_staged_bundles(file_keys, pass_sources)+ result.rows_read = read.rows_read+ logger.info(+ "person-property sync: read staged rows",team_id=team_id,- project_id=team.project_id,- binding=binding,- team_api_token=team.api_token,- team_uuid=str(team.uuid),- source=source,- bundles=bundles,- rows_read=len(rows),- run_token=job_id,+ **_log_fields(binding),+ job_id=job_id,+ sources=len(pass_sources),+ rows_read=read.rows_read,)- _accumulate(result, ps)+ for source in pass_sources:+ ps = await _process_accumulated_source(+ team_id=team_id,+ team=team,+ binding=binding,+ source=source,+ bundles_by_source=read.bundles_by_source,+ rows_read=read.rows_read,+ run_token=job_id,+ )+ _accumulate(result, ps)await _clear_staged(team_id, binding, job_id)return result@@ -572,10 +666,6 @@ async def run_person_property_sync(*, team_id: int, binding: WarehouseBinding, j# --- backfill (full Delta read) -----------------------------------------------------------# Rows per streamed Delta batch. Batches are processed then discarded, so peak memory is one batch-# plus the per-source {distinct_id: bundle} accumulator (bounded by distinct-person cardinality).-BACKFILL_BATCH_SIZE = 50_000-# Backfill runs share one snapshot filename per source (a full-table read replaces the whole set),# distinct from the incremental job-id filenames, so the two writers never collide.BACKFILL_RUN_TOKEN = "backfill"@@ -608,11 +698,8 @@ def _read_delta_bundles(dataset = deltalake.DeltaTable(uri, storage_options=storage_options).to_pyarrow_dataset()available = set(dataset.schema.names)- wanted = {- column for source in sources for column in (source.key_column, *(source.column_property_map or {}).keys())- }# Project only columns that exist — a misconfigured column drops out rather than erroring.- project = sorted(wanted & available)+ project = sorted(_wanted_columns(sources) & available)# A source whose key column itself is missing produces zero bundles for the whole table, which is# otherwise indistinguishable from a genuinely idle backfill. Report it back so the caller can fail@@ -629,13 +716,10 @@ def _read_delta_bundles()rows_read = 0- for batch in dataset.to_batches(columns=project, batch_size=BACKFILL_BATCH_SIZE):+ for batch in dataset.to_batches(columns=project, batch_size=ROW_BATCH_SIZE):rows = batch.to_pylist()rows_read += len(rows)- for source in sources:- bucket = accumulated[str(source.source_id)]- for distinct_id, bundle in build_bundles(rows, source.key_column, source.column_property_map or {}):- bucket[distinct_id] = bundle+ _accumulate_bundles(accumulated, rows, sources)return DeltaBundleRead(bundles_by_source=accumulated,rows_read=rows_read,@@ -726,15 +810,12 @@ async def run_person_property_backfill(*, team_id: int, binding: WarehouseBindin),)continue- bundles = list(read.bundles_by_source[str(source.source_id)].items())- ps = await _process_source_bundles(+ ps = await _process_accumulated_source(team_id=team_id,- project_id=team.project_id,+ team=team,binding=binding,- team_api_token=team.api_token,- team_uuid=str(team.uuid),source=source,- bundles=bundles,+ bundles_by_source=read.bundles_by_source,rows_read=read.rows_read,run_token=BACKFILL_RUN_TOKEN,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync_test.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync_test.pyindex 2a3b4fa1c6aa..f15105acbfda 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync_test.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync_test.py@@ -1,3 +1,4 @@+import ioimport jsonimport threadingfrom datetime import date, datetime@@ -11,6 +12,8 @@ from django.db.models import Qfrom django.test import TransactionTestCasefrom django.test.utils import CaptureQueriesContext+import pyarrow as pa+import pyarrow.parquet as pqfrom parameterized import parameterizedfrom posthog.models import Organization, PropertyDefinition, Team@@ -156,10 +159,12 @@ class TestRunOrchestration:order.append("produce")return 2+ fake = _FakeS3()+ _stage(fake, "chunk_1_000000", rows)with (+ _fake_s3_patch(fake),patch(f"{_MODULE}.person_property_sync_sources_for", return_value=[self._source()]),patch(f"{_MODULE}.Team") as team_cls,- patch(f"{_MODULE}._read_staged_rows", new=AsyncMock(return_value=rows)),patch(f"{_MODULE}._read_snapshot_hashes", new=AsyncMock(return_value={})),patch(f"{_MODULE}._filter_existing_ids", return_value={"a", "b"}) as existing,patch(f"{_MODULE}._produce_intents", side_effect=_record_produce) as produce,@@ -197,11 +202,55 @@ class TestRunOrchestration:clear.assert_awaited_once()assert order == ["reconcile", "produce", "snapshot"]+ @parameterized.expand([("one_pass", 4), ("pass_per_source", 1)])+ @pytest.mark.asyncio+ async def test_streams_staged_files_into_one_projected_bundle_per_distinct_id(self, _name, sources_per_pass):+ sources = [+ PersonPropertySyncSource(+ source_id="source-1",+ definition_id="def-1",+ key_column="distinct_id",+ column_property_map={"plan": "plan_tier", "seats": "seat_count"},+ ),+ PersonPropertySyncSource(+ source_id="source-2",+ definition_id="def-2",+ key_column="distinct_id",+ column_property_map={"seats": "seat_count"},+ ),+ ]+ fake = _FakeS3()+ _stage(fake, "chunk_1_000001", [{"distinct_id": "a", "plan": "pro", "seats": 3, "notes": "unmapped"}])+ _stage(fake, "chunk_1_000000", [{"distinct_id": "a", "plan": "free"}, {"distinct_id": "b", "plan": "pro"}])+ with (+ _fake_s3_patch(fake),+ patch(f"{_MODULE}.SOURCES_PER_STAGED_PASS", sources_per_pass),+ patch(f"{_MODULE}.person_property_sync_sources_for", return_value=sources),+ patch(f"{_MODULE}.Team") as team_cls,+ patch(f"{_MODULE}.build_bundles", wraps=pps.build_bundles) as build,+ patch(f"{_MODULE}._read_snapshot_hashes", new=AsyncMock(return_value={})),+ patch(f"{_MODULE}._filter_existing_ids", side_effect=lambda _team, _source, ids: set(ids)),+ patch(f"{_MODULE}._produce_intents", return_value=2) as produce,+ patch(f"{_MODULE}._write_snapshot_hashes", new=AsyncMock()),+ patch(f"{_MODULE}._reconcile_property_definitions"),+ patch(f"{_MODULE}._clear_staged", new=AsyncMock()),+ ):+ team_cls.objects.get.return_value = MagicMock(api_token="tok", project_id=7)+ result = await pps.run_person_property_sync(team_id=1, binding=_SCHEMA, job_id="job-1")++ produced = {call.args[2].source_id: sorted(call.args[3]) for call in produce.call_args_list}+ assert produced == {+ "source-1": [("a", {"plan_tier": "pro", "seat_count": 3}), ("b", {"plan_tier": "pro"})],+ "source-2": [("a", {"seat_count": 3})],+ }+ assert result.rows_read == 3+ assert all("notes" not in row for call in build.call_args_list for row in call.args[0])+@pytest.mark.asyncioasync def test_no_sources_is_a_noop(self):with (patch(f"{_MODULE}.person_property_sync_sources_for", return_value=None),- patch(f"{_MODULE}._read_staged_rows", new=AsyncMock()) as read,+ patch(f"{_MODULE}._read_staged_bundles", new=AsyncMock()) as read,patch(f"{_MODULE}._clear_staged", new=AsyncMock()) as clear,):result = await pps.run_person_property_sync(team_id=1, binding=_SCHEMA, job_id="job-1")@@ -464,7 +513,7 @@ class _FakeS3:async def _ls(self, path, detail=True):prefix = pps._s3_key(path).rstrip("/") + "/"entries = [- {"Key": key, "type": "file", "LastModified": self.times[key]}+ {"Key": key, "type": "file", "LastModified": self.times[key], "size": len(self.store[key])}for key in self.storeif key.startswith(prefix)]@@ -478,6 +527,12 @@ class _FakeS3:raise FileNotFoundError(path)return self.store[key]+ def open(self, path, mode="rb", **_kwargs):+ key = pps._s3_key(path)+ if key not in self.store:+ raise FileNotFoundError(path)+ return io.BytesIO(self.store[key])+async def _pipe_file(self, path, data):self._clock += 1key = pps._s3_key(path)@@ -498,7 +553,18 @@ def _fake_s3_patch(fake):async def _cm():yield fake- return patch(f"{_MODULE}.aget_s3_client", lambda: _cm())+ stack = contextlib.ExitStack()+ stack.enter_context(patch(f"{_MODULE}.aget_s3_client", lambda: _cm()))+ stack.enter_context(patch(f"{_MODULE}.get_s3_client", lambda: fake))+ return stack+++def _stage(fake, name, rows, job_id="job-1"):+ buffer = io.BytesIO()+ pq.write_table(pa.Table.from_pylist(rows), buffer)+ key = f"{pps._s3_key(pps.job_staged_prefix(1, _SCHEMA, job_id))}/{name}.parquet"+ fake.store[key] = buffer.getvalue()+ fake.times[key] = 0class TestSnapshotCompaction:@@ -591,11 +657,12 @@ class TestGroupTarget:# If the group type can't be resolved (deleted/misconfigured), the consumer would DLQ every# $groupidentify missing a group_type — so the source must be skipped, not produced.team = MagicMock(api_token="tok", uuid="team-uuid")- rows = [{"group_key": "acme", "plan": "pro"}]+ fake = _FakeS3()+ _stage(fake, "chunk_1_000000", [{"group_key": "acme", "plan": "pro"}])with (+ _fake_s3_patch(fake),patch(f"{_MODULE}.person_property_sync_sources_for", return_value=[self._group_source()]),patch(f"{_MODULE}.Team") as team_cls,- patch(f"{_MODULE}._read_staged_rows", new=AsyncMock(return_value=rows)),patch(f"{_MODULE}._read_snapshot_hashes", new=AsyncMock(return_value={})),patch(f"{_MODULE}._filter_existing_ids", return_value={"acme"}),patch(f"{_MODULE}._group_type_name", return_value=None),@@ -618,12 +685,13 @@ class TestGroupTarget:# a child environment has no rows under its own team id. Resolving by team id found nothing# there, which skipped every group source on that team and produced no intents at all.team = MagicMock(api_token="tok", uuid="team-uuid", project_id=77)- rows = [{"group_key": "acme", "plan": "pro"}]by_project = {77: [{"group_type_index": 0, "group_type": "organization"}]}+ fake = _FakeS3()+ _stage(fake, "chunk_1_000000", [{"group_key": "acme", "plan": "pro"}])with (+ _fake_s3_patch(fake),patch(f"{_MODULE}.person_property_sync_sources_for", return_value=[self._group_source()]),patch(f"{_MODULE}.Team") as team_cls,- patch(f"{_MODULE}._read_staged_rows", new=AsyncMock(return_value=rows)),patch(f"{_MODULE}._read_snapshot_hashes", new=AsyncMock(return_value={})),patch(f"{_MODULE}._filter_existing_ids", return_value={"acme"}),patch(