:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse-sources): resume person property syncs from the last delivered chunk (#122054)
Jovan Sakovic7ccab32af56f
diff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job.pyindex b64c8d857116..92f82e730142 100644--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job.py+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job.py@@ -45,7 +45,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.perlogger = structlog.get_logger(__name__)-INTERRUPTED_RUN_NEXT_STEP = "Click Backfill to try again."+INTERRUPTED_RUN_NEXT_STEP = "Click Backfill to continue from where it stopped."RECORD_FAILED_RUN_PATCH = "person-property-backfill-record-failed-run-2026-10"RETRY_INTERRUPTED_RUN_PATCH = "person-property-backfill-retry-interrupted-run-2026-10"diff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job.pyindex b23742187edf..8bd7158a9047 100644--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job.py+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job.py@@ -7,10 +7,10 @@ competes with the sync workers themselves; ``start_temporal_worker`` registers i``PERSON_PROPERTY_SYNC_WORKFLOWS``/``PERSON_PROPERTY_SYNC_ACTIVITIES``.Failure semantics: the activity retries up to 3 times, then the (abandoned) child workflow fails-without affecting the sync. Nothing is lost — staged files are only cleared on success and the-dedup snapshot only advances per produced person, so the next sync's run re-reads and re-diffs.-Each failed attempt is captured to error tracking. The workflow then records the run as failed and-counts it once in the metrics below, also when the activity was timed out, cancelled or killed.+without affecting the sync. The dedup snapshot only advances per produced person, so a Backfill can+recover rows left by an interrupted incremental run without resending confirmed updates. Each failed+attempt is captured to error tracking. The workflow then records the run as failed and counts it once+in the metrics below, also when the activity was timed out, cancelled or killed."""import json@@ -45,7 +45,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.perlogger = structlog.get_logger(__name__)-INTERRUPTED_RUN_NEXT_STEP = "Check the source status, then click Sync now to try again."+INTERRUPTED_RUN_NEXT_STEP = "Run a Backfill to deliver any rows left by the interrupted sync."# Three recorder attempts of up to 5 minutes each.RECORD_FAILED_RUN_RESERVE = timedelta(minutes=15)RECORD_FAILED_RUN_PATCH = "person-property-sync-record-failed-run-2026-10"diff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job_test.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job_test.pyindex ff16d355c333..f89d74052779 100644--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job_test.py+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job_test.py@@ -91,7 +91,7 @@ async def _backfill_activity_killed_by_worker_shutdown(inputs: PersonPropertyBac@pytest.mark.parametrize(- "workflow_cls,replaced_activity,killed_activity,real_activities,inputs,expected_job_id",+ "workflow_cls,replaced_activity,killed_activity,real_activities,inputs,expected_job_id,expected_next_step",[(SyncWarehousePersonPropertiesWorkflow,@@ -109,6 +109,7 @@ async def _backfill_activity_killed_by_worker_shutdown(inputs: PersonPropertyBacsaved_query_id=uuid.UUID("00000000-0000-4000-8000-000000000001"),),"job-1",+ "Run a Backfill to deliver any rows left by the interrupted sync.",),(BackfillWarehousePersonPropertiesWorkflow,@@ -124,11 +125,12 @@ async def _backfill_activity_killed_by_worker_shutdown(inputs: PersonPropertyBacsaved_query_id=uuid.UUID("00000000-0000-4000-8000-000000000001"),),None,+ "Click Backfill to continue from where it stopped.",),],)async def test_run_is_marked_failed_when_the_activity_dies_without_its_own_handler(- workflow_cls, replaced_activity, killed_activity, real_activities, inputs, expected_job_id+ workflow_cls, replaced_activity, killed_activity, real_activities, inputs, expected_job_id, expected_next_step) -> None:recorded: list[PersonPropertySyncRunRecord] = []source = PersonPropertySyncSource(@@ -162,6 +164,7 @@ async def test_run_is_marked_failed_when_the_activity_dies_without_its_own_handlassert failed[0].source_id == source.source_idassert failed[0].job_id == expected_job_idassert (failed[0].error or "").startswith("This run was interrupted before it finished.")+ assert (failed[0].error or "").endswith(expected_next_step)assert failed[0].finished_atassert failed[0].count_failure is Falsediff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.pyindex 14eef7fddf87..a35fff71044e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py@@ -24,6 +24,7 @@ S3/personhog/Kafka helpers are the boundaries and are mocked in the orchestratioimport ioimport json+import uuidimport asyncioimport hashlibimport dataclasses@@ -35,10 +36,12 @@ from django.conf import settingsimport structlogimport pyarrow.parquet as pq+from confluent_kafka import KafkaExceptionfrom temporalio.exceptions import ActivityError, ApplicationError, CancelledErrorfrom posthog.dataclasses import frozenfrom posthog.exceptions_capture import capture_exception+from posthog.kafka_client.client import ProduceResultfrom posthog.kafka_client.routing import producer_scopefrom posthog.kafka_client.topics import KAFKA_WAREHOUSE_PERSON_PROPERTY_UPDATESfrom posthog.models import Team@@ -97,6 +100,15 @@ ROW_BATCH_SIZE = 50_000# for every person at once. Each extra pass reads only its own sources' columns.SOURCES_PER_STAGED_PASS = 4+# Changed entities per produce-and-snapshot step. Each step writes the hashes Kafka confirmed to its own+# snapshot file before the next step starts, so a run that is stopped part way (a deploy stops the+# worker) keeps what it delivered, and the next run's diff resumes from there instead of from zero.+_PRODUCE_CHUNK_SIZE = 50_000++# A concurrent compaction writes its merged file before it deletes the old ones, so a fresh listing+# after a vanished file almost always reads complete. Giving up means re-sending every row.+_SNAPSHOT_READ_ATTEMPTS = 3+@frozen(frozen=False)class PerSourceResult:@@ -108,11 +120,24 @@ class PerSourceResult:existing: int = 0produced: int = 0skipped_missing_person: int = 0+ # Intents Kafka did not confirm. They are left out of the snapshot so a retry sends them again.+ undelivered: int = 0# Set when the source can't write anything for a reason its counts alone don't explain, so the# recorder persists a failed run instead of a clean one that produced nothing.error: str | None = None+class PropertyUpdatesNotDeliveredError(Exception):+ """Kafka did not confirm some property-update intents. Raised so the run fails and Temporal retries+ it. The snapshot holds only confirmed intents, so the retry sends only the undelivered ones."""+++@frozen+class _SnapshotMerge:+ hashes: dict[str, str]+ complete: bool++@dataclasses.dataclassclass SyncResult:sources: int = 0@@ -209,8 +234,8 @@ def _wanted_columns(sources: list[PersonPropertySyncSource]) -> set[str]:def _snapshot_prefix(team_id: int, binding: WarehouseBinding, source_id: str) -> str:- # A folder, not a single file: each run drops a uniquely-named parquet of the hashes it produced,- # and the reader unions every file in the folder. Two writers (a scheduled sync and a backfill)+ # A folder, not a single file: each run drops uniquely-named parquet files of the hashes it+ # produced (one per chunk, then one compacted file), and the reader unions every file in the folder. Two writers (a scheduled sync and a backfill)# for the same source can't clobber each other, since they write different filenames. Growth is# bounded by _write_snapshot_hashes, which compacts the folder back down as it writes.return snapshot_prefix(team_id, binding, source_id)@@ -320,10 +345,11 @@ async def _list_snapshot_files(s3_client, prefix: str) -> list[str]:return [f["Key"] for f in files]-async def _merge_snapshot_files(s3_client, file_keys: list[str]) -> dict[str, str]:+async def _merge_snapshot_files(s3_client, file_keys: list[str]) -> _SnapshotMerge:"""Union the given snapshot files into {distinct_id: sent_hash}, later files winning (callers passthem oldest-first). Decodes off the event loop so a large parquet can't starve the heartbeater."""hashes: dict[str, str] = {}+ complete = Truefor key in file_keys:try:data = await s3_client._cat_file(_s3_uri(key))@@ -331,9 +357,10 @@ async def _merge_snapshot_files(s3_client, file_keys: list[str]) -> dict[str, st# A concurrent _write_snapshot_hashes compacted this file into a new one and deleted it# after we listed the folder but before we read it. Its rows survive in that new file (or# whichever one replaces it next run) — skip rather than fail the whole read.+ complete = Falsecontinueawait asyncio.to_thread(_merge_snapshot_file, data, hashes)- return hashes+ return _SnapshotMerge(hashes=hashes, complete=complete)async def _read_snapshot_hashes(team_id: int, binding: WarehouseBinding, source_id: str) -> dict[str, str]:@@ -342,7 +369,31 @@ async def _read_snapshot_hashes(team_id: int, binding: WarehouseBinding, source_cost an idempotent re-send, but newest-wins avoids even that). Ordering never affects correctness."""prefix = _snapshot_prefix(team_id, binding, source_id)async with aget_s3_client() as s3_client:- return await _merge_snapshot_files(s3_client, await _list_snapshot_files(s3_client, prefix))+ for _ in range(_SNAPSHOT_READ_ATTEMPTS):+ merge = await _merge_snapshot_files(s3_client, await _list_snapshot_files(s3_client, prefix))+ if merge.complete:+ return merge.hashes+ return {}+++async def _put_snapshot_file(s3_client, path: str, hashes: dict[str, str]) -> None:+ import pyarrow as pa # noqa: PLC0415++ table = pa.table({"distinct_id": list(hashes.keys()), "sent_hash": list(hashes.values())})+ buffer = pa.BufferOutputStream()+ pq.write_table(table, buffer, compression="zstd")+ await s3_client._pipe_file(_s3_uri(path), buffer.getvalue().to_pybytes())+++async def _write_snapshot_chunk(+ team_id: int, binding: WarehouseBinding, source_id: str, file_name: str, hashes: dict[str, str]+) -> None:+ """Add ``hashes`` to the source's snapshot folder as one new file, without reading or merging the+ folder. The reader unions this file with the rest, and ``_write_snapshot_hashes`` folds it in when+ the run compacts. Merging here instead would re-read the whole folder once per chunk."""+ path = f"{_snapshot_prefix(team_id, binding, source_id)}/{file_name}.parquet"+ async with aget_s3_client() as s3_client:+ await _put_snapshot_file(s3_client, path, hashes)async def _write_snapshot_hashes(@@ -358,19 +409,19 @@ async def _write_snapshot_hashes(listing isn't in the merge set, so it's neither merged-away nor deleted, and its rows still surfacethrough the reader's union; the two files collapse on the next run. Write-then-delete keeps a crashbetween the two harmless — the reader unions both the new file and the not-yet-deleted old ones."""- import pyarrow as pa # noqa: PLC0415-prefix = _snapshot_prefix(team_id, binding, source_id)write_path = f"{prefix}/{run_token}.parquet"async with aget_s3_client() as s3_client:existing_keys = await _list_snapshot_files(s3_client, prefix)- merged = await _merge_snapshot_files(s3_client, existing_keys)- merged.update(hashes)-- table = pa.table({"distinct_id": list(merged.keys()), "sent_hash": list(merged.values())})- buffer = pa.BufferOutputStream()- pq.write_table(table, buffer, compression="zstd")- await s3_client._pipe_file(_s3_uri(write_path), buffer.getvalue().to_pybytes())+ if not hashes and len(existing_keys) <= 1:+ return+ merge = await _merge_snapshot_files(s3_client, existing_keys)+ if not hashes and not merge.complete:+ return+ merged = {**merge.hashes, **hashes} if merge.complete else hashes+ await _put_snapshot_file(s3_client, write_path, merged)+ if not merge.complete:+ return# Delete the files we merged, except the one we just wrote (the backfill token reuses its name).stale = [key for key in existing_keys if _s3_key(key) != _s3_key(write_path)]@@ -444,12 +495,13 @@ def _produce_intents(*,team_uuid: str,group_type_name: str | None,-) -> int:+) -> set[str]:"""Produce one property-update intent per entity to Kafka. Person intents map to a `$set` keyedby team:distinct_id; group intents map to a `$groupidentify` keyed by team:group_type:group_key so- per-group ordering is preserved. The consumer branches on ``kind``. Returns the count produced."""+ per-group ordering is preserved. The consumer branches on ``kind``. Returns the keys Kafka+ confirmed as delivered."""is_group = source.target == _GROUP_TARGET- produced = 0+ results: list[tuple[str, ProduceResult]] = []with producer_scope(topic=KAFKA_WAREHOUSE_PERSON_PROPERTY_UPDATES) as producer:for key, bundle in items:if is_group:@@ -476,9 +528,19 @@ def _produce_intents("event_source": EVENT_SOURCE,}kafka_key = f"{team_id}:{key}"- producer.produce(topic=KAFKA_WAREHOUSE_PERSON_PROPERTY_UPDATES, data=data, key=kafka_key)- produced += 1- return produced+ results.append(+ (key, producer.produce(topic=KAFKA_WAREHOUSE_PERSON_PROPERTY_UPDATES, data=data, key=kafka_key))+ )+ # The scope flushes on exit. flush() counts only messages still queued, and a failed delivery+ # leaves the queue, so only each message's result shows whether Kafka accepted it.+ delivered: set[str] = set()+ for key, result in results:+ try:+ result.get(timeout=0)+ except (KafkaException, TimeoutError):+ continue+ delivered.add(key)+ return delivereddef _reconcile_property_definitions(@@ -517,7 +579,9 @@ async def _process_source_bundles(run_token: str,) -> PerSourceResult:"""Diff one source's bundles against its snapshot, drop entities that don't exist, produce update- intents (person `$set` or group `$groupidentify`), stamp provenance, and advance the snapshot.+ intents (person `$set` or group `$groupidentify`), stamp provenance, and advance the snapshot. Works+ in chunks of ``_PRODUCE_CHUNK_SIZE`` and snapshots each chunk's confirmed deliveries before the next+ chunk, so a run stopped part way loses at most one chunk of progress.Shared by the incremental sync and the backfill — they differ only in how ``bundles`` are sourced(staged rows vs a full Delta read)."""ps = PerSourceResult(source_id=str(source.source_id), rows_read=rows_read)@@ -530,23 +594,7 @@ async def _process_source_bundles(changed, new_hashes = await asyncio.to_thread(select_changed, bundles, prior)ps.changed = len(changed)if not changed:- return ps-- existing = await database_sync_to_async(_filter_existing_ids, thread_sensitive=False)(- team_id, source, [key for key, _ in changed]- )- to_send = [(key, bundle) for key, bundle in changed if key in existing]- ps.existing = len(to_send)- ps.skipped_missing_person = len(changed) - len(to_send)- if not to_send:- logger.info(- "person-property sync: no existing entities among changed rows for source",- team_id=team_id,- **_log_fields(binding),- source_id=str(source.source_id),- target=source.target,- changed=len(changed),- )+ await _write_snapshot_hashes(team_id, binding, str(source.source_id), run_token, {})return psgroup_type_name = None@@ -565,15 +613,62 @@ async def _process_source_bundles(group_type_index=source.group_type_index,)return ps- produced = await asyncio.to_thread(- _produce_intents, team_id, team_api_token, source, to_send, team_uuid=team_uuid, group_type_name=group_type_name- )- ps.produced = produced- # Record only the distinct_ids we actually produced, as this run's snapshot file.- sent_ids = {distinct_id for distinct_id, _ in to_send}- produced_hashes = {d: h for d, h in new_hashes.items() if d in sent_ids}- await _write_snapshot_hashes(team_id, binding, str(source.source_id), run_token, produced_hashes)+ # Snapshot chunk files get a name unique to this attempt. A Temporal retry reuses ``run_token``, and+ # its chunk numbering starts again at zero, so a name built from the token and index alone would+ # overwrite the files that hold the previous attempt's delivered hashes.+ attempt = uuid.uuid4().hex[:12]+ chunks_written = 0+ confirmed_hashes: dict[str, str] = {}+ for start in range(0, len(changed), _PRODUCE_CHUNK_SIZE):+ chunk = changed[start : start + _PRODUCE_CHUNK_SIZE]+ existing = await database_sync_to_async(_filter_existing_ids, thread_sensitive=False)(+ team_id, source, [key for key, _ in chunk]+ )+ to_send = [(key, bundle) for key, bundle in chunk if key in existing]+ ps.existing += len(to_send)+ ps.skipped_missing_person += len(chunk) - len(to_send)+ if not to_send:+ continue++ delivered = await asyncio.to_thread(+ _produce_intents,+ team_id,+ team_api_token,+ source,+ to_send,+ team_uuid=team_uuid,+ group_type_name=group_type_name,+ )+ ps.produced += len(delivered)+ ps.undelivered += len(to_send) - len(delivered)++ # Write only after Kafka confirms delivery. A hash in the snapshot means "already sent", so a+ # hash written before delivery would make a lost intent look sent and the next run skip it.+ # An undelivered id stays out, so the next diff sees it as changed and sends it again.+ delivered_hashes = {key: new_hashes[key] for key in delivered}+ if delivered_hashes:+ await _write_snapshot_chunk(+ team_id, binding, str(source.source_id), f"{run_token}-{attempt}-{chunks_written:05d}", delivered_hashes+ )+ confirmed_hashes.update(delivered_hashes)+ chunks_written += 1++ if chunks_written:+ # Fold this run's chunk files, and any left by an earlier run that was stopped part way, into+ # one ``{run_token}.parquet`` so the folder stays at about one file per source.+ await _write_snapshot_hashes(team_id, binding, str(source.source_id), run_token, confirmed_hashes)++ if not ps.existing:+ logger.info(+ "person-property sync: no existing entities among changed rows for source",+ team_id=team_id,+ **_log_fields(binding),+ source_id=str(source.source_id),+ target=source.target,+ changed=len(changed),+ )+ return pslogger.info("person-property sync: source processed",@@ -582,8 +677,9 @@ async def _process_source_bundles(source_id=str(source.source_id),bundles=len(bundles),changed=len(changed),- existing=len(to_send),- produced=produced,+ existing=ps.existing,+ produced=ps.produced,+ undelivered=ps.undelivered,)return ps@@ -620,6 +716,15 @@ def _accumulate(result: SyncResult, ps: PerSourceResult) -> None:result.skipped_missing_person += ps.skipped_missing_person+def _raise_if_undelivered(result: SyncResult) -> None:+ undelivered = sum(ps.undelivered for ps in result.per_source)+ if undelivered:+ raise PropertyUpdatesNotDeliveredError(+ f"Kafka did not confirm {undelivered} of {undelivered + result.produced} property updates. "+ "A retry sends the undelivered updates again."+ )++async def run_person_property_sync(*, team_id: int, binding: WarehouseBinding, job_id: str) -> SyncResult:result = SyncResult()sources = await database_sync_to_async(person_property_sync_sources_for, thread_sensitive=False)(team_id, binding)@@ -660,6 +765,8 @@ async def run_person_property_sync(*, team_id: int, binding: WarehouseBinding, j)_accumulate(result, ps)+ # Raise before the staged rows are cleared, so the retry can read them again.+ _raise_if_undelivered(result)await _clear_staged(team_id, binding, job_id)return result@@ -821,6 +928,7 @@ async def run_person_property_backfill(*, team_id: int, binding: WarehouseBindin)_accumulate(result, ps)+ _raise_if_undelivered(result)return resultdiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync_test.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync_test.pyindex f15105acbfda..18e3271c3e82 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync_test.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync_test.py@@ -14,8 +14,10 @@ from django.test.utils import CaptureQueriesContextimport pyarrow as paimport pyarrow.parquet as pq+from confluent_kafka import KafkaErrorfrom parameterized import parameterized+from posthog.kafka_client.client import ProduceResultfrom posthog.models import Organization, PropertyDefinition, Teamfrom products.warehouse_sources.backend.temporal.data_imports.external_product_hooks import (@@ -155,9 +157,9 @@ class TestRunOrchestration:{"distinct_id": "ghost", "plan": "x"},]- def _record_produce(*_args: object, **_kwargs: object) -> int:+ def _record_produce(*_args: object, **_kwargs: object) -> set[str]:order.append("produce")- return 2+ return {"a", "b"}fake = _FakeS3()_stage(fake, "chunk_1_000000", rows)@@ -169,9 +171,13 @@ class TestRunOrchestration:patch(f"{_MODULE}._filter_existing_ids", return_value={"a", "b"}) as existing,patch(f"{_MODULE}._produce_intents", side_effect=_record_produce) as produce,patch(- f"{_MODULE}._write_snapshot_hashes",+ f"{_MODULE}._write_snapshot_chunk",new=AsyncMock(side_effect=lambda *_args, **_kwargs: order.append("snapshot")),- ) as write_snapshot,+ ) as write_chunk,+ patch(+ f"{_MODULE}._write_snapshot_hashes",+ new=AsyncMock(side_effect=lambda *_args, **_kwargs: order.append("compact")),+ ) as compact,patch(f"{_MODULE}._reconcile_property_definitions",side_effect=lambda *_args, **_kwargs: order.append("reconcile"),@@ -191,16 +197,85 @@ class TestRunOrchestration:assert result.per_source[0].produced == 2assert result.per_source[0].existing == 2- # snapshot advanced only for produced ids (args: team, schema, source, run_token, hashes).- assert write_snapshot.await_args is not None- assert write_snapshot.await_args.args[3] == "job-1"- written = write_snapshot.await_args.args[4]- assert set(written) == {"a", "b"}+ # snapshot advanced only for produced ids (args: team, schema, source, file name, hashes).+ assert write_chunk.await_args is not None+ assert write_chunk.await_args.args[3].startswith("job-1-")+ assert set(write_chunk.await_args.args[4]) == {"a", "b"}+ assert compact.await_args is not None+ assert compact.await_args.args[3] == "job-1"+ assert set(compact.await_args.args[4]) == {"a", "b"}existing.assert_called_once()reconcile.assert_called_once()clear.assert_awaited_once()- assert order == ["reconcile", "produce", "snapshot"]+ assert order == ["reconcile", "produce", "snapshot", "compact"]++ @pytest.mark.asyncio+ async def test_undelivered_intents_stay_out_of_the_snapshot_and_fail_the_run(self):+ team = MagicMock(api_token="tok", project_id=7)+ fake = _FakeS3()+ _stage(fake, "chunk_1_000000", [{"distinct_id": "a", "plan": "pro"}, {"distinct_id": "b", "plan": "free"}])+ with (+ _fake_s3_patch(fake),+ patch(f"{_MODULE}.person_property_sync_sources_for", return_value=[self._source()]),+ patch(f"{_MODULE}.Team") as team_cls,+ patch(f"{_MODULE}._filter_existing_ids", return_value={"a", "b"}),+ patch(f"{_MODULE}._produce_intents", return_value={"a"}),+ patch(f"{_MODULE}._reconcile_property_definitions"),+ patch(f"{_MODULE}._clear_staged", new=AsyncMock()) as clear,+ ):+ team_cls.objects.get.return_value = team+ with pytest.raises(pps.PropertyUpdatesNotDeliveredError, match="1 of 2"):+ await pps.run_person_property_sync(team_id=1, binding=_SCHEMA, job_id="job-1")+ hashes = await pps._read_snapshot_hashes(1, _SCHEMA, "source-1")++ assert set(hashes) == {"a"}+ clear.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_a_killed_run_keeps_confirmed_chunks_but_a_later_job_cannot_resume_undelivered_rows(self):+ team = MagicMock(api_token="tok", project_id=7)+ fake = _FakeS3()+ _stage(fake, "chunk_1_000000", [{"distinct_id": f"p{n}", "plan": "pro"} for n in range(6)], job_id="job-1")+ _stage(fake, "chunk_1_000000", [{"distinct_id": "p6", "plan": "pro"}], job_id="job-2")+ sent_per_run: dict[str, list[str]] = {"job-1": [], "job-2": []}+ current_job = "job-1"+ kill_after = 2++ def _produce(_team_id, _token, _source, items, **_kwargs):+ # Run 1 is killed (a deploy stops the worker) once it has delivered `kill_after` intents.+ sent = sent_per_run[current_job]+ if current_job == "job-1" and len(sent) + len(items) > kill_after:+ raise RuntimeError("worker stopped")+ sent.extend(key for key, _ in items)+ return {key for key, _ in items}++ with (+ _fake_s3_patch(fake),+ patch(f"{_MODULE}._PRODUCE_CHUNK_SIZE", 2),+ patch(f"{_MODULE}.person_property_sync_sources_for", return_value=[self._source()]),+ patch(f"{_MODULE}.Team") as team_cls,+ patch(f"{_MODULE}._filter_existing_ids", side_effect=lambda _team_id, _source, ids: set(ids)),+ patch(f"{_MODULE}._produce_intents", side_effect=_produce),+ patch(f"{_MODULE}._reconcile_property_definitions"),+ ):+ team_cls.objects.get.return_value = team+ with pytest.raises(RuntimeError, match="worker stopped"):+ await pps.run_person_property_sync(team_id=1, binding=_SCHEMA, job_id="job-1")++ current_job = "job-2"+ result = await pps.run_person_property_sync(team_id=1, binding=_SCHEMA, job_id="job-2")++ prefix = pps._snapshot_prefix(1, _SCHEMA, "source-1")+ snapshot_files = await pps._list_snapshot_files(fake, prefix)+ hashes = await pps._read_snapshot_hashes(1, _SCHEMA, "source-1")++ assert sent_per_run["job-1"] == ["p0", "p1"]+ assert sent_per_run["job-2"] == ["p6"]+ assert result.produced == 1+ # The completed run folds the killed run's leftover file and its own chunk files into one.+ assert [pps._s3_key(key) for key in snapshot_files] == [f"{prefix}/job-2.parquet"]+ assert set(hashes) == {"p0", "p1", "p6"}@parameterized.expand([("one_pass", 4), ("pass_per_source", 1)])@pytest.mark.asyncio@@ -230,7 +305,11 @@ class TestRunOrchestration:patch(f"{_MODULE}.build_bundles", wraps=pps.build_bundles) as build,patch(f"{_MODULE}._read_snapshot_hashes", new=AsyncMock(return_value={})),patch(f"{_MODULE}._filter_existing_ids", side_effect=lambda _team, _source, ids: set(ids)),- patch(f"{_MODULE}._produce_intents", return_value=2) as produce,+ patch(+ f"{_MODULE}._produce_intents",+ side_effect=lambda _team_id, _token, _source, items, **_kwargs: {key for key, _ in items},+ ) as produce,+ patch(f"{_MODULE}._write_snapshot_chunk", new=AsyncMock()),patch(f"{_MODULE}._write_snapshot_hashes", new=AsyncMock()),patch(f"{_MODULE}._reconcile_property_definitions"),patch(f"{_MODULE}._clear_staged", new=AsyncMock()),@@ -268,6 +347,7 @@ class TestRunOrchestration:patch(f"{_MODULE}._reconcile_property_definitions") as reconcile,patch(f"{_MODULE}._filter_existing_ids") as existing,patch(f"{_MODULE}._produce_intents") as produce,+ patch(f"{_MODULE}._write_snapshot_hashes", new=AsyncMock()) as compact,):result = await pps._process_source_bundles(team_id=1,@@ -284,6 +364,7 @@ class TestRunOrchestration:reconcile.assert_called_once_with(1, 7, _SCHEMA, source, ["plan_tier"])existing.assert_not_called()produce.assert_not_called()+ compact.assert_awaited_once_with(1, _SCHEMA, "source-1", "job-1", {})assert result.changed == 0 and result.produced == 0@@ -379,7 +460,8 @@ class TestBackfillOrchestration:) as read_delta,patch(f"{_MODULE}._read_snapshot_hashes", new=AsyncMock(return_value={})),patch(f"{_MODULE}._filter_existing_ids", return_value={"a"}),- patch(f"{_MODULE}._produce_intents", return_value=1) as produce,+ patch(f"{_MODULE}._produce_intents", return_value={"a"}) as produce,+ patch(f"{_MODULE}._write_snapshot_chunk", new=AsyncMock()),patch(f"{_MODULE}._write_snapshot_hashes", new=AsyncMock()) as write_snapshot,patch(f"{_MODULE}._reconcile_property_definitions"),):@@ -392,7 +474,7 @@ class TestBackfillOrchestration:assert result.rows_read == 5assert result.produced == 2 # one produced per sourceassert sorted(ps.source_id for ps in result.per_source) == ["s1", "s2"]- # Both sources write a snapshot file under the shared backfill run token.+ # Both sources compact their snapshot into the shared backfill run token's file.assert write_snapshot.await_count == 2assert all(call.args[3] == pps.BACKFILL_RUN_TOKEN for call in write_snapshot.await_args_list)assert produce.call_count == 2@@ -491,6 +573,7 @@ class TestBackfillOrchestration:),) as read_delta,patch(f"{_MODULE}._read_snapshot_hashes", new=AsyncMock(return_value={})),+ patch(f"{_MODULE}._write_snapshot_hashes", new=AsyncMock()),patch(f"{_MODULE}._reconcile_property_definitions"),):team_cls.objects.get.return_value = team@@ -609,9 +692,72 @@ class TestSnapshotCompaction:keys = await pps._list_snapshot_files(fake, pps._snapshot_prefix(1, _SCHEMA, "src"))await fake._rm(keys)- hashes = await pps._merge_snapshot_files(fake, keys)+ merge = await pps._merge_snapshot_files(fake, keys)- assert hashes == {}+ assert merge.hashes == {}+ assert not merge.complete++ @pytest.mark.asyncio+ async def test_read_relists_after_concurrent_compaction_instead_of_resending_everything(self):+ fake = _FakeS3()+ prefix = pps._snapshot_prefix(1, _SCHEMA, "src")+ with _fake_s3_patch(fake):+ await pps._write_snapshot_chunk(1, _SCHEMA, "src", "job-1", {"a": "h1"})+ await pps._write_snapshot_chunk(1, _SCHEMA, "src", "job-2", {"b": "h2"})+ original_cat_file = fake._cat_file+ compacted = False++ async def _cat_with_one_concurrent_compaction(path):+ nonlocal compacted+ if not compacted:+ compacted = True+ await pps._put_snapshot_file(fake, f"{prefix}/concurrent.parquet", {"a": "h1", "b": "h2"})+ await fake._rm([f"{prefix}/job-1.parquet", f"{prefix}/job-2.parquet"])+ return await original_cat_file(path)++ with patch.object(fake, "_cat_file", _cat_with_one_concurrent_compaction):+ hashes = await pps._read_snapshot_hashes(1, _SCHEMA, "src")++ assert hashes == {"a": "h1", "b": "h2"}++ @pytest.mark.asyncio+ async def test_current_deliveries_win_when_snapshot_timestamps_tie(self):+ fake = _FakeS3()+ prefix = pps._snapshot_prefix(1, _SCHEMA, "src")+ with _fake_s3_patch(fake):+ await pps._write_snapshot_chunk(1, _SCHEMA, "src", "aaa-chunk", {"a": "new"})+ await pps._write_snapshot_chunk(1, _SCHEMA, "src", pps.BACKFILL_RUN_TOKEN, {"a": "old"})+ fake.times = dict.fromkeys(fake.times, 1)++ await pps._write_snapshot_hashes(1, _SCHEMA, "src", "job-1", {"a": "new"})+ hashes = await pps._read_snapshot_hashes(1, _SCHEMA, "src")++ assert hashes == {"a": "new"}+ assert list(fake.store) == [f"{prefix}/job-1.parquet"]++ @pytest.mark.asyncio+ async def test_incomplete_merge_does_not_publish_or_delete_partial_snapshot(self):+ fake = _FakeS3()+ prefix = pps._snapshot_prefix(1, _SCHEMA, "src")+ with _fake_s3_patch(fake):+ await pps._write_snapshot_chunk(1, _SCHEMA, "src", "job-1", {"a": "old"})+ await pps._write_snapshot_chunk(1, _SCHEMA, "src", "job-2", {"a": "new"})+ original_cat_file = fake._cat_file++ async def _cat_after_concurrent_compaction(path):+ data = await original_cat_file(path)+ if path.endswith("/job-1.parquet"):+ await pps._put_snapshot_file(fake, f"{prefix}/concurrent.parquet", {"a": "new"})+ await fake._rm([f"{prefix}/job-1.parquet", f"{prefix}/job-2.parquet"])+ return data++ with patch.object(fake, "_cat_file", _cat_after_concurrent_compaction):+ await pps._write_snapshot_hashes(1, _SCHEMA, "src", "job-3", {})++ hashes = await pps._read_snapshot_hashes(1, _SCHEMA, "src")++ assert hashes == {"a": "new"}+ assert list(fake.store) == [f"{prefix}/concurrent.parquet"]class TestGroupTarget:@@ -624,7 +770,14 @@ class TestGroupTarget:def test_produce_intents_group_payload(self):captured: list[dict] = []producer = MagicMock()- producer.produce.side_effect = lambda **kw: captured.append(kw)++ def _produce(**kw: object) -> ProduceResult:+ captured.append(kw)+ result = ProduceResult(topic="t")+ result.set_result(None, None)+ return result++ producer.produce.side_effect = _producewith patch(f"{_MODULE}.producer_scope") as producer_scope:producer_scope.return_value.__enter__.return_value = producerproduced = pps._produce_intents(@@ -635,7 +788,7 @@ class TestGroupTarget:team_uuid="team-uuid",group_type_name="organization",)- assert produced == 1+ assert produced == {"acme"}data = captured[0]["data"]assert data["kind"] == "group"assert data["group_type"] == "organization"@@ -645,6 +798,31 @@ class TestGroupTarget:# keyed per (team, group_type_index, group_key) so per-group ordering is preservedassert captured[0]["key"] == "1:0:acme"+ @parameterized.expand([("delivery_error", KafkaError(-192, "Local: Message timed out")), ("never_acked", None)])+ def test_produce_intents_returns_only_confirmed_keys(self, _name: str, failure: KafkaError | None) -> None:+ def _produce(**kw: object) -> ProduceResult:+ result = ProduceResult(topic="t")+ if kw["key"] == "1:ok":+ result.set_result(None, None)+ elif failure is not None:+ result.set_result(failure, None)+ return result++ producer = MagicMock()+ producer.produce.side_effect = _produce+ source = PersonPropertySyncSource("s1", "d1", "distinct_id", {"plan": "plan_tier"})+ with patch(f"{_MODULE}.producer_scope") as producer_scope:+ producer_scope.return_value.__enter__.return_value = producer+ delivered = pps._produce_intents(+ 1,+ "tok",+ source,+ [("ok", {"plan_tier": "pro"}), ("lost", {"plan_tier": "free"})],+ team_uuid="team-uuid",+ group_type_name=None,+ )+ assert delivered == {"ok"}+def test_filter_existing_ids_uses_group_lookup(self):group = MagicMock(group_key="acme")with patch(f"{_MODULE}.get_groups_by_identifiers", return_value=[group]) as lookup:@@ -666,8 +844,8 @@ class TestGroupTarget:patch(f"{_MODULE}._read_snapshot_hashes", new=AsyncMock(return_value={})),patch(f"{_MODULE}._filter_existing_ids", return_value={"acme"}),patch(f"{_MODULE}._group_type_name", return_value=None),- patch(f"{_MODULE}._produce_intents", return_value=1) as produce,- patch(f"{_MODULE}._write_snapshot_hashes", new=AsyncMock()) as write_snapshot,+ patch(f"{_MODULE}._produce_intents", return_value={"acme"}) as produce,+ patch(f"{_MODULE}._write_snapshot_chunk", new=AsyncMock()) as write_snapshot,patch(f"{_MODULE}._reconcile_property_definitions") as reconcile,patch(f"{_MODULE}._clear_staged", new=AsyncMock()),):@@ -698,7 +876,8 @@ class TestGroupTarget:f"{_MODULE}.get_group_types_for_project",side_effect=lambda project_id, **_kwargs: by_project.get(project_id, []),) as lookup,- patch(f"{_MODULE}._produce_intents", return_value=1) as produce,+ patch(f"{_MODULE}._produce_intents", return_value={"acme"}) as produce,+ patch(f"{_MODULE}._write_snapshot_chunk", new=AsyncMock()),patch(f"{_MODULE}._write_snapshot_hashes", new=AsyncMock()),patch(f"{_MODULE}._reconcile_property_definitions"),patch(f"{_MODULE}._clear_staged", new=AsyncMock()),