Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): resume person property syncs from the last delivered chunk (#122054)

Jovan Sakovic7ccab32af56f
diff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job.py
index b64c8d857116..92f82e730142 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job.py
@@ -45,7 +45,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.per
logger = structlog.get_logger(__name__)
-INTERRUPTED_RUN_NEXT_STEP = "Click Backfill to try again."
+INTERRUPTED_RUN_NEXT_STEP = "Click Backfill to continue from where it stopped."
RECORD_FAILED_RUN_PATCH = "person-property-backfill-record-failed-run-2026-10"
RETRY_INTERRUPTED_RUN_PATCH = "person-property-backfill-retry-interrupted-run-2026-10"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job.py
index b23742187edf..8bd7158a9047 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job.py
@@ -7,10 +7,10 @@ competes with the sync workers themselves; ``start_temporal_worker`` registers i
``PERSON_PROPERTY_SYNC_WORKFLOWS``/``PERSON_PROPERTY_SYNC_ACTIVITIES``.
Failure semantics: the activity retries up to 3 times, then the (abandoned) child workflow fails
-without affecting the sync. Nothing is lost — staged files are only cleared on success and the
-dedup snapshot only advances per produced person, so the next sync's run re-reads and re-diffs.
-Each failed attempt is captured to error tracking. The workflow then records the run as failed and
-counts it once in the metrics below, also when the activity was timed out, cancelled or killed.
+without affecting the sync. The dedup snapshot only advances per produced person, so a Backfill can
+recover rows left by an interrupted incremental run without resending confirmed updates. Each failed
+attempt is captured to error tracking. The workflow then records the run as failed and counts it once
+in the metrics below, also when the activity was timed out, cancelled or killed.
"""
import json
@@ -45,7 +45,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.per
logger = structlog.get_logger(__name__)
-INTERRUPTED_RUN_NEXT_STEP = "Check the source status, then click Sync now to try again."
+INTERRUPTED_RUN_NEXT_STEP = "Run a Backfill to deliver any rows left by the interrupted sync."
# Three recorder attempts of up to 5 minutes each.
RECORD_FAILED_RUN_RESERVE = timedelta(minutes=15)
RECORD_FAILED_RUN_PATCH = "person-property-sync-record-failed-run-2026-10"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job_test.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job_test.py
index ff16d355c333..f89d74052779 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job_test.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job_test.py
@@ -91,7 +91,7 @@ async def _backfill_activity_killed_by_worker_shutdown(inputs: PersonPropertyBac
@pytest.mark.parametrize(
- "workflow_cls,replaced_activity,killed_activity,real_activities,inputs,expected_job_id",
+ "workflow_cls,replaced_activity,killed_activity,real_activities,inputs,expected_job_id,expected_next_step",
[
(
SyncWarehousePersonPropertiesWorkflow,
@@ -109,6 +109,7 @@ async def _backfill_activity_killed_by_worker_shutdown(inputs: PersonPropertyBac
saved_query_id=uuid.UUID("00000000-0000-4000-8000-000000000001"),
),
"job-1",
+ "Run a Backfill to deliver any rows left by the interrupted sync.",
),
(
BackfillWarehousePersonPropertiesWorkflow,
@@ -124,11 +125,12 @@ async def _backfill_activity_killed_by_worker_shutdown(inputs: PersonPropertyBac
saved_query_id=uuid.UUID("00000000-0000-4000-8000-000000000001"),
),
None,
+ "Click Backfill to continue from where it stopped.",
),
],
)
async def test_run_is_marked_failed_when_the_activity_dies_without_its_own_handler(
- workflow_cls, replaced_activity, killed_activity, real_activities, inputs, expected_job_id
+ workflow_cls, replaced_activity, killed_activity, real_activities, inputs, expected_job_id, expected_next_step
) -> None:
recorded: list[PersonPropertySyncRunRecord] = []
source = PersonPropertySyncSource(
@@ -162,6 +164,7 @@ async def test_run_is_marked_failed_when_the_activity_dies_without_its_own_handl
assert failed[0].source_id == source.source_id
assert failed[0].job_id == expected_job_id
assert (failed[0].error or "").startswith("This run was interrupted before it finished.")
+ assert (failed[0].error or "").endswith(expected_next_step)
assert failed[0].finished_at
assert failed[0].count_failure is False
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py
index 14eef7fddf87..a35fff71044e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py
@@ -24,6 +24,7 @@ S3/personhog/Kafka helpers are the boundaries and are mocked in the orchestratio
import io
import json
+import uuid
import asyncio
import hashlib
import dataclasses
@@ -35,10 +36,12 @@ from django.conf import settings
import structlog
import pyarrow.parquet as pq
+from confluent_kafka import KafkaException
from temporalio.exceptions import ActivityError, ApplicationError, CancelledError
from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
+from posthog.kafka_client.client import ProduceResult
from posthog.kafka_client.routing import producer_scope
from posthog.kafka_client.topics import KAFKA_WAREHOUSE_PERSON_PROPERTY_UPDATES
from posthog.models import Team
@@ -97,6 +100,15 @@ ROW_BATCH_SIZE = 50_000
# for every person at once. Each extra pass reads only its own sources' columns.
SOURCES_PER_STAGED_PASS = 4
+# Changed entities per produce-and-snapshot step. Each step writes the hashes Kafka confirmed to its own
+# snapshot file before the next step starts, so a run that is stopped part way (a deploy stops the
+# worker) keeps what it delivered, and the next run's diff resumes from there instead of from zero.
+_PRODUCE_CHUNK_SIZE = 50_000
+
+# A concurrent compaction writes its merged file before it deletes the old ones, so a fresh listing
+# after a vanished file almost always reads complete. Giving up means re-sending every row.
+_SNAPSHOT_READ_ATTEMPTS = 3
+
@frozen(frozen=False)
class PerSourceResult:
@@ -108,11 +120,24 @@ class PerSourceResult:
existing: int = 0
produced: int = 0
skipped_missing_person: int = 0
+ # Intents Kafka did not confirm. They are left out of the snapshot so a retry sends them again.
+ undelivered: int = 0
# Set when the source can't write anything for a reason its counts alone don't explain, so the
# recorder persists a failed run instead of a clean one that produced nothing.
error: str | None = None
+class PropertyUpdatesNotDeliveredError(Exception):
+ """Kafka did not confirm some property-update intents. Raised so the run fails and Temporal retries
+ it. The snapshot holds only confirmed intents, so the retry sends only the undelivered ones."""
+
+
+@frozen
+class _SnapshotMerge:
+ hashes: dict[str, str]
+ complete: bool
+
+
@dataclasses.dataclass
class SyncResult:
sources: int = 0
@@ -209,8 +234,8 @@ def _wanted_columns(sources: list[PersonPropertySyncSource]) -> set[str]:
def _snapshot_prefix(team_id: int, binding: WarehouseBinding, source_id: str) -> str:
- # A folder, not a single file: each run drops a uniquely-named parquet of the hashes it produced,
- # and the reader unions every file in the folder. Two writers (a scheduled sync and a backfill)
+ # A folder, not a single file: each run drops uniquely-named parquet files of the hashes it
+ # produced (one per chunk, then one compacted file), and the reader unions every file in the folder. Two writers (a scheduled sync and a backfill)
# for the same source can't clobber each other, since they write different filenames. Growth is
# bounded by _write_snapshot_hashes, which compacts the folder back down as it writes.
return snapshot_prefix(team_id, binding, source_id)
@@ -320,10 +345,11 @@ async def _list_snapshot_files(s3_client, prefix: str) -> list[str]:
return [f["Key"] for f in files]
-async def _merge_snapshot_files(s3_client, file_keys: list[str]) -> dict[str, str]:
+async def _merge_snapshot_files(s3_client, file_keys: list[str]) -> _SnapshotMerge:
"""Union the given snapshot files into {distinct_id: sent_hash}, later files winning (callers pass
them oldest-first). Decodes off the event loop so a large parquet can't starve the heartbeater."""
hashes: dict[str, str] = {}
+ complete = True
for key in file_keys:
try:
data = await s3_client._cat_file(_s3_uri(key))
@@ -331,9 +357,10 @@ async def _merge_snapshot_files(s3_client, file_keys: list[str]) -> dict[str, st
# A concurrent _write_snapshot_hashes compacted this file into a new one and deleted it
# after we listed the folder but before we read it. Its rows survive in that new file (or
# whichever one replaces it next run) — skip rather than fail the whole read.
+ complete = False
continue
await asyncio.to_thread(_merge_snapshot_file, data, hashes)
- return hashes
+ return _SnapshotMerge(hashes=hashes, complete=complete)
async def _read_snapshot_hashes(team_id: int, binding: WarehouseBinding, source_id: str) -> dict[str, str]:
@@ -342,7 +369,31 @@ async def _read_snapshot_hashes(team_id: int, binding: WarehouseBinding, source_
cost an idempotent re-send, but newest-wins avoids even that). Ordering never affects correctness."""
prefix = _snapshot_prefix(team_id, binding, source_id)
async with aget_s3_client() as s3_client:
- return await _merge_snapshot_files(s3_client, await _list_snapshot_files(s3_client, prefix))
+ for _ in range(_SNAPSHOT_READ_ATTEMPTS):
+ merge = await _merge_snapshot_files(s3_client, await _list_snapshot_files(s3_client, prefix))
+ if merge.complete:
+ return merge.hashes
+ return {}
+
+
+async def _put_snapshot_file(s3_client, path: str, hashes: dict[str, str]) -> None:
+ import pyarrow as pa # noqa: PLC0415
+
+ table = pa.table({"distinct_id": list(hashes.keys()), "sent_hash": list(hashes.values())})
+ buffer = pa.BufferOutputStream()
+ pq.write_table(table, buffer, compression="zstd")
+ await s3_client._pipe_file(_s3_uri(path), buffer.getvalue().to_pybytes())
+
+
+async def _write_snapshot_chunk(
+ team_id: int, binding: WarehouseBinding, source_id: str, file_name: str, hashes: dict[str, str]
+) -> None:
+ """Add ``hashes`` to the source's snapshot folder as one new file, without reading or merging the
+ folder. The reader unions this file with the rest, and ``_write_snapshot_hashes`` folds it in when
+ the run compacts. Merging here instead would re-read the whole folder once per chunk."""
+ path = f"{_snapshot_prefix(team_id, binding, source_id)}/{file_name}.parquet"
+ async with aget_s3_client() as s3_client:
+ await _put_snapshot_file(s3_client, path, hashes)
async def _write_snapshot_hashes(
@@ -358,19 +409,19 @@ async def _write_snapshot_hashes(
listing isn't in the merge set, so it's neither merged-away nor deleted, and its rows still surface
through the reader's union; the two files collapse on the next run. Write-then-delete keeps a crash
between the two harmless — the reader unions both the new file and the not-yet-deleted old ones."""
- import pyarrow as pa # noqa: PLC0415
-
prefix = _snapshot_prefix(team_id, binding, source_id)
write_path = f"{prefix}/{run_token}.parquet"
async with aget_s3_client() as s3_client:
existing_keys = await _list_snapshot_files(s3_client, prefix)
- merged = await _merge_snapshot_files(s3_client, existing_keys)
- merged.update(hashes)
-
- table = pa.table({"distinct_id": list(merged.keys()), "sent_hash": list(merged.values())})
- buffer = pa.BufferOutputStream()
- pq.write_table(table, buffer, compression="zstd")
- await s3_client._pipe_file(_s3_uri(write_path), buffer.getvalue().to_pybytes())
+ if not hashes and len(existing_keys) <= 1:
+ return
+ merge = await _merge_snapshot_files(s3_client, existing_keys)
+ if not hashes and not merge.complete:
+ return
+ merged = {**merge.hashes, **hashes} if merge.complete else hashes
+ await _put_snapshot_file(s3_client, write_path, merged)
+ if not merge.complete:
+ return
# Delete the files we merged, except the one we just wrote (the backfill token reuses its name).
stale = [key for key in existing_keys if _s3_key(key) != _s3_key(write_path)]
@@ -444,12 +495,13 @@ def _produce_intents(
*,
team_uuid: str,
group_type_name: str | None,
-) -> int:
+) -> set[str]:
"""Produce one property-update intent per entity to Kafka. Person intents map to a `$set` keyed
by team:distinct_id; group intents map to a `$groupidentify` keyed by team:group_type:group_key so
- per-group ordering is preserved. The consumer branches on ``kind``. Returns the count produced."""
+ per-group ordering is preserved. The consumer branches on ``kind``. Returns the keys Kafka
+ confirmed as delivered."""
is_group = source.target == _GROUP_TARGET
- produced = 0
+ results: list[tuple[str, ProduceResult]] = []
with producer_scope(topic=KAFKA_WAREHOUSE_PERSON_PROPERTY_UPDATES) as producer:
for key, bundle in items:
if is_group:
@@ -476,9 +528,19 @@ def _produce_intents(
"event_source": EVENT_SOURCE,
}
kafka_key = f"{team_id}:{key}"
- producer.produce(topic=KAFKA_WAREHOUSE_PERSON_PROPERTY_UPDATES, data=data, key=kafka_key)
- produced += 1
- return produced
+ results.append(
+ (key, producer.produce(topic=KAFKA_WAREHOUSE_PERSON_PROPERTY_UPDATES, data=data, key=kafka_key))
+ )
+ # The scope flushes on exit. flush() counts only messages still queued, and a failed delivery
+ # leaves the queue, so only each message's result shows whether Kafka accepted it.
+ delivered: set[str] = set()
+ for key, result in results:
+ try:
+ result.get(timeout=0)
+ except (KafkaException, TimeoutError):
+ continue
+ delivered.add(key)
+ return delivered
def _reconcile_property_definitions(
@@ -517,7 +579,9 @@ async def _process_source_bundles(
run_token: str,
) -> PerSourceResult:
"""Diff one source's bundles against its snapshot, drop entities that don't exist, produce update
- intents (person `$set` or group `$groupidentify`), stamp provenance, and advance the snapshot.
+ intents (person `$set` or group `$groupidentify`), stamp provenance, and advance the snapshot. Works
+ in chunks of ``_PRODUCE_CHUNK_SIZE`` and snapshots each chunk's confirmed deliveries before the next
+ chunk, so a run stopped part way loses at most one chunk of progress.
Shared by the incremental sync and the backfill — they differ only in how ``bundles`` are sourced
(staged rows vs a full Delta read)."""
ps = PerSourceResult(source_id=str(source.source_id), rows_read=rows_read)
@@ -530,23 +594,7 @@ async def _process_source_bundles(
changed, new_hashes = await asyncio.to_thread(select_changed, bundles, prior)
ps.changed = len(changed)
if not changed:
- return ps
-
- existing = await database_sync_to_async(_filter_existing_ids, thread_sensitive=False)(
- team_id, source, [key for key, _ in changed]
- )
- to_send = [(key, bundle) for key, bundle in changed if key in existing]
- ps.existing = len(to_send)
- ps.skipped_missing_person = len(changed) - len(to_send)
- if not to_send:
- logger.info(
- "person-property sync: no existing entities among changed rows for source",
- team_id=team_id,
- **_log_fields(binding),
- source_id=str(source.source_id),
- target=source.target,
- changed=len(changed),
- )
+ await _write_snapshot_hashes(team_id, binding, str(source.source_id), run_token, {})
return ps
group_type_name = None
@@ -565,15 +613,62 @@ async def _process_source_bundles(
group_type_index=source.group_type_index,
)
return ps
- produced = await asyncio.to_thread(
- _produce_intents, team_id, team_api_token, source, to_send, team_uuid=team_uuid, group_type_name=group_type_name
- )
- ps.produced = produced
- # Record only the distinct_ids we actually produced, as this run's snapshot file.
- sent_ids = {distinct_id for distinct_id, _ in to_send}
- produced_hashes = {d: h for d, h in new_hashes.items() if d in sent_ids}
- await _write_snapshot_hashes(team_id, binding, str(source.source_id), run_token, produced_hashes)
+ # Snapshot chunk files get a name unique to this attempt. A Temporal retry reuses ``run_token``, and
+ # its chunk numbering starts again at zero, so a name built from the token and index alone would
+ # overwrite the files that hold the previous attempt's delivered hashes.
+ attempt = uuid.uuid4().hex[:12]
+ chunks_written = 0
+ confirmed_hashes: dict[str, str] = {}
+ for start in range(0, len(changed), _PRODUCE_CHUNK_SIZE):
+ chunk = changed[start : start + _PRODUCE_CHUNK_SIZE]
+ existing = await database_sync_to_async(_filter_existing_ids, thread_sensitive=False)(
+ team_id, source, [key for key, _ in chunk]
+ )
+ to_send = [(key, bundle) for key, bundle in chunk if key in existing]
+ ps.existing += len(to_send)
+ ps.skipped_missing_person += len(chunk) - len(to_send)
+ if not to_send:
+ continue
+
+ delivered = await asyncio.to_thread(
+ _produce_intents,
+ team_id,
+ team_api_token,
+ source,
+ to_send,
+ team_uuid=team_uuid,
+ group_type_name=group_type_name,
+ )
+ ps.produced += len(delivered)
+ ps.undelivered += len(to_send) - len(delivered)
+
+ # Write only after Kafka confirms delivery. A hash in the snapshot means "already sent", so a
+ # hash written before delivery would make a lost intent look sent and the next run skip it.
+ # An undelivered id stays out, so the next diff sees it as changed and sends it again.
+ delivered_hashes = {key: new_hashes[key] for key in delivered}
+ if delivered_hashes:
+ await _write_snapshot_chunk(
+ team_id, binding, str(source.source_id), f"{run_token}-{attempt}-{chunks_written:05d}", delivered_hashes
+ )
+ confirmed_hashes.update(delivered_hashes)
+ chunks_written += 1
+
+ if chunks_written:
+ # Fold this run's chunk files, and any left by an earlier run that was stopped part way, into
+ # one ``{run_token}.parquet`` so the folder stays at about one file per source.
+ await _write_snapshot_hashes(team_id, binding, str(source.source_id), run_token, confirmed_hashes)
+
+ if not ps.existing:
+ logger.info(
+ "person-property sync: no existing entities among changed rows for source",
+ team_id=team_id,
+ **_log_fields(binding),
+ source_id=str(source.source_id),
+ target=source.target,
+ changed=len(changed),
+ )
+ return ps
logger.info(
"person-property sync: source processed",
@@ -582,8 +677,9 @@ async def _process_source_bundles(
source_id=str(source.source_id),
bundles=len(bundles),
changed=len(changed),
- existing=len(to_send),
- produced=produced,
+ existing=ps.existing,
+ produced=ps.produced,
+ undelivered=ps.undelivered,
)
return ps
@@ -620,6 +716,15 @@ def _accumulate(result: SyncResult, ps: PerSourceResult) -> None:
result.skipped_missing_person += ps.skipped_missing_person
+def _raise_if_undelivered(result: SyncResult) -> None:
+ undelivered = sum(ps.undelivered for ps in result.per_source)
+ if undelivered:
+ raise PropertyUpdatesNotDeliveredError(
+ f"Kafka did not confirm {undelivered} of {undelivered + result.produced} property updates. "
+ "A retry sends the undelivered updates again."
+ )
+
+
async def run_person_property_sync(*, team_id: int, binding: WarehouseBinding, job_id: str) -> SyncResult:
result = SyncResult()
sources = await database_sync_to_async(person_property_sync_sources_for, thread_sensitive=False)(team_id, binding)
@@ -660,6 +765,8 @@ async def run_person_property_sync(*, team_id: int, binding: WarehouseBinding, j
)
_accumulate(result, ps)
+ # Raise before the staged rows are cleared, so the retry can read them again.
+ _raise_if_undelivered(result)
await _clear_staged(team_id, binding, job_id)
return result
@@ -821,6 +928,7 @@ async def run_person_property_backfill(*, team_id: int, binding: WarehouseBindin
)
_accumulate(result, ps)
+ _raise_if_undelivered(result)
return result
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync_test.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync_test.py
index f15105acbfda..18e3271c3e82 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync_test.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync_test.py
@@ -14,8 +14,10 @@ from django.test.utils import CaptureQueriesContext
import pyarrow as pa
import pyarrow.parquet as pq
+from confluent_kafka import KafkaError
from parameterized import parameterized
+from posthog.kafka_client.client import ProduceResult
from posthog.models import Organization, PropertyDefinition, Team
from products.warehouse_sources.backend.temporal.data_imports.external_product_hooks import (
@@ -155,9 +157,9 @@ class TestRunOrchestration:
{"distinct_id": "ghost", "plan": "x"},
]
- def _record_produce(*_args: object, **_kwargs: object) -> int:
+ def _record_produce(*_args: object, **_kwargs: object) -> set[str]:
order.append("produce")
- return 2
+ return {"a", "b"}
fake = _FakeS3()
_stage(fake, "chunk_1_000000", rows)
@@ -169,9 +171,13 @@ class TestRunOrchestration:
patch(f"{_MODULE}._filter_existing_ids", return_value={"a", "b"}) as existing,
patch(f"{_MODULE}._produce_intents", side_effect=_record_produce) as produce,
patch(
- f"{_MODULE}._write_snapshot_hashes",
+ f"{_MODULE}._write_snapshot_chunk",
new=AsyncMock(side_effect=lambda *_args, **_kwargs: order.append("snapshot")),
- ) as write_snapshot,
+ ) as write_chunk,
+ patch(
+ f"{_MODULE}._write_snapshot_hashes",
+ new=AsyncMock(side_effect=lambda *_args, **_kwargs: order.append("compact")),
+ ) as compact,
patch(
f"{_MODULE}._reconcile_property_definitions",
side_effect=lambda *_args, **_kwargs: order.append("reconcile"),
@@ -191,16 +197,85 @@ class TestRunOrchestration:
assert result.per_source[0].produced == 2
assert result.per_source[0].existing == 2
- # snapshot advanced only for produced ids (args: team, schema, source, run_token, hashes).
- assert write_snapshot.await_args is not None
- assert write_snapshot.await_args.args[3] == "job-1"
- written = write_snapshot.await_args.args[4]
- assert set(written) == {"a", "b"}
+ # snapshot advanced only for produced ids (args: team, schema, source, file name, hashes).
+ assert write_chunk.await_args is not None
+ assert write_chunk.await_args.args[3].startswith("job-1-")
+ assert set(write_chunk.await_args.args[4]) == {"a", "b"}
+ assert compact.await_args is not None
+ assert compact.await_args.args[3] == "job-1"
+ assert set(compact.await_args.args[4]) == {"a", "b"}
existing.assert_called_once()
reconcile.assert_called_once()
clear.assert_awaited_once()
- assert order == ["reconcile", "produce", "snapshot"]
+ assert order == ["reconcile", "produce", "snapshot", "compact"]
+
+ @pytest.mark.asyncio
+ async def test_undelivered_intents_stay_out_of_the_snapshot_and_fail_the_run(self):
+ team = MagicMock(api_token="tok", project_id=7)
+ fake = _FakeS3()
+ _stage(fake, "chunk_1_000000", [{"distinct_id": "a", "plan": "pro"}, {"distinct_id": "b", "plan": "free"}])
+ with (
+ _fake_s3_patch(fake),
+ patch(f"{_MODULE}.person_property_sync_sources_for", return_value=[self._source()]),
+ patch(f"{_MODULE}.Team") as team_cls,
+ patch(f"{_MODULE}._filter_existing_ids", return_value={"a", "b"}),
+ patch(f"{_MODULE}._produce_intents", return_value={"a"}),
+ patch(f"{_MODULE}._reconcile_property_definitions"),
+ patch(f"{_MODULE}._clear_staged", new=AsyncMock()) as clear,
+ ):
+ team_cls.objects.get.return_value = team
+ with pytest.raises(pps.PropertyUpdatesNotDeliveredError, match="1 of 2"):
+ await pps.run_person_property_sync(team_id=1, binding=_SCHEMA, job_id="job-1")
+ hashes = await pps._read_snapshot_hashes(1, _SCHEMA, "source-1")
+
+ assert set(hashes) == {"a"}
+ clear.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ async def test_a_killed_run_keeps_confirmed_chunks_but_a_later_job_cannot_resume_undelivered_rows(self):
+ team = MagicMock(api_token="tok", project_id=7)
+ fake = _FakeS3()
+ _stage(fake, "chunk_1_000000", [{"distinct_id": f"p{n}", "plan": "pro"} for n in range(6)], job_id="job-1")
+ _stage(fake, "chunk_1_000000", [{"distinct_id": "p6", "plan": "pro"}], job_id="job-2")
+ sent_per_run: dict[str, list[str]] = {"job-1": [], "job-2": []}
+ current_job = "job-1"
+ kill_after = 2
+
+ def _produce(_team_id, _token, _source, items, **_kwargs):
+ # Run 1 is killed (a deploy stops the worker) once it has delivered `kill_after` intents.
+ sent = sent_per_run[current_job]
+ if current_job == "job-1" and len(sent) + len(items) > kill_after:
+ raise RuntimeError("worker stopped")
+ sent.extend(key for key, _ in items)
+ return {key for key, _ in items}
+
+ with (
+ _fake_s3_patch(fake),
+ patch(f"{_MODULE}._PRODUCE_CHUNK_SIZE", 2),
+ patch(f"{_MODULE}.person_property_sync_sources_for", return_value=[self._source()]),
+ patch(f"{_MODULE}.Team") as team_cls,
+ patch(f"{_MODULE}._filter_existing_ids", side_effect=lambda _team_id, _source, ids: set(ids)),
+ patch(f"{_MODULE}._produce_intents", side_effect=_produce),
+ patch(f"{_MODULE}._reconcile_property_definitions"),
+ ):
+ team_cls.objects.get.return_value = team
+ with pytest.raises(RuntimeError, match="worker stopped"):
+ await pps.run_person_property_sync(team_id=1, binding=_SCHEMA, job_id="job-1")
+
+ current_job = "job-2"
+ result = await pps.run_person_property_sync(team_id=1, binding=_SCHEMA, job_id="job-2")
+
+ prefix = pps._snapshot_prefix(1, _SCHEMA, "source-1")
+ snapshot_files = await pps._list_snapshot_files(fake, prefix)
+ hashes = await pps._read_snapshot_hashes(1, _SCHEMA, "source-1")
+
+ assert sent_per_run["job-1"] == ["p0", "p1"]
+ assert sent_per_run["job-2"] == ["p6"]
+ assert result.produced == 1
+ # The completed run folds the killed run's leftover file and its own chunk files into one.
+ assert [pps._s3_key(key) for key in snapshot_files] == [f"{prefix}/job-2.parquet"]
+ assert set(hashes) == {"p0", "p1", "p6"}
@parameterized.expand([("one_pass", 4), ("pass_per_source", 1)])
@pytest.mark.asyncio
@@ -230,7 +305,11 @@ class TestRunOrchestration:
patch(f"{_MODULE}.build_bundles", wraps=pps.build_bundles) as build,
patch(f"{_MODULE}._read_snapshot_hashes", new=AsyncMock(return_value={})),
patch(f"{_MODULE}._filter_existing_ids", side_effect=lambda _team, _source, ids: set(ids)),
- patch(f"{_MODULE}._produce_intents", return_value=2) as produce,
+ patch(
+ f"{_MODULE}._produce_intents",
+ side_effect=lambda _team_id, _token, _source, items, **_kwargs: {key for key, _ in items},
+ ) as produce,
+ patch(f"{_MODULE}._write_snapshot_chunk", new=AsyncMock()),
patch(f"{_MODULE}._write_snapshot_hashes", new=AsyncMock()),
patch(f"{_MODULE}._reconcile_property_definitions"),
patch(f"{_MODULE}._clear_staged", new=AsyncMock()),
@@ -268,6 +347,7 @@ class TestRunOrchestration:
patch(f"{_MODULE}._reconcile_property_definitions") as reconcile,
patch(f"{_MODULE}._filter_existing_ids") as existing,
patch(f"{_MODULE}._produce_intents") as produce,
+ patch(f"{_MODULE}._write_snapshot_hashes", new=AsyncMock()) as compact,
):
result = await pps._process_source_bundles(
team_id=1,
@@ -284,6 +364,7 @@ class TestRunOrchestration:
reconcile.assert_called_once_with(1, 7, _SCHEMA, source, ["plan_tier"])
existing.assert_not_called()
produce.assert_not_called()
+ compact.assert_awaited_once_with(1, _SCHEMA, "source-1", "job-1", {})
assert result.changed == 0 and result.produced == 0
@@ -379,7 +460,8 @@ class TestBackfillOrchestration:
) as read_delta,
patch(f"{_MODULE}._read_snapshot_hashes", new=AsyncMock(return_value={})),
patch(f"{_MODULE}._filter_existing_ids", return_value={"a"}),
- patch(f"{_MODULE}._produce_intents", return_value=1) as produce,
+ patch(f"{_MODULE}._produce_intents", return_value={"a"}) as produce,
+ patch(f"{_MODULE}._write_snapshot_chunk", new=AsyncMock()),
patch(f"{_MODULE}._write_snapshot_hashes", new=AsyncMock()) as write_snapshot,
patch(f"{_MODULE}._reconcile_property_definitions"),
):
@@ -392,7 +474,7 @@ class TestBackfillOrchestration:
assert result.rows_read == 5
assert result.produced == 2 # one produced per source
assert sorted(ps.source_id for ps in result.per_source) == ["s1", "s2"]
- # Both sources write a snapshot file under the shared backfill run token.
+ # Both sources compact their snapshot into the shared backfill run token's file.
assert write_snapshot.await_count == 2
assert all(call.args[3] == pps.BACKFILL_RUN_TOKEN for call in write_snapshot.await_args_list)
assert produce.call_count == 2
@@ -491,6 +573,7 @@ class TestBackfillOrchestration:
),
) as read_delta,
patch(f"{_MODULE}._read_snapshot_hashes", new=AsyncMock(return_value={})),
+ patch(f"{_MODULE}._write_snapshot_hashes", new=AsyncMock()),
patch(f"{_MODULE}._reconcile_property_definitions"),
):
team_cls.objects.get.return_value = team
@@ -609,9 +692,72 @@ class TestSnapshotCompaction:
keys = await pps._list_snapshot_files(fake, pps._snapshot_prefix(1, _SCHEMA, "src"))
await fake._rm(keys)
- hashes = await pps._merge_snapshot_files(fake, keys)
+ merge = await pps._merge_snapshot_files(fake, keys)
- assert hashes == {}
+ assert merge.hashes == {}
+ assert not merge.complete
+
+ @pytest.mark.asyncio
+ async def test_read_relists_after_concurrent_compaction_instead_of_resending_everything(self):
+ fake = _FakeS3()
+ prefix = pps._snapshot_prefix(1, _SCHEMA, "src")
+ with _fake_s3_patch(fake):
+ await pps._write_snapshot_chunk(1, _SCHEMA, "src", "job-1", {"a": "h1"})
+ await pps._write_snapshot_chunk(1, _SCHEMA, "src", "job-2", {"b": "h2"})
+ original_cat_file = fake._cat_file
+ compacted = False
+
+ async def _cat_with_one_concurrent_compaction(path):
+ nonlocal compacted
+ if not compacted:
+ compacted = True
+ await pps._put_snapshot_file(fake, f"{prefix}/concurrent.parquet", {"a": "h1", "b": "h2"})
+ await fake._rm([f"{prefix}/job-1.parquet", f"{prefix}/job-2.parquet"])
+ return await original_cat_file(path)
+
+ with patch.object(fake, "_cat_file", _cat_with_one_concurrent_compaction):
+ hashes = await pps._read_snapshot_hashes(1, _SCHEMA, "src")
+
+ assert hashes == {"a": "h1", "b": "h2"}
+
+ @pytest.mark.asyncio
+ async def test_current_deliveries_win_when_snapshot_timestamps_tie(self):
+ fake = _FakeS3()
+ prefix = pps._snapshot_prefix(1, _SCHEMA, "src")
+ with _fake_s3_patch(fake):
+ await pps._write_snapshot_chunk(1, _SCHEMA, "src", "aaa-chunk", {"a": "new"})
+ await pps._write_snapshot_chunk(1, _SCHEMA, "src", pps.BACKFILL_RUN_TOKEN, {"a": "old"})
+ fake.times = dict.fromkeys(fake.times, 1)
+
+ await pps._write_snapshot_hashes(1, _SCHEMA, "src", "job-1", {"a": "new"})
+ hashes = await pps._read_snapshot_hashes(1, _SCHEMA, "src")
+
+ assert hashes == {"a": "new"}
+ assert list(fake.store) == [f"{prefix}/job-1.parquet"]
+
+ @pytest.mark.asyncio
+ async def test_incomplete_merge_does_not_publish_or_delete_partial_snapshot(self):
+ fake = _FakeS3()
+ prefix = pps._snapshot_prefix(1, _SCHEMA, "src")
+ with _fake_s3_patch(fake):
+ await pps._write_snapshot_chunk(1, _SCHEMA, "src", "job-1", {"a": "old"})
+ await pps._write_snapshot_chunk(1, _SCHEMA, "src", "job-2", {"a": "new"})
+ original_cat_file = fake._cat_file
+
+ async def _cat_after_concurrent_compaction(path):
+ data = await original_cat_file(path)
+ if path.endswith("/job-1.parquet"):
+ await pps._put_snapshot_file(fake, f"{prefix}/concurrent.parquet", {"a": "new"})
+ await fake._rm([f"{prefix}/job-1.parquet", f"{prefix}/job-2.parquet"])
+ return data
+
+ with patch.object(fake, "_cat_file", _cat_after_concurrent_compaction):
+ await pps._write_snapshot_hashes(1, _SCHEMA, "src", "job-3", {})
+
+ hashes = await pps._read_snapshot_hashes(1, _SCHEMA, "src")
+
+ assert hashes == {"a": "new"}
+ assert list(fake.store) == [f"{prefix}/concurrent.parquet"]
class TestGroupTarget:
@@ -624,7 +770,14 @@ class TestGroupTarget:
def test_produce_intents_group_payload(self):
captured: list[dict] = []
producer = MagicMock()
- producer.produce.side_effect = lambda **kw: captured.append(kw)
+
+ def _produce(**kw: object) -> ProduceResult:
+ captured.append(kw)
+ result = ProduceResult(topic="t")
+ result.set_result(None, None)
+ return result
+
+ producer.produce.side_effect = _produce
with patch(f"{_MODULE}.producer_scope") as producer_scope:
producer_scope.return_value.__enter__.return_value = producer
produced = pps._produce_intents(
@@ -635,7 +788,7 @@ class TestGroupTarget:
team_uuid="team-uuid",
group_type_name="organization",
)
- assert produced == 1
+ assert produced == {"acme"}
data = captured[0]["data"]
assert data["kind"] == "group"
assert data["group_type"] == "organization"
@@ -645,6 +798,31 @@ class TestGroupTarget:
# keyed per (team, group_type_index, group_key) so per-group ordering is preserved
assert captured[0]["key"] == "1:0:acme"
+ @parameterized.expand([("delivery_error", KafkaError(-192, "Local: Message timed out")), ("never_acked", None)])
+ def test_produce_intents_returns_only_confirmed_keys(self, _name: str, failure: KafkaError | None) -> None:
+ def _produce(**kw: object) -> ProduceResult:
+ result = ProduceResult(topic="t")
+ if kw["key"] == "1:ok":
+ result.set_result(None, None)
+ elif failure is not None:
+ result.set_result(failure, None)
+ return result
+
+ producer = MagicMock()
+ producer.produce.side_effect = _produce
+ source = PersonPropertySyncSource("s1", "d1", "distinct_id", {"plan": "plan_tier"})
+ with patch(f"{_MODULE}.producer_scope") as producer_scope:
+ producer_scope.return_value.__enter__.return_value = producer
+ delivered = pps._produce_intents(
+ 1,
+ "tok",
+ source,
+ [("ok", {"plan_tier": "pro"}), ("lost", {"plan_tier": "free"})],
+ team_uuid="team-uuid",
+ group_type_name=None,
+ )
+ assert delivered == {"ok"}
+
def test_filter_existing_ids_uses_group_lookup(self):
group = MagicMock(group_key="acme")
with patch(f"{_MODULE}.get_groups_by_identifiers", return_value=[group]) as lookup:
@@ -666,8 +844,8 @@ class TestGroupTarget:
patch(f"{_MODULE}._read_snapshot_hashes", new=AsyncMock(return_value={})),
patch(f"{_MODULE}._filter_existing_ids", return_value={"acme"}),
patch(f"{_MODULE}._group_type_name", return_value=None),
- patch(f"{_MODULE}._produce_intents", return_value=1) as produce,
- patch(f"{_MODULE}._write_snapshot_hashes", new=AsyncMock()) as write_snapshot,
+ patch(f"{_MODULE}._produce_intents", return_value={"acme"}) as produce,
+ patch(f"{_MODULE}._write_snapshot_chunk", new=AsyncMock()) as write_snapshot,
patch(f"{_MODULE}._reconcile_property_definitions") as reconcile,
patch(f"{_MODULE}._clear_staged", new=AsyncMock()),
):
@@ -698,7 +876,8 @@ class TestGroupTarget:
f"{_MODULE}.get_group_types_for_project",
side_effect=lambda project_id, **_kwargs: by_project.get(project_id, []),
) as lookup,
- patch(f"{_MODULE}._produce_intents", return_value=1) as produce,
+ patch(f"{_MODULE}._produce_intents", return_value={"acme"}) as produce,
+ patch(f"{_MODULE}._write_snapshot_chunk", new=AsyncMock()),
patch(f"{_MODULE}._write_snapshot_hashes", new=AsyncMock()),
patch(f"{_MODULE}._reconcile_property_definitions"),
patch(f"{_MODULE}._clear_staged", new=AsyncMock()),