:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(customer-analytics): resume account property source syncs from a checkpoint (#120593)
Arthur Moreira de Deuse6bc978d3cc7
diff --git a/products/customer_analytics/backend/logic/account_property_sync.py b/products/customer_analytics/backend/logic/account_property_sync.pyindex 6f455d2cb4cc..a5f9f5d9c67a 100644--- a/products/customer_analytics/backend/logic/account_property_sync.py+++ b/products/customer_analytics/backend/logic/account_property_sync.py@@ -2,8 +2,8 @@ import ioimport jsonimport asyncioimport hashlib-from collections.abc import AsyncIterator, Iterator-from dataclasses import field+from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Sequence+from dataclasses import asdict, field, fieldsfrom datetime import date, datetime, timefrom decimal import Decimalfrom enum import StrEnum@@ -53,6 +53,7 @@ logger = structlog.get_logger(__name__)_ACCOUNT_LOOKUP_CHUNK_SIZE = 1_000_PARQUET_BATCH_SIZE = 50_000+_CHECKPOINT_INTERVAL_SECONDS = 60.0_SEGMENTS_REQUIRED_FOR_CLEANUP = frozenset({"tracked", "ignored"})_COUNT_KEYS = ("rows_read", "changed", "matched", "written", "source_errors", "source_count")_RUN_FAILED_ERROR = "Couldn't update accounts. Run the source view again. If it keeps failing, contact support."@@ -119,6 +120,64 @@ class SourceSyncState:error: str | None = None+@frozen+class SourceSyncCheckpoint:+ next_batch: int+ rows_read: int+ changed: int+ matched: int+ written: int+ failed: bool+ key_column: str+ source_column: str++ def __post_init__(self) -> None:+ for name in ("next_batch", "rows_read", "changed", "matched", "written"):+ value = getattr(self, name)+ if not isinstance(value, int) or isinstance(value, bool) or value < 0:+ raise ValueError(f"{name} must be a non-negative integer")+ if not isinstance(self.failed, bool):+ raise ValueError("failed must be a boolean")+ if not isinstance(self.key_column, str) or not isinstance(self.source_column, str):+ raise ValueError("key_column and source_column must be strings")++ @classmethod+ def from_state(cls, state: SourceSyncState, next_batch: int) -> "SourceSyncCheckpoint":+ return cls(+ next_batch=next_batch,+ rows_read=state.rows_read,+ changed=state.changed,+ matched=state.matched,+ written=state.written,+ failed=state.error is not None,+ key_column=state.source.key_column,+ source_column=state.source.source_column or "",+ )++ def matches_mapping(self, source: CustomPropertySource) -> bool:+ return (self.key_column, self.source_column) == (source.key_column, source.source_column)++ @classmethod+ def from_heartbeat_details(cls, details: Sequence[object]) -> "SourceSyncCheckpoint | None":+ if not details or not isinstance(details[0], dict):+ return None+ stored = details[0]+ try:+ return cls(**{checkpoint_field.name: stored[checkpoint_field.name] for checkpoint_field in fields(cls)})+ except (KeyError, ValueError):+ return None++ def to_heartbeat_details(self) -> dict[str, int | bool | str]:+ return asdict(self)++ def restore(self, state: SourceSyncState, counts: dict[str, int]) -> None:+ for key in ("rows_read", "changed", "matched", "written"):+ setattr(state, key, getattr(self, key))+ counts[key] = getattr(self, key)+ if self.failed:+ state.error = _INVALID_VALUE_ERROR++def _record_phase_duration(log: FilteringBoundLogger,segment: AccountPropertySyncSegment,@@ -188,8 +247,14 @@ def _projected_parquet_batches(s3_client: "S3FileSystem", path: str, columns: liasync def _iter_parquet_row_batches(- team_id: int, binding: WarehouseBinding, job_id: str, *, columns: list[str] | None = None+ team_id: int,+ binding: WarehouseBinding,+ job_id: str,+ *,+ columns: list[str] | None = None,+ skip_batches: int = 0,) -> AsyncIterator[list[dict[str, Any]]]:+ skipped = 0prefix = account_property_job_staged_prefix(team_id, binding, job_id)async with aget_s3_client() as s3_client:try:@@ -205,6 +270,9 @@ async def _iter_parquet_row_batches(else:batches = _projected_parquet_batches(get_s3_client(), _s3_uri(file_path), columns)while (batch := await asyncio.to_thread(next, batches, None)) is not None:+ if skipped < skip_batches:+ skipped += 1+ continueyield await asyncio.to_thread(batch.to_pylist)@@ -256,22 +324,39 @@ async def _read_snapshot_hashes(return merge.hashes if merge.complete else {}-async def _write_snapshot_hashes(+async def _write_snapshot_delta(team_id: int,binding: WarehouseBinding,source_id: str,segment: AccountPropertySyncSegment,- job_id: str,+ name: str,hashes: dict[str, str],) -> None:if not hashes:return+ prefix = account_property_snapshot_prefix(team_id, binding, source_id, segment.value)+ snapshot = await asyncio.to_thread(_encode_snapshot, hashes)+ async with aget_s3_client() as s3_client:+ await s3_client._pipe_file(_s3_uri(f"{prefix}/{name}.parquet"), snapshot)++async def _write_snapshot_hashes(+ team_id: int,+ binding: WarehouseBinding,+ source_id: str,+ segment: AccountPropertySyncSegment,+ job_id: str,+ hashes: dict[str, str],+) -> None:prefix = account_property_snapshot_prefix(team_id, binding, source_id, segment.value)path = f"{prefix}/{job_id}.parquet"async with aget_s3_client() as s3_client:existing_files = await _list_snapshot_files(s3_client, prefix)+ if not hashes and len(existing_files) <= 1:+ returnmerge = await _merge_snapshot_files(s3_client, existing_files)+ if not hashes and not merge.complete:+ returnmerged = {**merge.hashes, **hashes} if merge.complete else hashessnapshot = await asyncio.to_thread(_encode_snapshot, merged)await s3_client._pipe_file(_s3_uri(path), snapshot)@@ -489,9 +574,11 @@ async def _apply_staged_rows(counts: dict[str, int],log: FilteringBoundLogger,columns: list[str] | None = None,+ start_batch: int = 0,+ on_batch_applied: Callable[[int], Awaitable[None]] | None = None,) -> None:- batch_index = 0- batches = aiter(_iter_parquet_row_batches(team_id, binding, job_id, columns=columns))+ batch_index = start_batch+ batches = aiter(_iter_parquet_row_batches(team_id, binding, job_id, columns=columns, skip_batches=start_batch))while True:phase_started_at = asyncio.get_running_loop().time()try:@@ -575,6 +662,8 @@ async def _apply_staged_rows(},)batch_index += 1+ if on_batch_applied is not None:+ await on_batch_applied(batch_index)async def _persist_states(@@ -706,6 +795,8 @@ async def run_account_property_source_sync(segment: AccountPropertySyncSegment,source_id: str,final_attempt: bool = False,+ resume_from: SourceSyncCheckpoint | None = None,+ save_checkpoint: Callable[[SourceSyncCheckpoint], None] | None = None,) -> dict[str, int]:source_uuid = _parse_source_id(source_id)log = logger.bind(@@ -739,7 +830,34 @@ async def run_account_property_source_sync(if _is_source_applicable(source):counts["source_count"] = 1states = [state]+ start_batch = 0+ if resume_from is not None and not resume_from.matches_mapping(source):+ log.info("Account-property source mapping changed since the checkpoint, restarting the sync")+ elif resume_from is not None:+ resume_from.restore(state, counts)+ start_batch = resume_from.next_batch+ log.info("Resuming account-property source sync from a checkpoint", next_batch=start_batch)await _load_prior_hashes(team_id=team_id, binding=binding, segment=segment, states=states, log=log)+ loop = asyncio.get_running_loop()+ last_checkpoint_at = loop.time()+ checkpointed_hashes: dict[str, str] = {}++ async def checkpoint(next_batch: int) -> None:+ nonlocal last_checkpoint_at+ if save_checkpoint is None or loop.time() - last_checkpoint_at < _CHECKPOINT_INTERVAL_SECONDS:+ return+ new_hashes = {+ external_id: value_hash+ for external_id, value_hash in state.applied_hashes.items()+ if checkpointed_hashes.get(external_id) != value_hash+ }+ await _write_snapshot_delta(+ team_id, binding, str(source.id), segment, f"{job_id}-checkpoint-{next_batch}", new_hashes+ )+ checkpointed_hashes.update(new_hashes)+ save_checkpoint(SourceSyncCheckpoint.from_state(state, next_batch))+ last_checkpoint_at = loop.time()+await _apply_staged_rows(team_id=team_id,binding=binding,@@ -751,6 +869,8 @@ async def run_account_property_source_sync(columns=list(dict.fromkeys(column for column in (source.key_column, source.source_column) if column is not None)),+ start_batch=start_batch,+ on_batch_applied=checkpoint,)await _persist_states(team_id=team_id, binding=binding, job_id=job_id, segment=segment, states=states, counts=counts, log=logdiff --git a/products/customer_analytics/backend/logic/account_property_sync_test.py b/products/customer_analytics/backend/logic/account_property_sync_test.pyindex 0571845626dd..9a35e07b02bb 100644--- a/products/customer_analytics/backend/logic/account_property_sync_test.py+++ b/products/customer_analytics/backend/logic/account_property_sync_test.py@@ -1,6 +1,6 @@import ioimport hashlib-from collections.abc import AsyncIterator+from collections.abc import AsyncIterator, Callablefrom contextlib import AbstractContextManager, ExitStackfrom datetime import UTC, datetimefrom typing import TYPE_CHECKING, Any, Generic, TypeVar@@ -32,6 +32,7 @@ from products.customer_analytics.backend.logic.account_property_sync import (AccountPropertySyncPhase,AccountPropertySyncSegment,AppliedSourceValues,+ SourceSyncCheckpoint,_iter_parquet_row_batches,_list_snapshot_files,_mark_completed_and_maybe_cleanup,@@ -61,10 +62,17 @@ if TYPE_CHECKING:_MODULE = "products.customer_analytics.backend.logic.account_property_sync"DataWarehouseSavedQuery = apps.get_model("data_modeling", "DataWarehouseSavedQuery")-_ACCOUNT_IDS = {"acct-1": UUID("00000000-0000-4000-8000-0000000000a1")}+_ACCOUNT_IDS = {+ "acct-1": UUID("00000000-0000-4000-8000-0000000000a1"),+ "acct-2": UUID("00000000-0000-4000-8000-0000000000a2"),+}_EMPTY_COUNTS = {"rows_read": 0, "changed": 0, "matched": 0, "written": 0, "source_errors": 0, "source_count": 0}+class _WorkerStopped(Exception):+ pass++@frozenclass _RunState:status: str@@ -178,6 +186,8 @@ class AccountPropertySegmentTest(TeamScopedTestMixin, BaseTest):*,segment: AccountPropertySyncSegment = AccountPropertySyncSegment.TRACKED,final_attempt: bool = False,+ resume_from: SourceSyncCheckpoint | None = None,+ save_checkpoint: Callable[[SourceSyncCheckpoint], None] | None = None,) -> dict[str, int]:with self._fanout_boundaries(fake):return async_to_sync(run_account_property_source_sync)(@@ -187,6 +197,8 @@ class AccountPropertySegmentTest(TeamScopedTestMixin, BaseTest):segment=segment,source_id=source_id,final_attempt=final_attempt,+ resume_from=resume_from,+ save_checkpoint=save_checkpoint,)def _complete_segment(@@ -261,6 +273,59 @@ class AccountPropertySegmentTest(TeamScopedTestMixin, BaseTest):assert self._run_states(view.plan_source)["tracked"] == _COMPLETED_ONE_ROWassert self._snapshot(fake, view.binding, view.plan_source) == {"acct-1": _value_hash("enterprise")}+ @parameterized.expand(+ [+ ("same_mapping", "plan", {"acct-1": "enterprise", "acct-2": "starter"}),+ ("mapping_changed", "tier", {"acct-1": "gold", "acct-2": "silver"}),+ ]+ )+ def test_cancelled_source_sync_resumes_after_its_last_checkpoint(+ self, _case: str, resumed_column: str, expected_values: dict[str, str]+ ) -> None:+ fake = _FakeS3()+ view = self._create_view_with_two_sources(fake)+ self._stage_rows(+ fake,+ view.binding,+ [+ {"external_id": "acct-1", "plan": "enterprise", "tier": "gold"},+ {"external_id": "acct-2", "plan": "starter", "tier": "silver"},+ ],+ )+ checkpoints: list[SourceSyncCheckpoint] = []++ def cancel_after_first_checkpoint(checkpoint: SourceSyncCheckpoint) -> None:+ checkpoints.append(checkpoint)+ raise _WorkerStopped++ with (+ patch(f"{_MODULE}._PARQUET_BATCH_SIZE", 1),+ patch(f"{_MODULE}._CHECKPOINT_INTERVAL_SECONDS", 0.0),+ ):+ with pytest.raises(_WorkerStopped):+ self._run_source(+ fake, view.binding, str(view.plan_source.id), save_checkpoint=cancel_after_first_checkpoint+ )+ view.plan_source.source_column = resumed_column+ view.plan_source.save(update_fields=["source_column"])+ heartbeat_details = [checkpoints[0].to_heartbeat_details(), {"host": "worker-1", "ts": 1.0}]+ counts = self._run_source(+ fake,+ view.binding,+ str(view.plan_source.id),+ resume_from=SourceSyncCheckpoint.from_heartbeat_details(heartbeat_details),+ )++ assert checkpoints[0].next_batch == 1+ assert counts == {**_EMPTY_COUNTS, "rows_read": 2, "changed": 2, "matched": 2, "written": 2, "source_count": 1}+ assert self._run_states(view.plan_source)["tracked"] == _RunState(+ status="completed", phase="completed", rows_read=2, produced=2+ )+ assert self._snapshot(fake, view.binding, view.plan_source) == {+ external_id: _value_hash(value) for external_id, value in expected_values.items()+ }+ assert [key for key in fake.store if "-checkpoint-" in key] == []+@parameterized.expand([("other_view", AccountPropertySourceRejectedError),@@ -625,6 +690,31 @@ def test_source_values_preserve_explicit_nulls_but_skip_missing_columns(rows, exassert _source_values(rows, "external_id", "plan") == expected+_CHECKPOINT_DETAILS = {+ "next_batch": 3,+ "rows_read": 3,+ "changed": 1,+ "matched": 1,+ "written": 1,+ "failed": False,+ "key_column": "external_id",+ "source_column": "plan",+}+++ "details",+ [+ [],+ [{"host": "worker-1", "ts": 1.0}],+ [{**_CHECKPOINT_DETAILS, "next_batch": -1}],+ [{**_CHECKPOINT_DETAILS, "failed": "no"}],+ ],+)+def test_unusable_heartbeat_details_restart_the_source_sync(details: list[object]) -> None:+ assert SourceSyncCheckpoint.from_heartbeat_details(details) is None++@pytest.mark.asyncio@pytest.mark.parametrize("source_columns", [("value_0", "value_1"), ("value_0", None), ()])async def test_each_staged_batch_is_shared_across_sources(source_columns: tuple[str | None, ...]) -> None:@@ -876,13 +966,16 @@ async def test_snapshot_files_without_timestamps_merge_before_dated_files() -> N@pytest.mark.asyncio-async def test_snapshot_write_compacts_prior_files_newest_hash_winning() -> None:[email protected]("final_hashes", [{"b": "h2", "c": "h2"}, {}])+async def test_snapshot_write_compacts_prior_files_newest_hash_winning(final_hashes: dict[str, str]) -> None:fake = _FakeS3()await _write(fake, "job-1", {"a": "h1", "b": "h1"})- await _write(fake, "job-2", {"b": "h2", "c": "h2"})+ with _fake_s3_patch(fake):+ await aps._write_snapshot_delta(7, _SNAPSHOT_BINDING, "src", _SEGMENT, "job-2-checkpoint-1", {"b": "h2"})+ await _write(fake, "job-2", final_hashes)assert len(fake.store) == 1- assert await _read(fake) == {"a": "h1", "b": "h2", "c": "h2"}+ assert await _read(fake) == {"a": "h1", "b": "h2", **final_hashes}@pytest.mark.asynciodiff --git a/products/customer_analytics/backend/temporal/account_property_sync.py b/products/customer_analytics/backend/temporal/account_property_sync.pyindex 9554b1390708..0ed755942c6f 100644--- a/products/customer_analytics/backend/temporal/account_property_sync.py+++ b/products/customer_analytics/backend/temporal/account_property_sync.py@@ -51,6 +51,7 @@ with workflow.unsafe.imports_passed_through():AccountPropertySourceRejectedError,AccountPropertySourceValueError,AccountPropertySyncSegment,+ SourceSyncCheckpoint,complete_account_property_sync_segment,get_account_property_sync_source_ids,run_account_property_segment_sync,@@ -458,7 +459,14 @@ async def sync_warehouse_account_property_source_activity(input: AccountPropertysegment=segment,source_id=source_uuid,)- async with Heartbeater():+ resume_from = SourceSyncCheckpoint.from_heartbeat_details(activity_info.heartbeat_details)+ async with Heartbeater() as heartbeater:++ def save_checkpoint(checkpoint: SourceSyncCheckpoint) -> None:+ heartbeater.details = (checkpoint.to_heartbeat_details(),)++ if resume_from is not None:+ save_checkpoint(resume_from)result = await run_account_property_source_sync(team_id=input.team_id,binding=saved_query_binding(input.saved_query_id),@@ -466,6 +474,8 @@ async def sync_warehouse_account_property_source_activity(input: AccountPropertysegment=segment,source_id=input.source_id,final_attempt=activity_info.attempt >= ACCOUNT_PROPERTY_ACTIVITY_MAX_ATTEMPTS,+ resume_from=resume_from,+ save_checkpoint=save_checkpoint,)except (AccountPropertySourceValueError, AccountPropertySourceRejectedError) as error:# A retry reads the same staged rows and source config, so it can't fix these errors.