Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(customer-analytics): resume account property source syncs from a checkpoint (#120593)

Arthur Moreira de Deuse6bc978d3cc7
diff --git a/products/customer_analytics/backend/logic/account_property_sync.py b/products/customer_analytics/backend/logic/account_property_sync.py
index 6f455d2cb4cc..a5f9f5d9c67a 100644
--- a/products/customer_analytics/backend/logic/account_property_sync.py
+++ b/products/customer_analytics/backend/logic/account_property_sync.py
@@ -2,8 +2,8 @@ import io
import json
import asyncio
import hashlib
-from collections.abc import AsyncIterator, Iterator
-from dataclasses import field
+from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Sequence
+from dataclasses import asdict, field, fields
from datetime import date, datetime, time
from decimal import Decimal
from enum import StrEnum
@@ -53,6 +53,7 @@ logger = structlog.get_logger(__name__)
_ACCOUNT_LOOKUP_CHUNK_SIZE = 1_000
_PARQUET_BATCH_SIZE = 50_000
+_CHECKPOINT_INTERVAL_SECONDS = 60.0
_SEGMENTS_REQUIRED_FOR_CLEANUP = frozenset({"tracked", "ignored"})
_COUNT_KEYS = ("rows_read", "changed", "matched", "written", "source_errors", "source_count")
_RUN_FAILED_ERROR = "Couldn't update accounts. Run the source view again. If it keeps failing, contact support."
@@ -119,6 +120,64 @@ class SourceSyncState:
error: str | None = None
+@frozen
+class SourceSyncCheckpoint:
+ next_batch: int
+ rows_read: int
+ changed: int
+ matched: int
+ written: int
+ failed: bool
+ key_column: str
+ source_column: str
+
+ def __post_init__(self) -> None:
+ for name in ("next_batch", "rows_read", "changed", "matched", "written"):
+ value = getattr(self, name)
+ if not isinstance(value, int) or isinstance(value, bool) or value < 0:
+ raise ValueError(f"{name} must be a non-negative integer")
+ if not isinstance(self.failed, bool):
+ raise ValueError("failed must be a boolean")
+ if not isinstance(self.key_column, str) or not isinstance(self.source_column, str):
+ raise ValueError("key_column and source_column must be strings")
+
+ @classmethod
+ def from_state(cls, state: SourceSyncState, next_batch: int) -> "SourceSyncCheckpoint":
+ return cls(
+ next_batch=next_batch,
+ rows_read=state.rows_read,
+ changed=state.changed,
+ matched=state.matched,
+ written=state.written,
+ failed=state.error is not None,
+ key_column=state.source.key_column,
+ source_column=state.source.source_column or "",
+ )
+
+ def matches_mapping(self, source: CustomPropertySource) -> bool:
+ return (self.key_column, self.source_column) == (source.key_column, source.source_column)
+
+ @classmethod
+ def from_heartbeat_details(cls, details: Sequence[object]) -> "SourceSyncCheckpoint | None":
+ if not details or not isinstance(details[0], dict):
+ return None
+ stored = details[0]
+ try:
+ return cls(**{checkpoint_field.name: stored[checkpoint_field.name] for checkpoint_field in fields(cls)})
+ except (KeyError, ValueError):
+ return None
+
+ def to_heartbeat_details(self) -> dict[str, int | bool | str]:
+ return asdict(self)
+
+ def restore(self, state: SourceSyncState, counts: dict[str, int]) -> None:
+ for key in ("rows_read", "changed", "matched", "written"):
+ setattr(state, key, getattr(self, key))
+ counts[key] = getattr(self, key)
+ if self.failed:
+ state.error = _INVALID_VALUE_ERROR
+
+
def _record_phase_duration(
log: FilteringBoundLogger,
segment: AccountPropertySyncSegment,
@@ -188,8 +247,14 @@ def _projected_parquet_batches(s3_client: "S3FileSystem", path: str, columns: li
async def _iter_parquet_row_batches(
- team_id: int, binding: WarehouseBinding, job_id: str, *, columns: list[str] | None = None
+ team_id: int,
+ binding: WarehouseBinding,
+ job_id: str,
+ *,
+ columns: list[str] | None = None,
+ skip_batches: int = 0,
) -> AsyncIterator[list[dict[str, Any]]]:
+ skipped = 0
prefix = account_property_job_staged_prefix(team_id, binding, job_id)
async with aget_s3_client() as s3_client:
try:
@@ -205,6 +270,9 @@ async def _iter_parquet_row_batches(
else:
batches = _projected_parquet_batches(get_s3_client(), _s3_uri(file_path), columns)
while (batch := await asyncio.to_thread(next, batches, None)) is not None:
+ if skipped < skip_batches:
+ skipped += 1
+ continue
yield await asyncio.to_thread(batch.to_pylist)
@@ -256,22 +324,39 @@ async def _read_snapshot_hashes(
return merge.hashes if merge.complete else {}
-async def _write_snapshot_hashes(
+async def _write_snapshot_delta(
team_id: int,
binding: WarehouseBinding,
source_id: str,
segment: AccountPropertySyncSegment,
- job_id: str,
+ name: str,
hashes: dict[str, str],
) -> None:
if not hashes:
return
+ prefix = account_property_snapshot_prefix(team_id, binding, source_id, segment.value)
+ snapshot = await asyncio.to_thread(_encode_snapshot, hashes)
+ async with aget_s3_client() as s3_client:
+ await s3_client._pipe_file(_s3_uri(f"{prefix}/{name}.parquet"), snapshot)
+
+async def _write_snapshot_hashes(
+ team_id: int,
+ binding: WarehouseBinding,
+ source_id: str,
+ segment: AccountPropertySyncSegment,
+ job_id: str,
+ hashes: dict[str, str],
+) -> None:
prefix = account_property_snapshot_prefix(team_id, binding, source_id, segment.value)
path = f"{prefix}/{job_id}.parquet"
async with aget_s3_client() as s3_client:
existing_files = await _list_snapshot_files(s3_client, prefix)
+ if not hashes and len(existing_files) <= 1:
+ return
merge = await _merge_snapshot_files(s3_client, existing_files)
+ if not hashes and not merge.complete:
+ return
merged = {**merge.hashes, **hashes} if merge.complete else hashes
snapshot = await asyncio.to_thread(_encode_snapshot, merged)
await s3_client._pipe_file(_s3_uri(path), snapshot)
@@ -489,9 +574,11 @@ async def _apply_staged_rows(
counts: dict[str, int],
log: FilteringBoundLogger,
columns: list[str] | None = None,
+ start_batch: int = 0,
+ on_batch_applied: Callable[[int], Awaitable[None]] | None = None,
) -> None:
- batch_index = 0
- batches = aiter(_iter_parquet_row_batches(team_id, binding, job_id, columns=columns))
+ batch_index = start_batch
+ batches = aiter(_iter_parquet_row_batches(team_id, binding, job_id, columns=columns, skip_batches=start_batch))
while True:
phase_started_at = asyncio.get_running_loop().time()
try:
@@ -575,6 +662,8 @@ async def _apply_staged_rows(
},
)
batch_index += 1
+ if on_batch_applied is not None:
+ await on_batch_applied(batch_index)
async def _persist_states(
@@ -706,6 +795,8 @@ async def run_account_property_source_sync(
segment: AccountPropertySyncSegment,
source_id: str,
final_attempt: bool = False,
+ resume_from: SourceSyncCheckpoint | None = None,
+ save_checkpoint: Callable[[SourceSyncCheckpoint], None] | None = None,
) -> dict[str, int]:
source_uuid = _parse_source_id(source_id)
log = logger.bind(
@@ -739,7 +830,34 @@ async def run_account_property_source_sync(
if _is_source_applicable(source):
counts["source_count"] = 1
states = [state]
+ start_batch = 0
+ if resume_from is not None and not resume_from.matches_mapping(source):
+ log.info("Account-property source mapping changed since the checkpoint, restarting the sync")
+ elif resume_from is not None:
+ resume_from.restore(state, counts)
+ start_batch = resume_from.next_batch
+ log.info("Resuming account-property source sync from a checkpoint", next_batch=start_batch)
await _load_prior_hashes(team_id=team_id, binding=binding, segment=segment, states=states, log=log)
+ loop = asyncio.get_running_loop()
+ last_checkpoint_at = loop.time()
+ checkpointed_hashes: dict[str, str] = {}
+
+ async def checkpoint(next_batch: int) -> None:
+ nonlocal last_checkpoint_at
+ if save_checkpoint is None or loop.time() - last_checkpoint_at < _CHECKPOINT_INTERVAL_SECONDS:
+ return
+ new_hashes = {
+ external_id: value_hash
+ for external_id, value_hash in state.applied_hashes.items()
+ if checkpointed_hashes.get(external_id) != value_hash
+ }
+ await _write_snapshot_delta(
+ team_id, binding, str(source.id), segment, f"{job_id}-checkpoint-{next_batch}", new_hashes
+ )
+ checkpointed_hashes.update(new_hashes)
+ save_checkpoint(SourceSyncCheckpoint.from_state(state, next_batch))
+ last_checkpoint_at = loop.time()
+
await _apply_staged_rows(
team_id=team_id,
binding=binding,
@@ -751,6 +869,8 @@ async def run_account_property_source_sync(
columns=list(
dict.fromkeys(column for column in (source.key_column, source.source_column) if column is not None)
),
+ start_batch=start_batch,
+ on_batch_applied=checkpoint,
)
await _persist_states(
team_id=team_id, binding=binding, job_id=job_id, segment=segment, states=states, counts=counts, log=log
diff --git a/products/customer_analytics/backend/logic/account_property_sync_test.py b/products/customer_analytics/backend/logic/account_property_sync_test.py
index 0571845626dd..9a35e07b02bb 100644
--- a/products/customer_analytics/backend/logic/account_property_sync_test.py
+++ b/products/customer_analytics/backend/logic/account_property_sync_test.py
@@ -1,6 +1,6 @@
import io
import hashlib
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Callable
from contextlib import AbstractContextManager, ExitStack
from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any, Generic, TypeVar
@@ -32,6 +32,7 @@ from products.customer_analytics.backend.logic.account_property_sync import (
AccountPropertySyncPhase,
AccountPropertySyncSegment,
AppliedSourceValues,
+ SourceSyncCheckpoint,
_iter_parquet_row_batches,
_list_snapshot_files,
_mark_completed_and_maybe_cleanup,
@@ -61,10 +62,17 @@ if TYPE_CHECKING:
_MODULE = "products.customer_analytics.backend.logic.account_property_sync"
DataWarehouseSavedQuery = apps.get_model("data_modeling", "DataWarehouseSavedQuery")
-_ACCOUNT_IDS = {"acct-1": UUID("00000000-0000-4000-8000-0000000000a1")}
+_ACCOUNT_IDS = {
+ "acct-1": UUID("00000000-0000-4000-8000-0000000000a1"),
+ "acct-2": UUID("00000000-0000-4000-8000-0000000000a2"),
+}
_EMPTY_COUNTS = {"rows_read": 0, "changed": 0, "matched": 0, "written": 0, "source_errors": 0, "source_count": 0}
+class _WorkerStopped(Exception):
+ pass
+
+
@frozen
class _RunState:
status: str
@@ -178,6 +186,8 @@ class AccountPropertySegmentTest(TeamScopedTestMixin, BaseTest):
*,
segment: AccountPropertySyncSegment = AccountPropertySyncSegment.TRACKED,
final_attempt: bool = False,
+ resume_from: SourceSyncCheckpoint | None = None,
+ save_checkpoint: Callable[[SourceSyncCheckpoint], None] | None = None,
) -> dict[str, int]:
with self._fanout_boundaries(fake):
return async_to_sync(run_account_property_source_sync)(
@@ -187,6 +197,8 @@ class AccountPropertySegmentTest(TeamScopedTestMixin, BaseTest):
segment=segment,
source_id=source_id,
final_attempt=final_attempt,
+ resume_from=resume_from,
+ save_checkpoint=save_checkpoint,
)
def _complete_segment(
@@ -261,6 +273,59 @@ class AccountPropertySegmentTest(TeamScopedTestMixin, BaseTest):
assert self._run_states(view.plan_source)["tracked"] == _COMPLETED_ONE_ROW
assert self._snapshot(fake, view.binding, view.plan_source) == {"acct-1": _value_hash("enterprise")}
+ @parameterized.expand(
+ [
+ ("same_mapping", "plan", {"acct-1": "enterprise", "acct-2": "starter"}),
+ ("mapping_changed", "tier", {"acct-1": "gold", "acct-2": "silver"}),
+ ]
+ )
+ def test_cancelled_source_sync_resumes_after_its_last_checkpoint(
+ self, _case: str, resumed_column: str, expected_values: dict[str, str]
+ ) -> None:
+ fake = _FakeS3()
+ view = self._create_view_with_two_sources(fake)
+ self._stage_rows(
+ fake,
+ view.binding,
+ [
+ {"external_id": "acct-1", "plan": "enterprise", "tier": "gold"},
+ {"external_id": "acct-2", "plan": "starter", "tier": "silver"},
+ ],
+ )
+ checkpoints: list[SourceSyncCheckpoint] = []
+
+ def cancel_after_first_checkpoint(checkpoint: SourceSyncCheckpoint) -> None:
+ checkpoints.append(checkpoint)
+ raise _WorkerStopped
+
+ with (
+ patch(f"{_MODULE}._PARQUET_BATCH_SIZE", 1),
+ patch(f"{_MODULE}._CHECKPOINT_INTERVAL_SECONDS", 0.0),
+ ):
+ with pytest.raises(_WorkerStopped):
+ self._run_source(
+ fake, view.binding, str(view.plan_source.id), save_checkpoint=cancel_after_first_checkpoint
+ )
+ view.plan_source.source_column = resumed_column
+ view.plan_source.save(update_fields=["source_column"])
+ heartbeat_details = [checkpoints[0].to_heartbeat_details(), {"host": "worker-1", "ts": 1.0}]
+ counts = self._run_source(
+ fake,
+ view.binding,
+ str(view.plan_source.id),
+ resume_from=SourceSyncCheckpoint.from_heartbeat_details(heartbeat_details),
+ )
+
+ assert checkpoints[0].next_batch == 1
+ assert counts == {**_EMPTY_COUNTS, "rows_read": 2, "changed": 2, "matched": 2, "written": 2, "source_count": 1}
+ assert self._run_states(view.plan_source)["tracked"] == _RunState(
+ status="completed", phase="completed", rows_read=2, produced=2
+ )
+ assert self._snapshot(fake, view.binding, view.plan_source) == {
+ external_id: _value_hash(value) for external_id, value in expected_values.items()
+ }
+ assert [key for key in fake.store if "-checkpoint-" in key] == []
+
@parameterized.expand(
[
("other_view", AccountPropertySourceRejectedError),
@@ -625,6 +690,31 @@ def test_source_values_preserve_explicit_nulls_but_skip_missing_columns(rows, ex
assert _source_values(rows, "external_id", "plan") == expected
+_CHECKPOINT_DETAILS = {
+ "next_batch": 3,
+ "rows_read": 3,
+ "changed": 1,
+ "matched": 1,
+ "written": 1,
+ "failed": False,
+ "key_column": "external_id",
+ "source_column": "plan",
+}
+
+
+ "details",
+ [
+ [],
+ [{"host": "worker-1", "ts": 1.0}],
+ [{**_CHECKPOINT_DETAILS, "next_batch": -1}],
+ [{**_CHECKPOINT_DETAILS, "failed": "no"}],
+ ],
+)
+def test_unusable_heartbeat_details_restart_the_source_sync(details: list[object]) -> None:
+ assert SourceSyncCheckpoint.from_heartbeat_details(details) is None
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("source_columns", [("value_0", "value_1"), ("value_0", None), ()])
async def test_each_staged_batch_is_shared_across_sources(source_columns: tuple[str | None, ...]) -> None:
@@ -876,13 +966,16 @@ async def test_snapshot_files_without_timestamps_merge_before_dated_files() -> N
@pytest.mark.asyncio
-async def test_snapshot_write_compacts_prior_files_newest_hash_winning() -> None:
[email protected]("final_hashes", [{"b": "h2", "c": "h2"}, {}])
+async def test_snapshot_write_compacts_prior_files_newest_hash_winning(final_hashes: dict[str, str]) -> None:
fake = _FakeS3()
await _write(fake, "job-1", {"a": "h1", "b": "h1"})
- await _write(fake, "job-2", {"b": "h2", "c": "h2"})
+ with _fake_s3_patch(fake):
+ await aps._write_snapshot_delta(7, _SNAPSHOT_BINDING, "src", _SEGMENT, "job-2-checkpoint-1", {"b": "h2"})
+ await _write(fake, "job-2", final_hashes)
assert len(fake.store) == 1
- assert await _read(fake) == {"a": "h1", "b": "h2", "c": "h2"}
+ assert await _read(fake) == {"a": "h1", "b": "h2", **final_hashes}
@pytest.mark.asyncio
diff --git a/products/customer_analytics/backend/temporal/account_property_sync.py b/products/customer_analytics/backend/temporal/account_property_sync.py
index 9554b1390708..0ed755942c6f 100644
--- a/products/customer_analytics/backend/temporal/account_property_sync.py
+++ b/products/customer_analytics/backend/temporal/account_property_sync.py
@@ -51,6 +51,7 @@ with workflow.unsafe.imports_passed_through():
AccountPropertySourceRejectedError,
AccountPropertySourceValueError,
AccountPropertySyncSegment,
+ SourceSyncCheckpoint,
complete_account_property_sync_segment,
get_account_property_sync_source_ids,
run_account_property_segment_sync,
@@ -458,7 +459,14 @@ async def sync_warehouse_account_property_source_activity(input: AccountProperty
segment=segment,
source_id=source_uuid,
)
- async with Heartbeater():
+ resume_from = SourceSyncCheckpoint.from_heartbeat_details(activity_info.heartbeat_details)
+ async with Heartbeater() as heartbeater:
+
+ def save_checkpoint(checkpoint: SourceSyncCheckpoint) -> None:
+ heartbeater.details = (checkpoint.to_heartbeat_details(),)
+
+ if resume_from is not None:
+ save_checkpoint(resume_from)
result = await run_account_property_source_sync(
team_id=input.team_id,
binding=saved_query_binding(input.saved_query_id),
@@ -466,6 +474,8 @@ async def sync_warehouse_account_property_source_activity(input: AccountProperty
segment=segment,
source_id=input.source_id,
final_attempt=activity_info.attempt >= ACCOUNT_PROPERTY_ACTIVITY_MAX_ATTEMPTS,
+ resume_from=resume_from,
+ save_checkpoint=save_checkpoint,
)
except (AccountPropertySourceValueError, AccountPropertySourceRejectedError) as error:
# A retry reads the same staged rows and source config, so it can't fix these errors.