:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse-sources): retry interrupted property backfills (#124043)
Jovan Sakovic4eb710b4ed83
diff --git a/products/customer_analytics/backend/logic/person_property_runs.py b/products/customer_analytics/backend/logic/person_property_runs.pyindex bef71fe3e798..6f3735d760db 100644--- a/products/customer_analytics/backend/logic/person_property_runs.py+++ b/products/customer_analytics/backend/logic/person_property_runs.py@@ -145,7 +145,7 @@ def record_sync_run(record: PersonPropertySyncRunRecord) -> None:source,error=record.error,disable_event_id=record.job_id or str(run.id),- count_failure=not already_failed,+ count_failure=record.count_failure and not already_failed,)log.warning("person-property run recorded: failed",diff --git a/products/customer_analytics/backend/logic/person_property_runs_test.py b/products/customer_analytics/backend/logic/person_property_runs_test.pyindex d548cae582d8..04da8d173697 100644--- a/products/customer_analytics/backend/logic/person_property_runs_test.py+++ b/products/customer_analytics/backend/logic/person_property_runs_test.py@@ -2,7 +2,7 @@ from datetime import UTC, datetimefrom uuid import uuid4from posthog.test.base import APIBaseTest-from unittest.mock import patch+from unittest.mock import MagicMock, patchfrom parameterized import parameterized@@ -87,15 +87,22 @@ class TestRecordSyncRun(TeamScopedTestMixin, APIBaseTest):assert self.source.last_sync_error == "boom"assert self.source.is_enabled is True- def test_auto_disables_at_failure_cap(self):- # Regression: a source failing forever should stop syncing, matching the account path's cap.+ @parameterized.expand([(True,), (False,)])+ @patch(f"{HEALTH_SERVICE}.notify_source_auto_disabled")+ def test_auto_disables_at_failure_cap(self, count_failure: bool, mock_notify: MagicMock) -> None:self.source.consecutive_failures = MAX_CONSECUTIVE_SYNC_FAILURES - 1self.source.save()- record_sync_run(self._record(status="failed", error="boom"))+ with self.captureOnCommitCallbacks(execute=True):+ record_sync_run(self._record(status="failed", error="boom", count_failure=count_failure))self.source.refresh_from_db()- assert self.source.is_enabled is False+ assert self.source.is_enabled is (not count_failure)+ assert self.source.consecutive_failures == MAX_CONSECUTIVE_SYNC_FAILURES - 1 + int(count_failure)+ assert self.source.last_sync_error == "boom"+ run = CustomPropertySyncRun.objects.for_team(self.team.id).get(source_id=self.source.id)+ assert run.status == "failed" and run.error == "boom"+ assert mock_notify.call_count == int(count_failure)def test_unknown_source_is_a_noop(self):# A run for a since-deleted source must not crash or orphan a row.diff --git a/products/warehouse_sources/backend/temporal/data_imports/external_product_hooks.py b/products/warehouse_sources/backend/temporal/data_imports/external_product_hooks.pyindex 4e814b15ff34..cec1015cb93d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/external_product_hooks.py+++ b/products/warehouse_sources/backend/temporal/data_imports/external_product_hooks.py@@ -356,6 +356,7 @@ class PersonPropertyFailedRunInputs:trigger: strstarted_at: strerror: str+ count_failure: bool = True@propertydef binding(self) -> WarehouseBinding:@@ -432,6 +433,7 @@ class PersonPropertySyncRunRecord:produced: intskipped_missing_person: interror: str | None+ count_failure: bool = TruePersonPropertySyncRecorder = Callable[[PersonPropertySyncRunRecord], None]diff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job.pyindex 9f1c56ac4aec..b64c8d857116 100644--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job.py+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job.py@@ -24,7 +24,7 @@ import structlogfrom prometheus_client import Counter, Histogramfrom temporalio import activity, workflowfrom temporalio.common import RetryPolicy-from temporalio.exceptions import ActivityError+from temporalio.exceptions import ActivityError, ApplicationError, TimeoutError, TimeoutTypefrom posthog.exceptions_capture import capture_exceptionfrom posthog.temporal.common.base import PostHogWorkflow@@ -36,6 +36,7 @@ from products.warehouse_sources.backend.temporal.data_imports.external_product_h)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.person_property_sync import (failed_run_error,+ is_worker_interruption,record_completed_runs,record_failed_runs,record_started_runs,@@ -46,6 +47,7 @@ logger = structlog.get_logger(__name__)INTERRUPTED_RUN_NEXT_STEP = "Click Backfill to try again."RECORD_FAILED_RUN_PATCH = "person-property-backfill-record-failed-run-2026-10"+RETRY_INTERRUPTED_RUN_PATCH = "person-property-backfill-retry-interrupted-run-2026-10"PERSON_PROPERTY_BACKFILL_TOTAL = Counter("warehouse_person_property_backfill_total",@@ -147,6 +149,7 @@ async def record_failed_person_property_backfill_runs_activity(inputs: PersonProstarted_at=inputs.started_at,finished_at=datetime.now(UTC).isoformat(),error=inputs.error,+ count_failure=inputs.count_failure,)PERSON_PROPERTY_BACKFILL_TOTAL.labels(team_id=str(inputs.team_id), outcome="failed").inc()@@ -187,14 +190,25 @@ class BackfillWarehousePersonPropertiesWorkflow(PostHogWorkflow):started_at = workflow.now()records_failure = workflow.patched(RECORD_FAILED_RUN_PATCH)try:- await workflow.execute_activity(- backfill_warehouse_person_properties_activity,- dataclasses.replace(activity_inputs, workflow_records_failure=records_failure),- start_to_close_timeout=timedelta(hours=6),- heartbeat_timeout=timedelta(minutes=5),- # A one-off full-table read: don't silently re-scan the whole table on a transient error.- retry_policy=RetryPolicy(maximum_attempts=1),- )+ max_attempts = 3 if workflow.patched(RETRY_INTERRUPTED_RUN_PATCH) else 1+ for attempt in range(max_attempts):+ try:+ await workflow.execute_activity(+ backfill_warehouse_person_properties_activity,+ dataclasses.replace(activity_inputs, workflow_records_failure=records_failure),+ start_to_close_timeout=timedelta(hours=6),+ heartbeat_timeout=timedelta(minutes=5),+ # A full-table read must not retry application errors or the six-hour timeout.+ retry_policy=RetryPolicy(maximum_attempts=1),+ )+ break+ except ActivityError as error:+ cause = error.cause+ can_retry = (isinstance(cause, ApplicationError) and cause.type == "WorkerShutdown") or (+ isinstance(cause, TimeoutError) and cause.type == TimeoutType.HEARTBEAT+ )+ if not can_retry or attempt == max_attempts - 1:+ raiseexcept ActivityError as e:if records_failure:try:@@ -208,6 +222,7 @@ class BackfillWarehousePersonPropertiesWorkflow(PostHogWorkflow):trigger=activity_inputs.trigger,started_at=started_at.isoformat(),error=failed_run_error(e, INTERRUPTED_RUN_NEXT_STEP),+ count_failure=not is_worker_interruption(e),),start_to_close_timeout=timedelta(minutes=5),# The recorder swallows its own errors. A retry could only follow a writediff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job_test.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job_test.pyindex 712499d4fff5..c0f1609375dc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job_test.py+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job_test.py@@ -1,17 +1,19 @@import uuidimport asyncioimport dataclasses+from collections.abc import Callablefrom datetime import UTC, datetimeimport pytestfrom unittest.mock import AsyncMock, patch-from temporalio.exceptions import ActivityError+from temporalio.exceptions import ActivityError, ApplicationError, TimeoutError, TimeoutTypefrom temporalio.testing import ActivityEnvironmentfrom products.warehouse_sources.backend.temporal.data_imports import person_property_backfill_job as bjfrom products.warehouse_sources.backend.temporal.data_imports.external_product_hooks import (PersonPropertyBackfillActivityInputs,+ PersonPropertyFailedRunInputs,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.person_property_sync import SyncResult@@ -26,9 +28,8 @@ def _inputs(team_id: int) -> PersonPropertyBackfillActivityInputs:)-def _activity_error() -> ActivityError:- """What ``execute_activity`` raises once the activity exhausts its single attempt."""- return ActivityError(+def _activity_error(cause: Exception | None = None) -> ActivityError:+ error = ActivityError("backfill failed",scheduled_event_id=1,started_event_id=2,@@ -37,6 +38,8 @@ def _activity_error() -> ActivityError:activity_id="1",retry_state=None,)+ error.__cause__ = cause+ return error@pytest.mark.django_db(transaction=True)@@ -215,3 +218,62 @@ async def test_a_failed_run_with_nothing_queued_fails_the_workflow() -> None:await runner.run(seed)assert observed == [only]+++ "cause,failures,patch_enabled,expected_attempts",+ [+ (ApplicationError("Worker stopped", type="WorkerShutdown"), 1, True, 2),+ (ApplicationError("Worker stopped", type="WorkerShutdown"), 2, True, 3),+ (ApplicationError("Worker stopped", type="WorkerShutdown"), 3, True, 3),+ (TimeoutError("No heartbeat", type=TimeoutType.HEARTBEAT, last_heartbeat_details=[]), 1, True, 2),+ (TimeoutError("No heartbeat", type=TimeoutType.HEARTBEAT, last_heartbeat_details=[]), 3, True, 3),+ (TimeoutError("Too slow", type=TimeoutType.START_TO_CLOSE, last_heartbeat_details=[]), 1, True, 1),+ (ApplicationError("Invalid mapping", type="ValueError"), 1, True, 1),+ (ApplicationError("Worker stopped", type="WorkerShutdown"), 1, False, 1),+ ],+)+async def test_interrupted_backfill_retries_only_worker_shutdowns_and_heartbeat_timeouts(+ cause: Exception, failures: int, patch_enabled: bool, expected_attempts: int+) -> None:+ inputs = _inputs(900006)+ runner = bj.BackfillWarehousePersonPropertiesWorkflow(inputs)+ attempts = 0+ failed_records: list[PersonPropertyFailedRunInputs] = []++ async def execute_activity(+ activity_fn: Callable[..., object],+ activity_inputs: PersonPropertyBackfillActivityInputs | PersonPropertyFailedRunInputs,+ **_kwargs: object,+ ) -> None:+ nonlocal attempts+ if activity_fn is bj.record_failed_person_property_backfill_runs_activity:+ assert isinstance(activity_inputs, PersonPropertyFailedRunInputs)+ failed_records.append(activity_inputs)+ return+ attempts += 1+ if attempts <= failures:+ raise _activity_error(cause)++ def patched(patch_id: str) -> bool:+ return patch_enabled if patch_id == bj.RETRY_INTERRUPTED_RUN_PATCH else True++ with (+ patch.object(bj.workflow, "execute_activity", AsyncMock(side_effect=execute_activity)),+ patch.object(bj.workflow, "wait_condition", AsyncMock()),+ patch.object(bj.workflow, "now", return_value=datetime(2026, 10, 1, tzinfo=UTC)),+ patch.object(bj.workflow, "patched", side_effect=patched),+ ):+ if failures >= expected_attempts:+ with pytest.raises(ActivityError):+ await runner.run(inputs)+ else:+ await runner.run(inputs)++ assert attempts == expected_attempts+ assert len(failed_records) == (1 if failures >= expected_attempts else 0)+ if failed_records:+ assert failed_records[0].count_failure == (+ not (isinstance(cause, ApplicationError) and cause.type == "WorkerShutdown")+ )diff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job.pyindex e0543e1b46b8..b23742187edf 100644--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job.py+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job.py@@ -36,6 +36,7 @@ from products.warehouse_sources.backend.temporal.data_imports.external_product_h)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.person_property_sync import (failed_run_error,+ is_worker_interruption,record_completed_runs,record_failed_runs,record_started_runs,@@ -44,7 +45,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.perlogger = structlog.get_logger(__name__)-INTERRUPTED_RUN_NEXT_STEP = "The next sync will try again."+INTERRUPTED_RUN_NEXT_STEP = "Check the source status, then click Sync now to try again."# Three recorder attempts of up to 5 minutes each.RECORD_FAILED_RUN_RESERVE = timedelta(minutes=15)RECORD_FAILED_RUN_PATCH = "person-property-sync-record-failed-run-2026-10"@@ -149,6 +150,7 @@ async def record_failed_person_property_sync_runs_activity(inputs: PersonPropertstarted_at=inputs.started_at,finished_at=datetime.now(UTC).isoformat(),error=inputs.error,+ count_failure=inputs.count_failure,)PERSON_PROPERTY_SYNC_TOTAL.labels(team_id=str(inputs.team_id), outcome="failed").inc()@@ -191,6 +193,7 @@ class SyncWarehousePersonPropertiesWorkflow(PostHogWorkflow):trigger="scheduled",started_at=info.start_time.isoformat(),error=failed_run_error(e, INTERRUPTED_RUN_NEXT_STEP),+ count_failure=not is_worker_interruption(e),),start_to_close_timeout=timedelta(minutes=5),retry_policy=RetryPolicy(maximum_attempts=3),diff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job_test.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job_test.pyindex 3f86def1da59..ff16d355c333 100644--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job_test.py+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job_test.py@@ -1,4 +1,5 @@import uuid+import dataclassesfrom datetime import timedeltaimport pytest@@ -7,20 +8,23 @@ from unittest.mock import AsyncMock, MagicMock, patchfrom temporalio import activity, workflowfrom temporalio.client import WorkflowFailureErrorfrom temporalio.common import RetryPolicy-from temporalio.exceptions import ApplicationError+from temporalio.exceptions import ActivityError, ApplicationErrorfrom temporalio.testing import ActivityEnvironment, WorkflowEnvironmentfrom temporalio.worker import Replayer, UnsandboxedWorkflowRunner, Workerfrom products.warehouse_sources.backend.temporal.data_imports.external_product_hooks import (PersonPropertyBackfillActivityInputs,+ PersonPropertyFailedRunInputs,PersonPropertySyncActivityInputs,PersonPropertySyncRunRecord,PersonPropertySyncSource,)from products.warehouse_sources.backend.temporal.data_imports.person_property_backfill_job import (PERSON_PROPERTY_BACKFILL_ACTIVITIES,+ RECORD_FAILED_RUN_PATCH,BackfillWarehousePersonPropertiesWorkflow,backfill_warehouse_person_properties_activity,+ record_failed_person_property_backfill_runs_activity,)from products.warehouse_sources.backend.temporal.data_imports.person_property_sync_job import (PERSON_PROPERTY_SYNC_ACTIVITIES,@@ -159,6 +163,7 @@ async def test_run_is_marked_failed_when_the_activity_dies_without_its_own_handlassert failed[0].job_id == expected_job_idassert (failed[0].error or "").startswith("This run was interrupted before it finished.")assert failed[0].finished_at+ assert failed[0].count_failure is False@workflow.defn(name="sync-warehouse-person-properties")@@ -185,6 +190,44 @@ class _PrePatchBackfillWorkflow:)[email protected](name="record_failed_person_property_backfill_runs_activity")+async def _discard_backfill_failure(_inputs: PersonPropertyFailedRunInputs) -> None:+ pass++[email protected](name="backfill-warehouse-person-properties")+class _BeforeInterruptionRetryBackfillWorkflow:+ @workflow.run+ async def run(self, inputs: PersonPropertyBackfillActivityInputs) -> None:+ started_at = workflow.now()+ records_failure = workflow.patched(RECORD_FAILED_RUN_PATCH)+ try:+ await workflow.execute_activity(+ backfill_warehouse_person_properties_activity,+ dataclasses.replace(inputs, workflow_records_failure=records_failure),+ start_to_close_timeout=timedelta(hours=6),+ heartbeat_timeout=timedelta(minutes=5),+ retry_policy=RetryPolicy(maximum_attempts=1),+ )+ except ActivityError:+ if records_failure:+ await workflow.execute_activity(+ record_failed_person_property_backfill_runs_activity,+ PersonPropertyFailedRunInputs(+ team_id=inputs.team_id,+ schema_id=inputs.schema_id,+ saved_query_id=inputs.saved_query_id,+ job_id=None,+ trigger=inputs.trigger,+ started_at=started_at.isoformat(),+ error="This run was interrupted before it finished. Click Backfill to try again.",+ ),+ start_to_close_timeout=timedelta(minutes=5),+ retry_policy=RetryPolicy(maximum_attempts=1),+ )+ raise++@pytest.mark.parametrize("pre_patch_workflow,workflow_cls,killed_activity,inputs",[@@ -216,6 +259,19 @@ class _PrePatchBackfillWorkflow:saved_query_id=uuid.UUID("00000000-0000-4000-8000-000000000001"),),),+ (+ _BeforeInterruptionRetryBackfillWorkflow,+ BackfillWarehousePersonPropertiesWorkflow,+ _backfill_activity_killed_by_worker_shutdown,+ PersonPropertyBackfillActivityInputs(+ team_id=1,+ schema_id=None,+ source_type="saved_query",+ schema_name="example_view",+ trigger="manual",+ saved_query_id=uuid.UUID("00000000-0000-4000-8000-000000000001"),+ ),+ ),],)async def test_failed_history_from_before_the_failed_run_record_still_replays(@@ -227,7 +283,7 @@ async def test_failed_history_from_before_the_failed_run_record_still_replays(environment.client,task_queue=task_queue,workflows=[pre_patch_workflow],- activities=[killed_activity],+ activities=[killed_activity, _discard_backfill_failure],workflow_runner=UnsandboxedWorkflowRunner(),):handle = await environment.client.start_workflow(diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.pyindex 481ff6b6ef5f..590835b52ccb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py@@ -34,7 +34,7 @@ from django.conf import settingsimport structlogimport pyarrow.parquet as pq-from temporalio.exceptions import ActivityError, ApplicationError+from temporalio.exceptions import ActivityError, ApplicationError, CancelledErrorfrom posthog.dataclasses import frozenfrom posthog.exceptions_capture import capture_exception@@ -757,6 +757,7 @@ def _record(finished_at: str,ps: PerSourceResult,error: str | None,+ count_failure: bool = True,) -> None:record_person_property_sync_run(PersonPropertySyncRunRecord(@@ -775,6 +776,7 @@ def _record(produced=ps.produced,skipped_missing_person=ps.skipped_missing_person,error=error,+ count_failure=count_failure,))@@ -859,6 +861,7 @@ async def record_failed_runs(started_at: str,finished_at: str,error: str,+ count_failure: bool = True,) -> None:"""Persist a failed run row per source the binding feeds, so a failure is visible in the UI (notonly in error tracking). Never raises — this runs on the already-failing path, so any error here@@ -879,6 +882,7 @@ async def record_failed_runs(finished_at=finished_at,ps=PerSourceResult(source_id=str(source.source_id)),error=error,+ count_failure=count_failure,)except Exception as e:logger.exception(@@ -896,6 +900,13 @@ async def record_failed_runs(_INTERRUPTION_FAILURE_TYPES = frozenset({"WorkerShutdown", "ActivityPause", "ActivityReset"})+def is_worker_interruption(error: ActivityError) -> bool:+ cause = error.cause+ return isinstance(cause, CancelledError) or (+ isinstance(cause, ApplicationError) and cause.type in _INTERRUPTION_FAILURE_TYPES+ )++def failed_run_error(error: ActivityError, next_step: str) -> str:"""The error to show on a run whose activity gave up. An exception the activity raised keeps itsown message. A timeout, cancellation or worker shutdown says that the run was interrupted."""