Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): retry interrupted property backfills (#124043)

Jovan Sakovic4eb710b4ed83
diff --git a/products/customer_analytics/backend/logic/person_property_runs.py b/products/customer_analytics/backend/logic/person_property_runs.py
index bef71fe3e798..6f3735d760db 100644
--- a/products/customer_analytics/backend/logic/person_property_runs.py
+++ b/products/customer_analytics/backend/logic/person_property_runs.py
@@ -145,7 +145,7 @@ def record_sync_run(record: PersonPropertySyncRunRecord) -> None:
source,
error=record.error,
disable_event_id=record.job_id or str(run.id),
- count_failure=not already_failed,
+ count_failure=record.count_failure and not already_failed,
)
log.warning(
"person-property run recorded: failed",
diff --git a/products/customer_analytics/backend/logic/person_property_runs_test.py b/products/customer_analytics/backend/logic/person_property_runs_test.py
index d548cae582d8..04da8d173697 100644
--- a/products/customer_analytics/backend/logic/person_property_runs_test.py
+++ b/products/customer_analytics/backend/logic/person_property_runs_test.py
@@ -2,7 +2,7 @@ from datetime import UTC, datetime
from uuid import uuid4
from posthog.test.base import APIBaseTest
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
from parameterized import parameterized
@@ -87,15 +87,22 @@ class TestRecordSyncRun(TeamScopedTestMixin, APIBaseTest):
assert self.source.last_sync_error == "boom"
assert self.source.is_enabled is True
- def test_auto_disables_at_failure_cap(self):
- # Regression: a source failing forever should stop syncing, matching the account path's cap.
+ @parameterized.expand([(True,), (False,)])
+ @patch(f"{HEALTH_SERVICE}.notify_source_auto_disabled")
+ def test_auto_disables_at_failure_cap(self, count_failure: bool, mock_notify: MagicMock) -> None:
self.source.consecutive_failures = MAX_CONSECUTIVE_SYNC_FAILURES - 1
self.source.save()
- record_sync_run(self._record(status="failed", error="boom"))
+ with self.captureOnCommitCallbacks(execute=True):
+ record_sync_run(self._record(status="failed", error="boom", count_failure=count_failure))
self.source.refresh_from_db()
- assert self.source.is_enabled is False
+ assert self.source.is_enabled is (not count_failure)
+ assert self.source.consecutive_failures == MAX_CONSECUTIVE_SYNC_FAILURES - 1 + int(count_failure)
+ assert self.source.last_sync_error == "boom"
+ run = CustomPropertySyncRun.objects.for_team(self.team.id).get(source_id=self.source.id)
+ assert run.status == "failed" and run.error == "boom"
+ assert mock_notify.call_count == int(count_failure)
def test_unknown_source_is_a_noop(self):
# A run for a since-deleted source must not crash or orphan a row.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/external_product_hooks.py b/products/warehouse_sources/backend/temporal/data_imports/external_product_hooks.py
index 4e814b15ff34..cec1015cb93d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/external_product_hooks.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/external_product_hooks.py
@@ -356,6 +356,7 @@ class PersonPropertyFailedRunInputs:
trigger: str
started_at: str
error: str
+ count_failure: bool = True
@property
def binding(self) -> WarehouseBinding:
@@ -432,6 +433,7 @@ class PersonPropertySyncRunRecord:
produced: int
skipped_missing_person: int
error: str | None
+ count_failure: bool = True
PersonPropertySyncRecorder = Callable[[PersonPropertySyncRunRecord], None]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job.py
index 9f1c56ac4aec..b64c8d857116 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job.py
@@ -24,7 +24,7 @@ import structlog
from prometheus_client import Counter, Histogram
from temporalio import activity, workflow
from temporalio.common import RetryPolicy
-from temporalio.exceptions import ActivityError
+from temporalio.exceptions import ActivityError, ApplicationError, TimeoutError, TimeoutType
from posthog.exceptions_capture import capture_exception
from posthog.temporal.common.base import PostHogWorkflow
@@ -36,6 +36,7 @@ from products.warehouse_sources.backend.temporal.data_imports.external_product_h
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.person_property_sync import (
failed_run_error,
+ is_worker_interruption,
record_completed_runs,
record_failed_runs,
record_started_runs,
@@ -46,6 +47,7 @@ logger = structlog.get_logger(__name__)
INTERRUPTED_RUN_NEXT_STEP = "Click Backfill to try again."
RECORD_FAILED_RUN_PATCH = "person-property-backfill-record-failed-run-2026-10"
+RETRY_INTERRUPTED_RUN_PATCH = "person-property-backfill-retry-interrupted-run-2026-10"
PERSON_PROPERTY_BACKFILL_TOTAL = Counter(
"warehouse_person_property_backfill_total",
@@ -147,6 +149,7 @@ async def record_failed_person_property_backfill_runs_activity(inputs: PersonPro
started_at=inputs.started_at,
finished_at=datetime.now(UTC).isoformat(),
error=inputs.error,
+ count_failure=inputs.count_failure,
)
PERSON_PROPERTY_BACKFILL_TOTAL.labels(team_id=str(inputs.team_id), outcome="failed").inc()
@@ -187,14 +190,25 @@ class BackfillWarehousePersonPropertiesWorkflow(PostHogWorkflow):
started_at = workflow.now()
records_failure = workflow.patched(RECORD_FAILED_RUN_PATCH)
try:
- await workflow.execute_activity(
- backfill_warehouse_person_properties_activity,
- dataclasses.replace(activity_inputs, workflow_records_failure=records_failure),
- start_to_close_timeout=timedelta(hours=6),
- heartbeat_timeout=timedelta(minutes=5),
- # A one-off full-table read: don't silently re-scan the whole table on a transient error.
- retry_policy=RetryPolicy(maximum_attempts=1),
- )
+ max_attempts = 3 if workflow.patched(RETRY_INTERRUPTED_RUN_PATCH) else 1
+ for attempt in range(max_attempts):
+ try:
+ await workflow.execute_activity(
+ backfill_warehouse_person_properties_activity,
+ dataclasses.replace(activity_inputs, workflow_records_failure=records_failure),
+ start_to_close_timeout=timedelta(hours=6),
+ heartbeat_timeout=timedelta(minutes=5),
+ # A full-table read must not retry application errors or the six-hour timeout.
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ )
+ break
+ except ActivityError as error:
+ cause = error.cause
+ can_retry = (isinstance(cause, ApplicationError) and cause.type == "WorkerShutdown") or (
+ isinstance(cause, TimeoutError) and cause.type == TimeoutType.HEARTBEAT
+ )
+ if not can_retry or attempt == max_attempts - 1:
+ raise
except ActivityError as e:
if records_failure:
try:
@@ -208,6 +222,7 @@ class BackfillWarehousePersonPropertiesWorkflow(PostHogWorkflow):
trigger=activity_inputs.trigger,
started_at=started_at.isoformat(),
error=failed_run_error(e, INTERRUPTED_RUN_NEXT_STEP),
+ count_failure=not is_worker_interruption(e),
),
start_to_close_timeout=timedelta(minutes=5),
# The recorder swallows its own errors. A retry could only follow a write
diff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job_test.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job_test.py
index 712499d4fff5..c0f1609375dc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job_test.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job_test.py
@@ -1,17 +1,19 @@
import uuid
import asyncio
import dataclasses
+from collections.abc import Callable
from datetime import UTC, datetime
import pytest
from unittest.mock import AsyncMock, patch
-from temporalio.exceptions import ActivityError
+from temporalio.exceptions import ActivityError, ApplicationError, TimeoutError, TimeoutType
from temporalio.testing import ActivityEnvironment
from products.warehouse_sources.backend.temporal.data_imports import person_property_backfill_job as bj
from products.warehouse_sources.backend.temporal.data_imports.external_product_hooks import (
PersonPropertyBackfillActivityInputs,
+ PersonPropertyFailedRunInputs,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.person_property_sync import SyncResult
@@ -26,9 +28,8 @@ def _inputs(team_id: int) -> PersonPropertyBackfillActivityInputs:
)
-def _activity_error() -> ActivityError:
- """What ``execute_activity`` raises once the activity exhausts its single attempt."""
- return ActivityError(
+def _activity_error(cause: Exception | None = None) -> ActivityError:
+ error = ActivityError(
"backfill failed",
scheduled_event_id=1,
started_event_id=2,
@@ -37,6 +38,8 @@ def _activity_error() -> ActivityError:
activity_id="1",
retry_state=None,
)
+ error.__cause__ = cause
+ return error
@pytest.mark.django_db(transaction=True)
@@ -215,3 +218,62 @@ async def test_a_failed_run_with_nothing_queued_fails_the_workflow() -> None:
await runner.run(seed)
assert observed == [only]
+
+
+ "cause,failures,patch_enabled,expected_attempts",
+ [
+ (ApplicationError("Worker stopped", type="WorkerShutdown"), 1, True, 2),
+ (ApplicationError("Worker stopped", type="WorkerShutdown"), 2, True, 3),
+ (ApplicationError("Worker stopped", type="WorkerShutdown"), 3, True, 3),
+ (TimeoutError("No heartbeat", type=TimeoutType.HEARTBEAT, last_heartbeat_details=[]), 1, True, 2),
+ (TimeoutError("No heartbeat", type=TimeoutType.HEARTBEAT, last_heartbeat_details=[]), 3, True, 3),
+ (TimeoutError("Too slow", type=TimeoutType.START_TO_CLOSE, last_heartbeat_details=[]), 1, True, 1),
+ (ApplicationError("Invalid mapping", type="ValueError"), 1, True, 1),
+ (ApplicationError("Worker stopped", type="WorkerShutdown"), 1, False, 1),
+ ],
+)
+async def test_interrupted_backfill_retries_only_worker_shutdowns_and_heartbeat_timeouts(
+ cause: Exception, failures: int, patch_enabled: bool, expected_attempts: int
+) -> None:
+ inputs = _inputs(900006)
+ runner = bj.BackfillWarehousePersonPropertiesWorkflow(inputs)
+ attempts = 0
+ failed_records: list[PersonPropertyFailedRunInputs] = []
+
+ async def execute_activity(
+ activity_fn: Callable[..., object],
+ activity_inputs: PersonPropertyBackfillActivityInputs | PersonPropertyFailedRunInputs,
+ **_kwargs: object,
+ ) -> None:
+ nonlocal attempts
+ if activity_fn is bj.record_failed_person_property_backfill_runs_activity:
+ assert isinstance(activity_inputs, PersonPropertyFailedRunInputs)
+ failed_records.append(activity_inputs)
+ return
+ attempts += 1
+ if attempts <= failures:
+ raise _activity_error(cause)
+
+ def patched(patch_id: str) -> bool:
+ return patch_enabled if patch_id == bj.RETRY_INTERRUPTED_RUN_PATCH else True
+
+ with (
+ patch.object(bj.workflow, "execute_activity", AsyncMock(side_effect=execute_activity)),
+ patch.object(bj.workflow, "wait_condition", AsyncMock()),
+ patch.object(bj.workflow, "now", return_value=datetime(2026, 10, 1, tzinfo=UTC)),
+ patch.object(bj.workflow, "patched", side_effect=patched),
+ ):
+ if failures >= expected_attempts:
+ with pytest.raises(ActivityError):
+ await runner.run(inputs)
+ else:
+ await runner.run(inputs)
+
+ assert attempts == expected_attempts
+ assert len(failed_records) == (1 if failures >= expected_attempts else 0)
+ if failed_records:
+ assert failed_records[0].count_failure == (
+ not (isinstance(cause, ApplicationError) and cause.type == "WorkerShutdown")
+ )
diff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job.py
index e0543e1b46b8..b23742187edf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job.py
@@ -36,6 +36,7 @@ from products.warehouse_sources.backend.temporal.data_imports.external_product_h
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.person_property_sync import (
failed_run_error,
+ is_worker_interruption,
record_completed_runs,
record_failed_runs,
record_started_runs,
@@ -44,7 +45,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.per
logger = structlog.get_logger(__name__)
-INTERRUPTED_RUN_NEXT_STEP = "The next sync will try again."
+INTERRUPTED_RUN_NEXT_STEP = "Check the source status, then click Sync now to try again."
# Three recorder attempts of up to 5 minutes each.
RECORD_FAILED_RUN_RESERVE = timedelta(minutes=15)
RECORD_FAILED_RUN_PATCH = "person-property-sync-record-failed-run-2026-10"
@@ -149,6 +150,7 @@ async def record_failed_person_property_sync_runs_activity(inputs: PersonPropert
started_at=inputs.started_at,
finished_at=datetime.now(UTC).isoformat(),
error=inputs.error,
+ count_failure=inputs.count_failure,
)
PERSON_PROPERTY_SYNC_TOTAL.labels(team_id=str(inputs.team_id), outcome="failed").inc()
@@ -191,6 +193,7 @@ class SyncWarehousePersonPropertiesWorkflow(PostHogWorkflow):
trigger="scheduled",
started_at=info.start_time.isoformat(),
error=failed_run_error(e, INTERRUPTED_RUN_NEXT_STEP),
+ count_failure=not is_worker_interruption(e),
),
start_to_close_timeout=timedelta(minutes=5),
retry_policy=RetryPolicy(maximum_attempts=3),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job_test.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job_test.py
index 3f86def1da59..ff16d355c333 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job_test.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job_test.py
@@ -1,4 +1,5 @@
import uuid
+import dataclasses
from datetime import timedelta
import pytest
@@ -7,20 +8,23 @@ from unittest.mock import AsyncMock, MagicMock, patch
from temporalio import activity, workflow
from temporalio.client import WorkflowFailureError
from temporalio.common import RetryPolicy
-from temporalio.exceptions import ApplicationError
+from temporalio.exceptions import ActivityError, ApplicationError
from temporalio.testing import ActivityEnvironment, WorkflowEnvironment
from temporalio.worker import Replayer, UnsandboxedWorkflowRunner, Worker
from products.warehouse_sources.backend.temporal.data_imports.external_product_hooks import (
PersonPropertyBackfillActivityInputs,
+ PersonPropertyFailedRunInputs,
PersonPropertySyncActivityInputs,
PersonPropertySyncRunRecord,
PersonPropertySyncSource,
)
from products.warehouse_sources.backend.temporal.data_imports.person_property_backfill_job import (
PERSON_PROPERTY_BACKFILL_ACTIVITIES,
+ RECORD_FAILED_RUN_PATCH,
BackfillWarehousePersonPropertiesWorkflow,
backfill_warehouse_person_properties_activity,
+ record_failed_person_property_backfill_runs_activity,
)
from products.warehouse_sources.backend.temporal.data_imports.person_property_sync_job import (
PERSON_PROPERTY_SYNC_ACTIVITIES,
@@ -159,6 +163,7 @@ async def test_run_is_marked_failed_when_the_activity_dies_without_its_own_handl
assert failed[0].job_id == expected_job_id
assert (failed[0].error or "").startswith("This run was interrupted before it finished.")
assert failed[0].finished_at
+ assert failed[0].count_failure is False
@workflow.defn(name="sync-warehouse-person-properties")
@@ -185,6 +190,44 @@ class _PrePatchBackfillWorkflow:
)
[email protected](name="record_failed_person_property_backfill_runs_activity")
+async def _discard_backfill_failure(_inputs: PersonPropertyFailedRunInputs) -> None:
+ pass
+
+
[email protected](name="backfill-warehouse-person-properties")
+class _BeforeInterruptionRetryBackfillWorkflow:
+ @workflow.run
+ async def run(self, inputs: PersonPropertyBackfillActivityInputs) -> None:
+ started_at = workflow.now()
+ records_failure = workflow.patched(RECORD_FAILED_RUN_PATCH)
+ try:
+ await workflow.execute_activity(
+ backfill_warehouse_person_properties_activity,
+ dataclasses.replace(inputs, workflow_records_failure=records_failure),
+ start_to_close_timeout=timedelta(hours=6),
+ heartbeat_timeout=timedelta(minutes=5),
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ )
+ except ActivityError:
+ if records_failure:
+ await workflow.execute_activity(
+ record_failed_person_property_backfill_runs_activity,
+ PersonPropertyFailedRunInputs(
+ team_id=inputs.team_id,
+ schema_id=inputs.schema_id,
+ saved_query_id=inputs.saved_query_id,
+ job_id=None,
+ trigger=inputs.trigger,
+ started_at=started_at.isoformat(),
+ error="This run was interrupted before it finished. Click Backfill to try again.",
+ ),
+ start_to_close_timeout=timedelta(minutes=5),
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ )
+ raise
+
+
@pytest.mark.parametrize(
"pre_patch_workflow,workflow_cls,killed_activity,inputs",
[
@@ -216,6 +259,19 @@ class _PrePatchBackfillWorkflow:
saved_query_id=uuid.UUID("00000000-0000-4000-8000-000000000001"),
),
),
+ (
+ _BeforeInterruptionRetryBackfillWorkflow,
+ BackfillWarehousePersonPropertiesWorkflow,
+ _backfill_activity_killed_by_worker_shutdown,
+ PersonPropertyBackfillActivityInputs(
+ team_id=1,
+ schema_id=None,
+ source_type="saved_query",
+ schema_name="example_view",
+ trigger="manual",
+ saved_query_id=uuid.UUID("00000000-0000-4000-8000-000000000001"),
+ ),
+ ),
],
)
async def test_failed_history_from_before_the_failed_run_record_still_replays(
@@ -227,7 +283,7 @@ async def test_failed_history_from_before_the_failed_run_record_still_replays(
environment.client,
task_queue=task_queue,
workflows=[pre_patch_workflow],
- activities=[killed_activity],
+ activities=[killed_activity, _discard_backfill_failure],
workflow_runner=UnsandboxedWorkflowRunner(),
):
handle = await environment.client.start_workflow(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py
index 481ff6b6ef5f..590835b52ccb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py
@@ -34,7 +34,7 @@ from django.conf import settings
import structlog
import pyarrow.parquet as pq
-from temporalio.exceptions import ActivityError, ApplicationError
+from temporalio.exceptions import ActivityError, ApplicationError, CancelledError
from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
@@ -757,6 +757,7 @@ def _record(
finished_at: str,
ps: PerSourceResult,
error: str | None,
+ count_failure: bool = True,
) -> None:
record_person_property_sync_run(
PersonPropertySyncRunRecord(
@@ -775,6 +776,7 @@ def _record(
produced=ps.produced,
skipped_missing_person=ps.skipped_missing_person,
error=error,
+ count_failure=count_failure,
)
)
@@ -859,6 +861,7 @@ async def record_failed_runs(
started_at: str,
finished_at: str,
error: str,
+ count_failure: bool = True,
) -> None:
"""Persist a failed run row per source the binding feeds, so a failure is visible in the UI (not
only in error tracking). Never raises — this runs on the already-failing path, so any error here
@@ -879,6 +882,7 @@ async def record_failed_runs(
finished_at=finished_at,
ps=PerSourceResult(source_id=str(source.source_id)),
error=error,
+ count_failure=count_failure,
)
except Exception as e:
logger.exception(
@@ -896,6 +900,13 @@ async def record_failed_runs(
_INTERRUPTION_FAILURE_TYPES = frozenset({"WorkerShutdown", "ActivityPause", "ActivityReset"})
+def is_worker_interruption(error: ActivityError) -> bool:
+ cause = error.cause
+ return isinstance(cause, CancelledError) or (
+ isinstance(cause, ApplicationError) and cause.type in _INTERRUPTION_FAILURE_TYPES
+ )
+
+
def failed_run_error(error: ActivityError, next_step: str) -> str:
"""The error to show on a run whose activity gave up. An exception the activity raised keeps its
own message. A timeout, cancellation or worker shutdown says that the run was interrupted."""