Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): mark person property runs failed when interrupted (#122005)

Jovan Sakovica91cc9763ec8
diff --git a/products/warehouse_sources/backend/temporal/data_imports/external_product_hooks.py b/products/warehouse_sources/backend/temporal/data_imports/external_product_hooks.py
index d68b2fdba590..4e814b15ff34 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/external_product_hooks.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/external_product_hooks.py
@@ -324,6 +324,9 @@ class PersonPropertyBackfillActivityInputs:
# Signal-with-start uses an empty workflow seed, then supplies the real activity input via its
# start signal. The default preserves replay for executions started before that trigger existed.
skip_initial_run: bool = False
+ # True when the workflow records a failed run itself. A workflow from before that change leaves it
+ # False, so the activity still records its own failure for it during a rolling deploy.
+ workflow_records_failure: bool = False
@property
def binding(self) -> WarehouseBinding:
@@ -341,6 +344,24 @@ class PersonPropertyBackfillActivityInputs:
}
[email protected](frozen=True)
+class PersonPropertyFailedRunInputs:
+ """Payload the sync and backfill workflows send to record a run as failed once its activity
+ has given up. The activity cannot do this itself when it was timed out, cancelled or killed."""
+
+ team_id: int
+ schema_id: uuid.UUID | None
+ saved_query_id: uuid.UUID | None
+ job_id: str | None
+ trigger: str
+ started_at: str
+ error: str
+
+ @property
+ def binding(self) -> WarehouseBinding:
+ return _binding_from_ids(self.schema_id, self.saved_query_id)
+
+
# --- Data-quality checks gate ---------------------------------------------------------
DataQualityChecksGate = Callable[[int, "str | uuid.UUID"], bool]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job.py
index 47cd3263cff9..9f1c56ac4aec 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job.py
@@ -32,8 +32,10 @@ from posthog.temporal.common.heartbeat import LivenessHeartbeater as Heartbeater
from products.warehouse_sources.backend.temporal.data_imports.external_product_hooks import (
PersonPropertyBackfillActivityInputs,
+ PersonPropertyFailedRunInputs,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.person_property_sync import (
+ failed_run_error,
record_completed_runs,
record_failed_runs,
record_started_runs,
@@ -42,9 +44,12 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.per
logger = structlog.get_logger(__name__)
+INTERRUPTED_RUN_NEXT_STEP = "Click Backfill to try again."
+RECORD_FAILED_RUN_PATCH = "person-property-backfill-record-failed-run-2026-10"
+
PERSON_PROPERTY_BACKFILL_TOTAL = Counter(
"warehouse_person_property_backfill_total",
- "Person-property backfill activity attempts by outcome",
+ "Person-property backfill runs by outcome",
labelnames=["team_id", "outcome"],
)
@@ -82,18 +87,21 @@ async def backfill_warehouse_person_properties_activity(inputs: PersonPropertyBa
async with Heartbeater():
result = await run_person_property_backfill(team_id=inputs.team_id, binding=binding, trigger=inputs.trigger)
except Exception as e:
- PERSON_PROPERTY_BACKFILL_TOTAL.labels(team_id=str(inputs.team_id), outcome="failed").inc()
log.exception("Person-property backfill failed")
capture_exception(e)
- await record_failed_runs(
- team_id=inputs.team_id,
- binding=binding,
- job_id=None,
- trigger=inputs.trigger,
- started_at=started_at,
- finished_at=datetime.now(UTC).isoformat(),
- error=str(e),
- )
+ # A backfill run has no job_id, so a second failed record would reconcile a different
+ # running row. Record here only when the workflow will not.
+ if not inputs.workflow_records_failure:
+ PERSON_PROPERTY_BACKFILL_TOTAL.labels(team_id=str(inputs.team_id), outcome="failed").inc()
+ await record_failed_runs(
+ team_id=inputs.team_id,
+ binding=binding,
+ job_id=None,
+ trigger=inputs.trigger,
+ started_at=started_at,
+ finished_at=datetime.now(UTC).isoformat(),
+ error=str(e),
+ )
raise
await record_completed_runs(
@@ -129,6 +137,20 @@ async def backfill_warehouse_person_properties_activity(inputs: PersonPropertyBa
return dataclasses.asdict(result)
+async def record_failed_person_property_backfill_runs_activity(inputs: PersonPropertyFailedRunInputs) -> None:
+ await record_failed_runs(
+ team_id=inputs.team_id,
+ binding=inputs.binding,
+ job_id=inputs.job_id,
+ trigger=inputs.trigger,
+ started_at=inputs.started_at,
+ finished_at=datetime.now(UTC).isoformat(),
+ error=inputs.error,
+ )
+ PERSON_PROPERTY_BACKFILL_TOTAL.labels(team_id=str(inputs.team_id), outcome="failed").inc()
+
+
@workflow.defn(name="backfill-warehouse-person-properties")
class BackfillWarehousePersonPropertiesWorkflow(PostHogWorkflow):
@workflow.init
@@ -162,18 +184,40 @@ class BackfillWarehousePersonPropertiesWorkflow(PostHogWorkflow):
activity_inputs = self._pending_inputs
self._pending_inputs = None
assert activity_inputs is not None
+ started_at = workflow.now()
+ records_failure = workflow.patched(RECORD_FAILED_RUN_PATCH)
try:
await workflow.execute_activity(
backfill_warehouse_person_properties_activity,
- activity_inputs,
+ dataclasses.replace(activity_inputs, workflow_records_failure=records_failure),
start_to_close_timeout=timedelta(hours=6),
heartbeat_timeout=timedelta(minutes=5),
# A one-off full-table read: don't silently re-scan the whole table on a transient error.
retry_policy=RetryPolicy(maximum_attempts=1),
)
- except ActivityError:
- # The activity records its own failed runs before raising, so the failure already
- # reached the UI. A request that arrived while it ran was reported to its caller as
+ except ActivityError as e:
+ if records_failure:
+ try:
+ await workflow.execute_activity(
+ record_failed_person_property_backfill_runs_activity,
+ PersonPropertyFailedRunInputs(
+ team_id=activity_inputs.team_id,
+ schema_id=activity_inputs.schema_id,
+ saved_query_id=activity_inputs.saved_query_id,
+ job_id=None,
+ trigger=activity_inputs.trigger,
+ started_at=started_at.isoformat(),
+ error=failed_run_error(e, INTERRUPTED_RUN_NEXT_STEP),
+ ),
+ start_to_close_timeout=timedelta(minutes=5),
+ # The recorder swallows its own errors. A retry could only follow a write
+ # that already landed, and without a job_id it would add a second failed row.
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ )
+ except ActivityError:
+ # A lost failure record must not also drop a request queued behind this run.
+ workflow.logger.exception("Failed to record the failed person-property backfill run")
+ # A request that arrived while the activity ran was reported to its caller as
# queued, so serve that before giving up — a mapping edit must not die with the run
# it happened to overlap. With nothing pending, fail the workflow as before.
await workflow.wait_condition(workflow.all_handlers_finished)
@@ -189,4 +233,7 @@ class BackfillWarehousePersonPropertiesWorkflow(PostHogWorkflow):
PERSON_PROPERTY_BACKFILL_WORKFLOWS: list[type[PostHogWorkflow]] = [BackfillWarehousePersonPropertiesWorkflow]
-PERSON_PROPERTY_BACKFILL_ACTIVITIES = [backfill_warehouse_person_properties_activity]
+PERSON_PROPERTY_BACKFILL_ACTIVITIES = [
+ backfill_warehouse_person_properties_activity,
+ record_failed_person_property_backfill_runs_activity,
+]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job_test.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job_test.py
index 261c3d305cab..712499d4fff5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job_test.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_backfill_job_test.py
@@ -1,6 +1,7 @@
import uuid
import asyncio
import dataclasses
+from datetime import UTC, datetime
import pytest
from unittest.mock import AsyncMock, patch
@@ -97,6 +98,22 @@ class TestBackfillWarehousePersonPropertiesActivity:
assert stage_value("changed") == 0
assert stage_value("produced") == 0
+ @pytest.mark.parametrize("workflow_records_failure,expected_records", [(False, 1), (True, 0)])
+ async def test_activity_records_its_own_failure_only_for_workflows_that_do_not(
+ self, workflow_records_failure: bool, expected_records: int
+ ) -> None:
+ inputs = dataclasses.replace(_inputs(900003), workflow_records_failure=workflow_records_failure)
+ with (
+ patch.object(bj, "run_person_property_backfill", AsyncMock(side_effect=RuntimeError("boom"))),
+ patch.object(bj, "record_started_runs", AsyncMock()),
+ patch.object(bj, "record_failed_runs", AsyncMock()) as mock_failed,
+ patch.object(bj, "capture_exception"),
+ pytest.raises(RuntimeError),
+ ):
+ await ActivityEnvironment().run(bj.backfill_warehouse_person_properties_activity, inputs)
+
+ assert mock_failed.await_count == expected_records
+
@pytest.mark.asyncio
async def test_in_flight_backfill_runs_one_follow_up_with_the_latest_request() -> None:
@@ -120,6 +137,8 @@ async def test_in_flight_backfill_runs_one_follow_up_with_the_latest_request() -
with (
patch.object(bj.workflow, "execute_activity", AsyncMock(side_effect=execute_activity)),
patch.object(bj.workflow, "wait_condition", AsyncMock()),
+ patch.object(bj.workflow, "now", return_value=datetime(2026, 10, 1, tzinfo=UTC)),
+ patch.object(bj.workflow, "patched", return_value=False),
):
running = asyncio.create_task(runner.run(seed))
await first_started.wait()
@@ -132,7 +151,8 @@ async def test_in_flight_backfill_runs_one_follow_up_with_the_latest_request() -
@pytest.mark.asyncio
-async def test_a_failed_run_still_serves_a_request_that_arrived_while_it_ran() -> None:
[email protected]("records_failure", [False, True])
+async def test_a_failed_run_still_serves_a_request_that_arrived_while_it_ran(records_failure: bool) -> None:
# The API reports a request that lands mid-run as queued, so losing it with the run it happened
# to overlap would strand a mapping edit with nothing telling the user.
first = _inputs(900004)
@@ -145,7 +165,10 @@ async def test_a_failed_run_still_serves_a_request_that_arrived_while_it_ran() -
release_first = asyncio.Event()
observed: list[PersonPropertyBackfillActivityInputs] = []
- async def execute_activity(_activity, inputs, **_kwargs):
+ async def execute_activity(activity_fn, inputs, **_kwargs):
+ if activity_fn is bj.record_failed_person_property_backfill_runs_activity:
+ # The failed-run record can itself be lost to a worker shutdown.
+ raise _activity_error()
observed.append(inputs)
if len(observed) == 1:
first_started.set()
@@ -155,6 +178,9 @@ async def test_a_failed_run_still_serves_a_request_that_arrived_while_it_ran() -
with (
patch.object(bj.workflow, "execute_activity", AsyncMock(side_effect=execute_activity)),
patch.object(bj.workflow, "wait_condition", AsyncMock()),
+ patch.object(bj.workflow, "now", return_value=datetime(2026, 10, 1, tzinfo=UTC)),
+ patch.object(bj.workflow, "patched", return_value=records_failure),
+ patch.object(bj.workflow, "logger"),
):
running = asyncio.create_task(runner.run(seed))
await first_started.wait()
@@ -162,7 +188,7 @@ async def test_a_failed_run_still_serves_a_request_that_arrived_while_it_ran() -
release_first.set()
await running
- assert observed == [first, latest]
+ assert [i.schema_name for i in observed] == [first.schema_name, latest.schema_name]
@pytest.mark.asyncio
@@ -182,6 +208,8 @@ async def test_a_failed_run_with_nothing_queued_fails_the_workflow() -> None:
with (
patch.object(bj.workflow, "execute_activity", AsyncMock(side_effect=execute_activity)),
patch.object(bj.workflow, "wait_condition", AsyncMock()),
+ patch.object(bj.workflow, "now", return_value=datetime(2026, 10, 1, tzinfo=UTC)),
+ patch.object(bj.workflow, "patched", return_value=False),
pytest.raises(ActivityError),
):
await runner.run(seed)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job.py
index 8d36e6b228af..e0543e1b46b8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job.py
@@ -9,12 +9,14 @@ competes with the sync workers themselves; ``start_temporal_worker`` registers i
Failure semantics: the activity retries up to 3 times, then the (abandoned) child workflow fails
without affecting the sync. Nothing is lost — staged files are only cleared on success and the
dedup snapshot only advances per produced person, so the next sync's run re-reads and re-diffs.
-Every terminal failure is captured to error tracking and counted in the metrics below.
+Each failed attempt is captured to error tracking. The workflow then records the run as failed and
+counts it once in the metrics below, also when the activity was timed out, cancelled or killed.
"""
import json
import time
import dataclasses
+from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from typing import Any
@@ -22,15 +24,18 @@ import structlog
from prometheus_client import Counter, Histogram
from temporalio import activity, workflow
from temporalio.common import RetryPolicy
+from temporalio.exceptions import ActivityError
from posthog.exceptions_capture import capture_exception
from posthog.temporal.common.base import PostHogWorkflow
from posthog.temporal.common.heartbeat import LivenessHeartbeater as Heartbeater
from products.warehouse_sources.backend.temporal.data_imports.external_product_hooks import (
+ PersonPropertyFailedRunInputs,
PersonPropertySyncActivityInputs,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.person_property_sync import (
+ failed_run_error,
record_completed_runs,
record_failed_runs,
record_started_runs,
@@ -39,9 +44,14 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.per
logger = structlog.get_logger(__name__)
+INTERRUPTED_RUN_NEXT_STEP = "The next sync will try again."
+# Three recorder attempts of up to 5 minutes each.
+RECORD_FAILED_RUN_RESERVE = timedelta(minutes=15)
+RECORD_FAILED_RUN_PATCH = "person-property-sync-record-failed-run-2026-10"
+
PERSON_PROPERTY_SYNC_TOTAL = Counter(
"warehouse_person_property_sync_total",
- "Person-property sync activity attempts by outcome",
+ "Person-property sync runs by outcome",
labelnames=["team_id", "outcome"],
)
@@ -81,8 +91,8 @@ async def sync_warehouse_person_properties_activity(inputs: PersonPropertySyncAc
result = await run_person_property_sync(team_id=inputs.team_id, binding=binding, job_id=inputs.job_id)
except Exception as e:
# Re-raised so Temporal retries; captured so a terminal failure is visible in error
- # tracking rather than dying silently in an abandoned child workflow.
- PERSON_PROPERTY_SYNC_TOTAL.labels(team_id=str(inputs.team_id), outcome="failed").inc()
+ # tracking rather than dying silently in an abandoned child workflow. The failed outcome is
+ # counted by the workflow once retries are exhausted, not here per attempt.
log.exception("Person-property sync failed")
capture_exception(e, inputs.properties_to_log)
await record_failed_runs(
@@ -129,6 +139,20 @@ async def sync_warehouse_person_properties_activity(inputs: PersonPropertySyncAc
return dataclasses.asdict(result)
+async def record_failed_person_property_sync_runs_activity(inputs: PersonPropertyFailedRunInputs) -> None:
+ await record_failed_runs(
+ team_id=inputs.team_id,
+ binding=inputs.binding,
+ job_id=inputs.job_id,
+ trigger=inputs.trigger,
+ started_at=inputs.started_at,
+ finished_at=datetime.now(UTC).isoformat(),
+ error=inputs.error,
+ )
+ PERSON_PROPERTY_SYNC_TOTAL.labels(team_id=str(inputs.team_id), outcome="failed").inc()
+
+
@workflow.defn(name="sync-warehouse-person-properties")
class SyncWarehousePersonPropertiesWorkflow(PostHogWorkflow):
@staticmethod
@@ -138,14 +162,47 @@ class SyncWarehousePersonPropertiesWorkflow(PostHogWorkflow):
@workflow.run
async def run(self, inputs: PersonPropertySyncActivityInputs) -> None:
- await workflow.execute_activity(
- sync_warehouse_person_properties_activity,
- inputs,
- start_to_close_timeout=timedelta(hours=6),
- heartbeat_timeout=timedelta(minutes=5),
- retry_policy=RetryPolicy(maximum_attempts=3),
- )
+ # Temporal terminates the workflow at its own deadline without running the except branch
+ # below, so the activity has to give up early enough to leave time to record the failure.
+ schedule_to_close_timeout = None
+ info = workflow.info()
+ if info.execution_timeout is not None:
+ elapsed = workflow.now() - info.start_time
+ schedule_to_close_timeout = info.execution_timeout - elapsed - RECORD_FAILED_RUN_RESERVE
+ try:
+ await workflow.execute_activity(
+ sync_warehouse_person_properties_activity,
+ inputs,
+ start_to_close_timeout=timedelta(hours=6),
+ schedule_to_close_timeout=schedule_to_close_timeout,
+ heartbeat_timeout=timedelta(minutes=5),
+ retry_policy=RetryPolicy(maximum_attempts=3),
+ )
+ except ActivityError as e:
+ if workflow.patched(RECORD_FAILED_RUN_PATCH):
+ try:
+ await workflow.execute_activity(
+ record_failed_person_property_sync_runs_activity,
+ PersonPropertyFailedRunInputs(
+ team_id=inputs.team_id,
+ schema_id=inputs.schema_id,
+ saved_query_id=inputs.saved_query_id,
+ job_id=inputs.job_id,
+ trigger="scheduled",
+ started_at=info.start_time.isoformat(),
+ error=failed_run_error(e, INTERRUPTED_RUN_NEXT_STEP),
+ ),
+ start_to_close_timeout=timedelta(minutes=5),
+ retry_policy=RetryPolicy(maximum_attempts=3),
+ )
+ except ActivityError:
+ # Fail with the sync's own error, not the recorder's.
+ workflow.logger.exception("Failed to record the failed person-property sync run")
+ raise
PERSON_PROPERTY_SYNC_WORKFLOWS = [SyncWarehousePersonPropertiesWorkflow]
-PERSON_PROPERTY_SYNC_ACTIVITIES = [sync_warehouse_person_properties_activity]
+PERSON_PROPERTY_SYNC_ACTIVITIES: list[Callable[..., Any]] = [
+ sync_warehouse_person_properties_activity,
+ record_failed_person_property_sync_runs_activity,
+]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job_test.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job_test.py
index 859715381e44..3f86def1da59 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job_test.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_sync_job_test.py
@@ -1,14 +1,30 @@
import uuid
+from datetime import timedelta
import pytest
from unittest.mock import AsyncMock, MagicMock, patch
-from temporalio.testing import ActivityEnvironment
+from temporalio import activity, workflow
+from temporalio.client import WorkflowFailureError
+from temporalio.common import RetryPolicy
+from temporalio.exceptions import ApplicationError
+from temporalio.testing import ActivityEnvironment, WorkflowEnvironment
+from temporalio.worker import Replayer, UnsandboxedWorkflowRunner, Worker
from products.warehouse_sources.backend.temporal.data_imports.external_product_hooks import (
+ PersonPropertyBackfillActivityInputs,
PersonPropertySyncActivityInputs,
+ PersonPropertySyncRunRecord,
+ PersonPropertySyncSource,
+)
+from products.warehouse_sources.backend.temporal.data_imports.person_property_backfill_job import (
+ PERSON_PROPERTY_BACKFILL_ACTIVITIES,
+ BackfillWarehousePersonPropertiesWorkflow,
+ backfill_warehouse_person_properties_activity,
)
from products.warehouse_sources.backend.temporal.data_imports.person_property_sync_job import (
+ PERSON_PROPERTY_SYNC_ACTIVITIES,
+ SyncWarehousePersonPropertiesWorkflow,
sync_warehouse_person_properties_activity,
)
@@ -54,3 +70,220 @@ class TestSyncWarehousePersonPropertiesActivityCapturesFailureContext:
captured_properties = mock_capture_exception.call_args[0][1]
assert captured_properties["source_type"] == "Stripe"
assert captured_properties["schema_name"] == "public.charges"
+
+
+HOOKS = "products.warehouse_sources.backend.temporal.data_imports.external_product_hooks"
+
+
[email protected](name="sync_warehouse_person_properties_activity")
+async def _sync_activity_killed_by_worker_shutdown(_inputs: PersonPropertySyncActivityInputs) -> dict:
+ raise ApplicationError("Worker is shutting down", type="WorkerShutdown", non_retryable=True)
+
+
[email protected](name="backfill_warehouse_person_properties_activity")
+async def _backfill_activity_killed_by_worker_shutdown(inputs: PersonPropertyBackfillActivityInputs) -> dict:
+ assert inputs.workflow_records_failure
+ raise ApplicationError("Worker is shutting down", type="WorkerShutdown", non_retryable=True)
+
+
+ "workflow_cls,replaced_activity,killed_activity,real_activities,inputs,expected_job_id",
+ [
+ (
+ SyncWarehousePersonPropertiesWorkflow,
+ sync_warehouse_person_properties_activity,
+ _sync_activity_killed_by_worker_shutdown,
+ PERSON_PROPERTY_SYNC_ACTIVITIES,
+ PersonPropertySyncActivityInputs(
+ team_id=1,
+ schema_id=None,
+ source_id=None,
+ job_id="job-1",
+ source_type="saved_query",
+ schema_name="example_view",
+ last_synced_at=None,
+ saved_query_id=uuid.UUID("00000000-0000-4000-8000-000000000001"),
+ ),
+ "job-1",
+ ),
+ (
+ BackfillWarehousePersonPropertiesWorkflow,
+ backfill_warehouse_person_properties_activity,
+ _backfill_activity_killed_by_worker_shutdown,
+ PERSON_PROPERTY_BACKFILL_ACTIVITIES,
+ PersonPropertyBackfillActivityInputs(
+ team_id=1,
+ schema_id=None,
+ source_type="saved_query",
+ schema_name="example_view",
+ trigger="manual",
+ saved_query_id=uuid.UUID("00000000-0000-4000-8000-000000000001"),
+ ),
+ None,
+ ),
+ ],
+)
+async def test_run_is_marked_failed_when_the_activity_dies_without_its_own_handler(
+ workflow_cls, replaced_activity, killed_activity, real_activities, inputs, expected_job_id
+) -> None:
+ recorded: list[PersonPropertySyncRunRecord] = []
+ source = PersonPropertySyncSource(
+ source_id="00000000-0000-4000-8000-000000000002",
+ definition_id="00000000-0000-4000-8000-000000000003",
+ key_column="distinct_id",
+ column_property_map={"plan": "plan_name"},
+ )
+ other_activities = [a for a in real_activities if a is not replaced_activity]
+ task_queue = str(uuid.uuid4())
+
+ with (
+ patch(f"{HOOKS}._person_property_sync_sources_resolver", lambda _team_id, _binding: [source]),
+ patch(f"{HOOKS}._person_property_sync_recorder", recorded.append),
+ ):
+ async with await WorkflowEnvironment.start_time_skipping() as environment:
+ async with Worker(
+ environment.client,
+ task_queue=task_queue,
+ workflows=[workflow_cls],
+ activities=[killed_activity, *other_activities],
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ):
+ with pytest.raises(WorkflowFailureError):
+ await environment.client.execute_workflow(
+ workflow_cls.run, inputs, id=str(uuid.uuid4()), task_queue=task_queue
+ )
+
+ failed = [r for r in recorded if r.status == "failed"]
+ assert len(failed) == 1
+ assert failed[0].source_id == source.source_id
+ assert failed[0].job_id == expected_job_id
+ assert (failed[0].error or "").startswith("This run was interrupted before it finished.")
+ assert failed[0].finished_at
+
+
[email protected](name="sync-warehouse-person-properties")
+class _PrePatchSyncWorkflow:
+ @workflow.run
+ async def run(self, inputs: PersonPropertySyncActivityInputs) -> None:
+ await workflow.execute_activity(
+ "sync_warehouse_person_properties_activity",
+ inputs,
+ start_to_close_timeout=timedelta(hours=6),
+ retry_policy=RetryPolicy(maximum_attempts=3),
+ )
+
+
[email protected](name="backfill-warehouse-person-properties")
+class _PrePatchBackfillWorkflow:
+ @workflow.run
+ async def run(self, inputs: PersonPropertyBackfillActivityInputs) -> None:
+ await workflow.execute_activity(
+ "backfill_warehouse_person_properties_activity",
+ inputs,
+ start_to_close_timeout=timedelta(hours=6),
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ )
+
+
+ "pre_patch_workflow,workflow_cls,killed_activity,inputs",
+ [
+ (
+ _PrePatchSyncWorkflow,
+ SyncWarehousePersonPropertiesWorkflow,
+ _sync_activity_killed_by_worker_shutdown,
+ PersonPropertySyncActivityInputs(
+ team_id=1,
+ schema_id=None,
+ source_id=None,
+ job_id="job-1",
+ source_type="saved_query",
+ schema_name="example_view",
+ last_synced_at=None,
+ saved_query_id=uuid.UUID("00000000-0000-4000-8000-000000000001"),
+ ),
+ ),
+ (
+ _PrePatchBackfillWorkflow,
+ BackfillWarehousePersonPropertiesWorkflow,
+ _backfill_activity_killed_by_worker_shutdown,
+ PersonPropertyBackfillActivityInputs(
+ team_id=1,
+ schema_id=None,
+ source_type="saved_query",
+ schema_name="example_view",
+ trigger="manual",
+ saved_query_id=uuid.UUID("00000000-0000-4000-8000-000000000001"),
+ ),
+ ),
+ ],
+)
+async def test_failed_history_from_before_the_failed_run_record_still_replays(
+ pre_patch_workflow, workflow_cls, killed_activity, inputs
+) -> None:
+ task_queue = str(uuid.uuid4())
+ async with await WorkflowEnvironment.start_time_skipping() as environment:
+ async with Worker(
+ environment.client,
+ task_queue=task_queue,
+ workflows=[pre_patch_workflow],
+ activities=[killed_activity],
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ):
+ handle = await environment.client.start_workflow(
+ pre_patch_workflow.run, inputs, id=str(uuid.uuid4()), task_queue=task_queue
+ )
+ with pytest.raises(WorkflowFailureError):
+ await handle.result()
+ history = await handle.fetch_history()
+
+ await Replayer(workflows=[workflow_cls], workflow_runner=UnsandboxedWorkflowRunner()).replay_workflow(history)
+
+
+async def test_sync_activity_deadline_leaves_time_to_record_the_failure_before_the_workflow_deadline() -> None:
+ inputs = PersonPropertySyncActivityInputs(
+ team_id=1,
+ schema_id=None,
+ source_id=None,
+ job_id="job-deadline",
+ source_type="saved_query",
+ schema_name="example_view",
+ last_synced_at=None,
+ saved_query_id=uuid.UUID("00000000-0000-4000-8000-000000000001"),
+ )
+ other_activities = [
+ a for a in PERSON_PROPERTY_SYNC_ACTIVITIES if a is not sync_warehouse_person_properties_activity
+ ]
+ task_queue = str(uuid.uuid4())
+ with (
+ patch(f"{HOOKS}._person_property_sync_sources_resolver", lambda _team_id, _binding: []),
+ patch(f"{HOOKS}._person_property_sync_recorder", lambda _record: None),
+ ):
+ async with await WorkflowEnvironment.start_time_skipping() as environment:
+ async with Worker(
+ environment.client,
+ task_queue=task_queue,
+ workflows=[SyncWarehousePersonPropertiesWorkflow],
+ activities=[_sync_activity_killed_by_worker_shutdown, *other_activities],
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ):
+ handle = await environment.client.start_workflow(
+ SyncWarehousePersonPropertiesWorkflow.run,
+ inputs,
+ id=str(uuid.uuid4()),
+ task_queue=task_queue,
+ execution_timeout=timedelta(hours=6),
+ )
+ with pytest.raises(WorkflowFailureError):
+ await handle.result()
+ history = await handle.fetch_history()
+
+ scheduled = [
+ e.activity_task_scheduled_event_attributes
+ for e in history.events
+ if e.activity_task_scheduled_event_attributes.activity_type.name == "sync_warehouse_person_properties_activity"
+ ]
+ assert len(scheduled) == 1
+ # The recorder needs up to 3 attempts of 5 minutes after the sync activity gives up.
+ deadline = scheduled[0].schedule_to_close_timeout.ToTimedelta()
+ assert timedelta(0) < deadline <= timedelta(hours=6) - timedelta(minutes=15)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py
index ee7f10416d84..481ff6b6ef5f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/person_property_sync.py
@@ -34,6 +34,7 @@ from django.conf import settings
import structlog
import pyarrow.parquet as pq
+from temporalio.exceptions import ActivityError, ApplicationError
from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
@@ -888,3 +889,17 @@ async def record_failed_runs(
trigger=trigger,
)
capture_exception(e, {"team_id": team_id, **_log_fields(binding), "trigger": trigger})
+
+
+# Failure types Temporal sets when it stops an activity from outside. The activity's own exception
+# handler does not run for these, so there is no message from the activity to show.
+_INTERRUPTION_FAILURE_TYPES = frozenset({"WorkerShutdown", "ActivityPause", "ActivityReset"})
+
+
+def failed_run_error(error: ActivityError, next_step: str) -> str:
+ """The error to show on a run whose activity gave up. An exception the activity raised keeps its
+ own message. A timeout, cancellation or worker shutdown says that the run was interrupted."""
+ cause = error.cause
+ if isinstance(cause, ApplicationError) and cause.type and cause.type not in _INTERRUPTION_FAILURE_TYPES:
+ return cause.message
+ return f"This run was interrupted before it finished. {next_step}"