:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(customer-analytics): add flagged account property fanout (#119580)
Arthur Moreira de Deus56405f4bfe7a
diff --git a/products/customer_analytics/backend/facade/temporal_contracts.py b/products/customer_analytics/backend/facade/temporal_contracts.pyindex dc36a9f2afcc..6ef31ca6e402 100644--- a/products/customer_analytics/backend/facade/temporal_contracts.py+++ b/products/customer_analytics/backend/facade/temporal_contracts.py@@ -27,6 +27,8 @@ class FinalizeAccountPropertySyncRunsInput:status: strphase: strerror: str | None = None+ # None closes open runs in both segments. Histories recorded before this field existed decode to None.+ segment: str | None = None@frozen@@ -37,6 +39,15 @@ class AccountPropertySyncInput:segment: str+@frozen+class AccountPropertySourceSyncInput:+ team_id: int+ saved_query_id: str+ job_id: str+ segment: str+ source_id: str++@frozenclass AccountPropertySegmentTelemetryInput:team_id: intdiff --git a/products/customer_analytics/backend/logic/account_property_fanout_flags.py b/products/customer_analytics/backend/logic/account_property_fanout_flags.pynew file mode 100644index 000000000000..83919554a87f--- /dev/null+++ b/products/customer_analytics/backend/logic/account_property_fanout_flags.py@@ -0,0 +1,30 @@+import posthoganalytics++from posthog.exceptions_capture import capture_exception+from posthog.models import Team++# Selects the per-source fanout workflow for new account-property segment syncs. The flag is+# created default-off, so an absent flag keeps every team on the legacy workflow.+ACCOUNT_PROPERTY_SYNC_FANOUT_FLAG = "warehouse-account-property-sync-fanout"+++def account_property_sync_fanout_enabled(team_id: int) -> bool:+ """Whether new segment syncs for the team use the fanout workflow, targeted by the team's organization.++ Fails closed: an unknown team, a missing flag, or any lookup or flag-service error selects the legacy workflow.+ """+ try:+ organization_id = str(Team.objects.only("organization_id").get(id=team_id).organization_id)+ enabled = posthoganalytics.feature_enabled(+ ACCOUNT_PROPERTY_SYNC_FANOUT_FLAG,+ organization_id,+ groups={"organization": organization_id},+ only_evaluate_locally=False,+ send_feature_flag_events=False,+ )+ except Team.DoesNotExist:+ return False+ except Exception as error:+ capture_exception(error)+ return False+ return enabled is Truediff --git a/products/customer_analytics/backend/logic/account_property_fanout_flags_test.py b/products/customer_analytics/backend/logic/account_property_fanout_flags_test.pynew file mode 100644index 000000000000..976fd57e30ef--- /dev/null+++ b/products/customer_analytics/backend/logic/account_property_fanout_flags_test.py@@ -0,0 +1,48 @@+from posthog.test.base import BaseTest+from unittest.mock import patch++from parameterized import parameterized++from products.customer_analytics.backend.logic.account_property_fanout_flags import (+ ACCOUNT_PROPERTY_SYNC_FANOUT_FLAG,+ account_property_sync_fanout_enabled,+)++_FEATURE_ENABLED = (+ "products.customer_analytics.backend.logic.account_property_fanout_flags.posthoganalytics.feature_enabled"+)+++class AccountPropertySyncFanoutFlagTest(BaseTest):+ @parameterized.expand(+ [+ ("enabled", True, True),+ ("disabled", False, False),+ ("missing_flag", None, False),+ ("flag_service_error", RuntimeError("flag service unavailable"), False),+ ]+ )+ def test_targets_the_team_organization_and_fails_closed(+ self, _name: str, flag_result: object, expected: bool+ ) -> None:+ organization_id = str(self.team.organization_id)++ with (+ patch(_FEATURE_ENABLED, side_effect=[flag_result]) as feature_enabled,+ patch("products.customer_analytics.backend.logic.account_property_fanout_flags.capture_exception"),+ ):+ assert account_property_sync_fanout_enabled(self.team.id) is expected++ feature_enabled.assert_called_once_with(+ ACCOUNT_PROPERTY_SYNC_FANOUT_FLAG,+ organization_id,+ groups={"organization": organization_id},+ only_evaluate_locally=False,+ send_feature_flag_events=False,+ )++ def test_unknown_team_selects_legacy_without_calling_the_flag_service(self) -> None:+ with patch(_FEATURE_ENABLED) as feature_enabled:+ assert account_property_sync_fanout_enabled(self.team.id + 10_000) is False++ feature_enabled.assert_not_called()diff --git a/products/customer_analytics/backend/logic/account_property_sync.py b/products/customer_analytics/backend/logic/account_property_sync.pyindex 3deff65b2a97..6f455d2cb4cc 100644--- a/products/customer_analytics/backend/logic/account_property_sync.py+++ b/products/customer_analytics/backend/logic/account_property_sync.py@@ -7,7 +7,7 @@ from dataclasses import fieldfrom datetime import date, datetime, timefrom decimal import Decimalfrom enum import StrEnum-from typing import Any, Protocol+from typing import TYPE_CHECKING, Any, Protocolfrom uuid import UUIDfrom django.db.models import QuerySet@@ -38,7 +38,7 @@ from products.customer_analytics.backend.models.custom_property_sync_run importSyncSegment as AccountPropertySyncSegment,SyncStatus,)-from products.data_warehouse.backend.facade.api import aget_s3_client+from products.data_warehouse.backend.facade.api import aget_s3_client, get_s3_clientfrom products.warehouse_sources.backend.facade.hooks import WarehouseBindingfrom products.warehouse_sources.backend.facade.temporal import (account_property_completion_prefix,@@ -46,6 +46,9 @@ from products.warehouse_sources.backend.facade.temporal import (account_property_snapshot_prefix,)+if TYPE_CHECKING:+ from s3fs import S3FileSystem+logger = structlog.get_logger(__name__)_ACCOUNT_LOOKUP_CHUNK_SIZE = 1_000@@ -178,8 +181,14 @@ def _parquet_batches(data: bytes) -> Iterator[pa.RecordBatch]:return pq.ParquetFile(io.BytesIO(data)).iter_batches(batch_size=_PARQUET_BATCH_SIZE)+def _projected_parquet_batches(s3_client: "S3FileSystem", path: str, columns: list[str]) -> Iterator[pa.RecordBatch]:+ # Read-ahead can fetch unselected column chunks, which defeats per-source projection.+ with s3_client.open(path, "rb", cache_type="none") as file:+ yield from pq.ParquetFile(file, pre_buffer=False).iter_batches(batch_size=_PARQUET_BATCH_SIZE, columns=columns)++async def _iter_parquet_row_batches(- team_id: int, binding: WarehouseBinding, job_id: str+ team_id: int, binding: WarehouseBinding, job_id: str, *, columns: list[str] | None = None) -> AsyncIterator[list[dict[str, Any]]]:prefix = account_property_job_staged_prefix(team_id, binding, job_id)async with aget_s3_client() as s3_client:@@ -190,8 +199,11 @@ async def _iter_parquet_row_batches(entries = listing.values() if isinstance(listing, dict) else listingfile_paths = sorted(entry["Key"] for entry in entries if entry.get("type") != "directory")for file_path in file_paths:- data = await s3_client._cat_file(_s3_uri(file_path))- batches = await asyncio.to_thread(_parquet_batches, data)+ if columns is None:+ data = await s3_client._cat_file(_s3_uri(file_path))+ batches = await asyncio.to_thread(_parquet_batches, data)+ else:+ batches = _projected_parquet_batches(get_s3_client(), _s3_uri(file_path), columns)while (batch := await asyncio.to_thread(next, batches, None)) is not None:yield await asyncio.to_thread(batch.to_pylist)@@ -476,9 +488,10 @@ async def _apply_staged_rows(states: list[SourceSyncState],counts: dict[str, int],log: FilteringBoundLogger,+ columns: list[str] | None = None,) -> None:batch_index = 0- batches = aiter(_iter_parquet_row_batches(team_id, binding, job_id))+ batches = aiter(_iter_parquet_row_batches(team_id, binding, job_id, columns=columns))while True:phase_started_at = asyncio.get_running_loop().time()try:@@ -728,7 +741,16 @@ async def run_account_property_source_sync(states = [state]await _load_prior_hashes(team_id=team_id, binding=binding, segment=segment, states=states, log=log)await _apply_staged_rows(- team_id=team_id, binding=binding, job_id=job_id, segment=segment, states=states, counts=counts, log=log+ team_id=team_id,+ binding=binding,+ job_id=job_id,+ segment=segment,+ states=states,+ counts=counts,+ log=log,+ columns=list(+ dict.fromkeys(column for column in (source.key_column, source.source_column) if column is not None)+ ),)await _persist_states(team_id=team_id, binding=binding, job_id=job_id, segment=segment, states=states, counts=counts, log=logdiff --git a/products/customer_analytics/backend/logic/account_property_sync_test.py b/products/customer_analytics/backend/logic/account_property_sync_test.pyindex fd6d9563ca28..0571845626dd 100644--- a/products/customer_analytics/backend/logic/account_property_sync_test.py+++ b/products/customer_analytics/backend/logic/account_property_sync_test.py@@ -1,3 +1,5 @@+import io+import hashlibfrom collections.abc import AsyncIteratorfrom contextlib import AbstractContextManager, ExitStackfrom datetime import UTC, datetime@@ -163,6 +165,7 @@ class AccountPropertySegmentTest(TeamScopedTestMixin, BaseTest):def _fanout_boundaries(self, fake: "_FakeS3") -> ExitStack:stack = ExitStack()stack.enter_context(_fake_s3_patch(fake))+ stack.enter_context(patch(f"{_MODULE}.get_s3_client", return_value=fake))stack.enter_context(patch(f"{_MODULE}._matching_account_ids", side_effect=_match_known_accounts))stack.enter_context(patch(f"{_MODULE}._apply_source_values", side_effect=_apply_all_values))return stack@@ -423,7 +426,7 @@ class AccountPropertySegmentTest(TeamScopedTestMixin, BaseTest):workflow_run_id="00000000-0000-4000-8000-000000000001",)- async def batches(*args):+ async def batches(*args: object, **_kwargs: object) -> AsyncIterator[list[dict[str, Any]]]:yield [{"external_id": "acme", "plan": "enterprise"}]with (@@ -467,7 +470,7 @@ class AccountPropertySegmentTest(TeamScopedTestMixin, BaseTest):client._ls = AsyncMock(return_value=[{"Key": "prefix/deleted.parquet", "type": "file"}])client._cat_file = AsyncMock(side_effect=FileNotFoundError())- async def no_batches(*args: object) -> AsyncIterator[list[dict[str, Any]]]:+ async def no_batches(*args: object, **_kwargs: object) -> AsyncIterator[list[dict[str, Any]]]:for _ in range(0):yield []@@ -565,7 +568,7 @@ class AccountPropertySegmentTest(TeamScopedTestMixin, BaseTest):source.is_enabled = Falsesource.save(update_fields=["is_enabled"])- async def no_batches(*args):+ async def no_batches(*args: object, **_kwargs: object) -> AsyncIterator[list[dict[str, Any]]]:for _ in range(0):yield []@@ -635,7 +638,7 @@ async def test_each_staged_batch_is_shared_across_sources(source_columns: tuple[sources.append(source)batch_calls = 0- async def batches(*args):+ async def batches(*args: object, **_kwargs: object) -> AsyncIterator[list[dict[str, Any]]]:nonlocal batch_callsbatch_calls += 1yield [{"organization_id": "org-1", "value_0": 1, "value_1": 2}]@@ -672,7 +675,7 @@ async def test_sync_records_each_phase_duration(invalid_source_values: bool) ->source.source_column = "value"phase_duration_recorder = MagicMock()- async def batches(*args):+ async def batches(*args: object, **_kwargs: object) -> AsyncIterator[list[dict[str, Any]]]:yield [{"organization_id": "org-1", "value": 1}, {"organization_id": "org-2", "value": "invalid"}]with (@@ -725,27 +728,49 @@ async def test_sync_records_each_phase_duration(invalid_source_values: bool) ->]+class _ReadTrackingFile(io.BytesIO):+ def __init__(self, data: bytes) -> None:+ super().__init__(data)+ self.bytes_read = 0++ def read(self, size: int | None = -1, /) -> bytes:+ data = super().read(size)+ self.bytes_read += len(data)+ return data++@pytest.mark.asyncio-async def test_staged_parquet_is_decoded_in_bounded_batches() -> None:[email protected]("columns", [None, ["organization_id", "plan"], ["organization_id", "missing"]])+async def test_staged_parquet_is_decoded_in_bounded_batches(columns: list[str] | None) -> None:binding = saved_query_binding("019f0000-0000-7000-8000-000000000001")- table = pa.table({"organization_id": [f"org-{index}" for index in range(50_001)]})+ table = pa.table(+ {+ "organization_id": [f"org-{index}" for index in range(50_001)],+ "plan": ["gold"] * 50_001,+ "unused": [hashlib.sha256(str(index).encode()).digest() for index in range(50_001)],+ }+ )buffer = pa.BufferOutputStream()pq.write_table(table, buffer)+ data = buffer.getvalue().to_pybytes()+ projected_file = _ReadTrackingFile(data)client = MagicMock()client._ls = AsyncMock(return_value=[{"Key": "prefix/chunk.parquet", "type": "file"}])- client._cat_file = AsyncMock(return_value=buffer.getvalue().to_pybytes())+ client._cat_file = AsyncMock(return_value=data)+ client.open.return_value = projected_file- with patch(f"{_MODULE}.aget_s3_client", return_value=_S3ClientContext(client)):- batch_sizes = [- len(rows)- async for rows in _iter_parquet_row_batches(- 7,- binding,- "job-1",- )- ]+ with (+ patch(f"{_MODULE}.aget_s3_client", return_value=_S3ClientContext(client)),+ patch(f"{_MODULE}.get_s3_client", return_value=client),+ ):+ batches = [rows async for rows in _iter_parquet_row_batches(7, binding, "job-1", columns=columns)]- assert batch_sizes == [50_000, 1]+ assert [len(rows) for rows in batches] == [50_000, 1]+ expected_columns = set(table.column_names) if columns is None else set(columns) & set(table.column_names)+ assert [set(rows[0]) for rows in batches] == [expected_columns, expected_columns]+ if columns is not None:+ client._cat_file.assert_not_awaited()+ assert 0 < projected_file.bytes_read < len(data)@pytest.mark.asyncio@@ -799,6 +824,9 @@ class _FakeS3:raise FileNotFoundError(path)return self.store[key]+ def open(self, path: str, mode: str, *, cache_type: str) -> io.BytesIO:+ return io.BytesIO(self.store[aps._s3_key(path)])+async def _pipe_file(self, path: str, data: bytes) -> None:self._clock += 1key = aps._s3_key(path)diff --git a/products/customer_analytics/backend/temporal/account_property_sync.py b/products/customer_analytics/backend/temporal/account_property_sync.pyindex 9e3b666d3eb0..e7e696ddc1cc 100644--- a/products/customer_analytics/backend/temporal/account_property_sync.py+++ b/products/customer_analytics/backend/temporal/account_property_sync.py@@ -16,6 +16,7 @@ from posthog.temporal.common.heartbeat import LivenessHeartbeater as Heartbeaterfrom products.customer_analytics.backend.facade.temporal_contracts import (AccountPropertySegmentTelemetryInput,+ AccountPropertySourceSyncInput,AccountPropertySyncInput,DispatchAccountPropertySyncInput,FinalizeAccountPropertySyncRunsInput,@@ -24,25 +25,34 @@ from products.customer_analytics.backend.facade.temporal_contracts import (with workflow.unsafe.imports_passed_through():from datetime import timedelta+ from uuid import UUIDfrom django.conf import settingsimport structlogfrom prometheus_client import Counter, Histogram- from posthog.sync import database_sync_to_async+ from posthog.sync import database_sync_to_async, database_sync_to_async_poolfrom posthog.temporal.common.client import async_connect+ from products.customer_analytics.backend.logic.account_property_fanout_flags import (+ account_property_sync_fanout_enabled,+ )from products.customer_analytics.backend.logic.account_property_runs import (AccountPropertySyncRunContext,+ AccountPropertySyncSegmentIncompleteError,finalize_account_property_sync_runs,start_account_property_sync_runs,update_account_property_sync_runs_phase,)from products.customer_analytics.backend.logic.account_property_sync import (+ AccountPropertySourceRejectedError,AccountPropertySourceValueError,AccountPropertySyncSegment,+ complete_account_property_sync_segment,+ get_account_property_sync_source_ids,run_account_property_segment_sync,+ run_account_property_source_sync,)from products.customer_analytics.backend.logic.account_property_sync_monitoring import (capture_account_property_sync_event,@@ -55,9 +65,13 @@ logger = structlog.get_logger(__name__)ACCOUNT_PROPERTY_STAGING_WORKFLOW_NAME = "stage-warehouse-account-properties"ACCOUNT_PROPERTY_SYNC_WORKFLOW_NAME = "sync-warehouse-account-properties"+ACCOUNT_PROPERTY_SYNC_FANOUT_WORKFLOW_NAME = "sync-warehouse-account-properties-fanout"ACCOUNT_PROPERTY_RUN_HISTORY_PATCH = "account-property-run-history-2026-08"ACCOUNT_PROPERTY_SEGMENT_TELEMETRY_PATCH = "account-property-segment-telemetry-2026-10"ACCOUNT_PROPERTY_ACTIVITY_MAX_ATTEMPTS = 5+ACCOUNT_PROPERTY_SOURCE_CONCURRENCY = 3+# Source IDs are UUID strings, so this keeps the discovery result far below the 256 KB field rule.+ACCOUNT_PROPERTY_FANOUT_MAX_SOURCES = 2_000_STAGING_FAILED_ERROR = ("Couldn't prepare warehouse rows. Run the source view again. If it keeps failing, contact support."@@ -65,6 +79,8 @@ _STAGING_FAILED_ERROR = (_DISPATCH_FAILED_ERROR = ("Couldn't start account updates. Run the source view again. If it keeps failing, contact support.")+_SYNC_FAILED_ERROR = "Couldn't update accounts. Run the source view again. If it keeps failing, contact support."+_SOURCE_SYNC_FAILED_ERROR_TYPE = "AccountPropertySourceSyncFailed"ACCOUNT_PROPERTY_SYNC_TOTAL = Counter("warehouse_account_property_sync_total",@@ -83,6 +99,11 @@ _SyncPath = Literal["legacy", "fanout"]_TerminalStatus = Literal["completed", "failed", "cancelled"]+def _segment_workflow_id(job_id: str, segment: str) -> str:+ # Both workflow types share this ID, so a retried dispatch can't start a second path for a segment.+ return f"sync-warehouse-account-properties-{job_id}-{segment}"++def _failure_type(error: BaseException) -> str:cause = error.cause if isinstance(error, ActivityError) and error.cause is not None else errorif isinstance(cause, ApplicationError) and cause.type:@@ -209,6 +230,7 @@ async def finalize_warehouse_account_property_runs_activity(input: FinalizeAccoustatus=SyncStatus(input.status),phase=SyncPhase(input.phase),error=input.error,+ segment=None if input.segment is None else AccountPropertySyncSegment(input.segment),)@@ -266,19 +288,23 @@ async def dispatch_warehouse_account_property_sync_activity(input: DispatchAccouworkflow_run_id=activity_info.workflow_run_id,attempt=activity_info.attempt,)+ # Evaluated once per dispatch so both segments of this attempt start the same workflow type. A flag+ # change only affects segments that are not started yet, because both types share one workflow ID.+ use_fanout = await database_sync_to_async_pool(account_property_sync_fanout_enabled)(input.team_id)+ workflow_name = ACCOUNT_PROPERTY_SYNC_FANOUT_WORKFLOW_NAME if use_fanout else ACCOUNT_PROPERTY_SYNC_WORKFLOW_NAME+ path: _SyncPath = "fanout" if use_fanout else "legacy"client = await async_connect()for segment in ("tracked", "ignored"):- workflow_id = f"sync-warehouse-account-properties-{input.job_id}-{segment}"try:await client.start_workflow(- ACCOUNT_PROPERTY_SYNC_WORKFLOW_NAME,+ workflow_name,AccountPropertySyncInput(team_id=input.team_id,saved_query_id=input.saved_query_id,job_id=input.job_id,segment=segment,),- id=workflow_id,+ id=_segment_workflow_id(input.job_id, segment),id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY,task_queue=settings.DATA_WAREHOUSE_METADATA_TASK_QUEUE,execution_timeout=timedelta(hours=24),@@ -296,10 +322,17 @@ async def dispatch_warehouse_account_property_sync_activity(input: DispatchAccousaved_query_id=input.saved_query_id,job_id=input.job_id,segment=segment,- path="legacy",+ path=path,scope="segment",status="started",- workflow_id=workflow_id,+ workflow_id=_segment_workflow_id(input.job_id, segment),+ )+ logger.info(+ "Account-property segment sync started",+ team_id=input.team_id,+ job_id=input.job_id,+ segment=segment,+ path=path,)@@ -364,6 +397,103 @@ async def sync_warehouse_account_properties_activity(input: AccountPropertySyncIreturn result[email protected](name="list-warehouse-account-property-sources")+async def list_warehouse_account_property_sources_activity(input: AccountPropertySyncInput) -> list[str]:+ source_ids = await database_sync_to_async(get_account_property_sync_source_ids, thread_sensitive=False)(+ input.team_id, saved_query_binding(input.saved_query_id)+ )+ if len(source_ids) > ACCOUNT_PROPERTY_FANOUT_MAX_SOURCES:+ raise ApplicationError(+ f"View has {len(source_ids)} account-property sources, above the fanout limit of "+ f"{ACCOUNT_PROPERTY_FANOUT_MAX_SOURCES}",+ type="AccountPropertyFanoutSourceLimitExceeded",+ non_retryable=True,+ )+ return source_ids++[email protected](name="sync-warehouse-account-property-source")+async def sync_warehouse_account_property_source_activity(input: AccountPropertySourceSyncInput) -> dict[str, int]:+ segment = AccountPropertySyncSegment(input.segment)+ activity_info = activity.info()+ log = logger.bind(+ team_id=input.team_id,+ saved_query_id=input.saved_query_id,+ job_id=input.job_id,+ segment=segment.value,+ source_id=input.source_id,+ )+ telemetry = _AttemptTelemetry.start(+ team_id=input.team_id,+ saved_query_id=input.saved_query_id,+ job_id=input.job_id,+ segment=segment.value,+ path="fanout",+ source_id=input.source_id,+ )+ try:+ source_uuid = UUID(input.source_id)+ except ValueError as error:+ telemetry.finish("failed", error=error)+ raise ApplicationError("Account-property source ID is not a UUID", non_retryable=True) from error+ try:+ await database_sync_to_async(update_account_property_sync_runs_phase)(+ AccountPropertySyncRunContext(+ team_id=input.team_id,+ saved_query_id=input.saved_query_id,+ job_id=input.job_id,+ ),+ phase=SyncPhase.SYNCING,+ workflow_id=activity_info.workflow_id,+ workflow_run_id=activity_info.workflow_run_id,+ attempt=activity_info.attempt,+ segment=segment,+ source_id=source_uuid,+ )+ async with Heartbeater():+ result = await run_account_property_source_sync(+ team_id=input.team_id,+ binding=saved_query_binding(input.saved_query_id),+ job_id=input.job_id,+ segment=segment,+ source_id=input.source_id,+ final_attempt=activity_info.attempt >= ACCOUNT_PROPERTY_ACTIVITY_MAX_ATTEMPTS,+ )+ except (AccountPropertySourceValueError, AccountPropertySourceRejectedError) as error:+ # A retry reads the same staged rows and source config, so it can't fix these errors.+ failure_counts = error.counts if isinstance(error, AccountPropertySourceValueError) else None+ telemetry.finish("failed", counts=failure_counts, error=error)+ log.warning("Account-property source sync rejected", error_type=type(error).__name__)+ details = () if failure_counts is None else (failure_counts,)+ raise ApplicationError(str(error), *details, type=type(error).__name__, non_retryable=True) from error+ except asyncio.CancelledError:+ telemetry.finish("cancelled")+ raise+ except Exception as error:+ telemetry.finish("failed", error=error)+ log.exception("Account-property source sync failed")+ capture_exception(error)+ raise++ telemetry.finish("completed", counts=result)+ log.info("Account-property source sync completed", **result)+ return result++[email protected](name="complete-warehouse-account-property-segment")+async def complete_warehouse_account_property_segment_activity(input: AccountPropertySyncInput) -> None:+ try:+ async with Heartbeater():+ await complete_account_property_sync_segment(+ team_id=input.team_id,+ binding=saved_query_binding(input.saved_query_id),+ job_id=input.job_id,+ segment=AccountPropertySyncSegment(input.segment),+ )+ except AccountPropertySyncSegmentIncompleteError as error:+ raise ApplicationError(str(error), type=type(error).__name__, non_retryable=True) from error++@activity.defn(name="emit-warehouse-account-property-segment-telemetry")async def emit_warehouse_account_property_segment_telemetry_activity(input: AccountPropertySegmentTelemetryInput,@@ -561,9 +691,135 @@ class SyncWarehouseAccountPropertiesWorkflow(PostHogWorkflow):)+@frozen+class _SettledSources:+ counts: dict[str, int]+ failed: int+++async def _sync_sources(input: AccountPropertySyncInput, source_ids: list[str]) -> _SettledSources:+ semaphore = asyncio.Semaphore(ACCOUNT_PROPERTY_SOURCE_CONCURRENCY)++ async def sync_source(source_id: str) -> dict[str, int]:+ async with semaphore:+ return await workflow.execute_activity(+ sync_warehouse_account_property_source_activity,+ AccountPropertySourceSyncInput(+ team_id=input.team_id,+ saved_query_id=input.saved_query_id,+ job_id=input.job_id,+ segment=input.segment,+ source_id=source_id,+ ),+ activity_id=f"sync-account-property-source-{source_id}",+ start_to_close_timeout=timedelta(hours=6),+ heartbeat_timeout=timedelta(minutes=5),+ retry_policy=RetryPolicy(+ maximum_attempts=ACCOUNT_PROPERTY_ACTIVITY_MAX_ATTEMPTS,+ initial_interval=timedelta(seconds=30),+ ),+ )++ # return_exceptions keeps one failed source from ending the gather while its siblings still run.+ # The coordinator must not finalize the segment before every source activity settles.+ results = await asyncio.gather(*(sync_source(source_id) for source_id in source_ids), return_exceptions=True)+ counts: dict[str, int] = {}+ failed = 0+ for result in results:+ if isinstance(result, BaseException):+ failed += 1+ continue+ for key, value in result.items():+ counts[key] = counts.get(key, 0) + value+ counts["source_errors"] = failed+ return _SettledSources(counts=counts, failed=failed)++[email protected](name=ACCOUNT_PROPERTY_SYNC_FANOUT_WORKFLOW_NAME)+class SyncWarehouseAccountPropertiesFanoutWorkflow(PostHogWorkflow):+ @staticmethod+ def parse_inputs(inputs: list[str]) -> AccountPropertySyncInput:+ return AccountPropertySyncInput(**json.loads(inputs[0]))++ @workflow.run+ async def run(self, input: AccountPropertySyncInput) -> None:+ started_at = workflow.time()+ source_count: int | None = None+ counts: dict[str, int] | None = None+ try:+ listed_source_ids = await workflow.execute_activity(+ list_warehouse_account_property_sources_activity,+ input,+ start_to_close_timeout=timedelta(minutes=5),+ retry_policy=RetryPolicy(maximum_attempts=ACCOUNT_PROPERTY_ACTIVITY_MAX_ATTEMPTS),+ )+ source_ids = list(dict.fromkeys(listed_source_ids))+ source_count = len(source_ids)+ settled = await _sync_sources(input, source_ids)+ counts = settled.counts+ if settled.failed:+ raise ApplicationError(+ f"{settled.failed} of {source_count} account-property sources failed",+ type=_SOURCE_SYNC_FAILED_ERROR_TYPE,+ non_retryable=True,+ )+ await workflow.execute_activity(+ complete_warehouse_account_property_segment_activity,+ input,+ start_to_close_timeout=timedelta(hours=1),+ heartbeat_timeout=timedelta(minutes=5),+ retry_policy=RetryPolicy(+ maximum_attempts=ACCOUNT_PROPERTY_ACTIVITY_MAX_ATTEMPTS,+ initial_interval=timedelta(seconds=30),+ ),+ )+ except (asyncio.CancelledError, Exception) as error:+ # Workflow eviction raises a BaseException that is not caught here, so cleanup runs only for+ # a real failure or cancellation.+ cancelled = is_cancelled_exception(error)+ try:+ if not cancelled:+ # Closes only the runs still open in this segment. Completed sibling runs keep their+ # result, and the staged rows stay in place for the next run of the source view.+ await workflow.execute_activity(+ finalize_warehouse_account_property_runs_activity,+ FinalizeAccountPropertySyncRunsInput(+ team_id=input.team_id,+ saved_query_id=input.saved_query_id,+ job_id=input.job_id,+ status=SyncStatus.FAILED.value,+ phase=SyncPhase.SYNCING.value,+ error=_SYNC_FAILED_ERROR,+ segment=input.segment,+ ),+ start_to_close_timeout=timedelta(minutes=5),+ retry_policy=RetryPolicy(maximum_attempts=ACCOUNT_PROPERTY_ACTIVITY_MAX_ATTEMPTS),+ )+ finally:+ await _emit_segment_telemetry(+ input,+ path="fanout",+ status="cancelled" if cancelled else "failed",+ started_at=started_at,+ source_count=source_count,+ counts=counts,+ error=error,+ )+ raise+ await _emit_segment_telemetry(+ input,+ path="fanout",+ status="completed",+ started_at=started_at,+ source_count=source_count,+ counts=counts,+ )++ACCOUNT_PROPERTY_SYNC_WORKFLOWS = [StageWarehouseAccountPropertiesWorkflow,SyncWarehouseAccountPropertiesWorkflow,+ SyncWarehouseAccountPropertiesFanoutWorkflow,]ACCOUNT_PROPERTY_SYNC_ACTIVITIES = [start_warehouse_account_property_runs_activity,@@ -571,5 +827,8 @@ ACCOUNT_PROPERTY_SYNC_ACTIVITIES = [stage_warehouse_account_property_files_activity,dispatch_warehouse_account_property_sync_activity,sync_warehouse_account_properties_activity,+ list_warehouse_account_property_sources_activity,+ sync_warehouse_account_property_source_activity,+ complete_warehouse_account_property_segment_activity,emit_warehouse_account_property_segment_telemetry_activity,]diff --git a/products/customer_analytics/backend/temporal/account_property_sync_test.py b/products/customer_analytics/backend/temporal/account_property_sync_test.pyindex 295b0447a268..5852a03c2667 100644--- a/products/customer_analytics/backend/temporal/account_property_sync_test.py+++ b/products/customer_analytics/backend/temporal/account_property_sync_test.py@@ -1,8 +1,9 @@import uuidimport asyncio-from collections.abc import AsyncIterator+from collections.abc import AsyncIterator, Callablefrom contextlib import asynccontextmanagerfrom datetime import timedelta+from typing import Anyimport pytestfrom unittest.mock import AsyncMock, MagicMock, patch@@ -22,22 +23,29 @@ from temporalio.worker import Replayer, UnsandboxedWorkflowRunner, Workerfrom products.customer_analytics.backend.facade.temporal_contracts import (AccountPropertySegmentTelemetryInput,+ AccountPropertySourceSyncInput,DispatchAccountPropertySyncInput,FinalizeAccountPropertySyncRunsInput,StageAccountPropertySyncInput,)from products.customer_analytics.backend.logic.account_property_sync import AccountPropertySourceValueErrorfrom products.customer_analytics.backend.temporal.account_property_sync import (+ ACCOUNT_PROPERTY_SYNC_ACTIVITIES,ACCOUNT_PROPERTY_SYNC_WORKFLOW_NAME,+ ACCOUNT_PROPERTY_SYNC_WORKFLOWS,AccountPropertySyncInput,StageWarehouseAccountPropertiesWorkflow,+ SyncWarehouseAccountPropertiesFanoutWorkflow,SyncWarehouseAccountPropertiesWorkflow,+ complete_warehouse_account_property_segment_activity,dispatch_warehouse_account_property_sync_activity,emit_warehouse_account_property_segment_telemetry_activity,finalize_warehouse_account_property_runs_activity,+ list_warehouse_account_property_sources_activity,stage_warehouse_account_property_files_activity,start_warehouse_account_property_runs_activity,sync_warehouse_account_properties_activity,+ sync_warehouse_account_property_source_activity,)pytestmark = pytest.mark.asyncio@@ -45,6 +53,7 @@ pytestmark = pytest.mark.asyncio_MODULE = "products.customer_analytics.backend.temporal.account_property_sync"_SOURCE_COUNTS = {"rows_read": 10, "changed": 4, "matched": 3, "written": 2, "source_errors": 0, "source_count": 2}_FAILED_COUNTS = {**_SOURCE_COUNTS, "written": 1, "source_errors": 1}+_FANOUT_COUNTS = {**_SOURCE_COUNTS, "source_count": 1}def _create_activity_error(cause: Exception) -> ActivityError:@@ -221,7 +230,16 @@ async def test_staging_and_dispatch_recover_from_transient_activity_failures() -assert attempts == [("stage", 1), ("stage", 2), ("dispatch", 1), ("dispatch", 2)]-async def test_dispatch_starts_missing_segment_when_its_sibling_already_runs() -> None:+ "fanout_enabled,workflow_name",+ [+ (False, "sync-warehouse-account-properties"),+ (True, "sync-warehouse-account-properties-fanout"),+ ],+)+async def test_dispatch_selects_one_workflow_type_and_keeps_stable_segment_ids(+ fanout_enabled: bool, workflow_name: str+) -> None:client = AsyncMock()client.start_workflow.side_effect = [WorkflowAlreadyStartedError("tracked", "workflow"), None]input = DispatchAccountPropertySyncInput(@@ -231,25 +249,23 @@ async def test_dispatch_starts_missing_segment_when_its_sibling_already_runs() -)with (- patch("products.customer_analytics.backend.temporal.account_property_sync.async_connect", return_value=client),- patch("products.customer_analytics.backend.temporal.account_property_sync.activity.info") as activity_info,- patch(- "products.customer_analytics.backend.temporal.account_property_sync.update_account_property_sync_runs_phase"- ),+ patch(f"{_MODULE}.async_connect", return_value=client),+ patch(f"{_MODULE}.account_property_sync_fanout_enabled", return_value=fanout_enabled) as flag,patch("products.customer_analytics.backend.logic.account_property_sync_monitoring.ph_background_capture") as capture_factory,+ patch(f"{_MODULE}.activity.info") as activity_info,+ patch(f"{_MODULE}.update_account_property_sync_runs_phase"),):activity_info.return_value.attempt = 1activity_info.return_value.workflow_id = "stage-workflow-job-1"await dispatch_warehouse_account_property_sync_activity(input)+ flag.assert_called_once_with(7)calls = client.start_workflow.await_args_list- assert [call.args[0] for call in calls] == [- ACCOUNT_PROPERTY_SYNC_WORKFLOW_NAME,- ACCOUNT_PROPERTY_SYNC_WORKFLOW_NAME,- ]+ assert [call.args[0] for call in calls] == [workflow_name, workflow_name]assert [call.args[1].segment for call in calls] == ["tracked", "ignored"]+ # Both workflow types share these IDs, so a retry after a flag change finds the running segment.assert [call.kwargs["id"] for call in calls] == ["sync-warehouse-account-properties-job-1-tracked","sync-warehouse-account-properties-job-1-ignored",@@ -260,7 +276,7 @@ async def test_dispatch_starts_missing_segment_when_its_sibling_already_runs() -assert {key: properties[key] for key in ("scope", "status", "path", "segment", "workflow_id")} == {"scope": "segment","status": "started",- "path": "legacy",+ "path": "fanout" if fanout_enabled else "legacy","segment": "ignored","workflow_id": "sync-warehouse-account-properties-job-1-ignored",}@@ -340,6 +356,232 @@ async def test_legacy_workflow_preserves_sync_outcome_when_telemetry_times_out(sawait SyncWarehouseAccountPropertiesWorkflow().run(_segment_input())+class _FakeFanoutActivities:+ def __init__(self, source_ids: list[str]) -> None:+ loop = asyncio.get_running_loop()+ self.source_ids = source_ids+ self.log: list[str] = []+ self.inputs: dict[str, object] = {}+ self.in_flight = 0+ self.max_in_flight = 0+ self.started = {source_id: asyncio.Event() for source_id in source_ids}+ self.results: dict[str, asyncio.Future[dict[str, int]]] = {+ source_id: loop.create_future() for source_id in source_ids+ }++ async def execute_activity(self, activity_fn: Callable[..., Any], arg: object, **_kwargs: object) -> object:+ if activity_fn is list_warehouse_account_property_sources_activity:+ return list(self.source_ids)+ if activity_fn is sync_warehouse_account_property_source_activity:+ assert isinstance(arg, AccountPropertySourceSyncInput)+ return await self._sync_source(arg.source_id)+ self.log.append(activity_fn.__name__)+ self.inputs[activity_fn.__name__] = arg+ return None++ async def _sync_source(self, source_id: str) -> dict[str, int]:+ self.in_flight += 1+ self.max_in_flight = max(self.max_in_flight, self.in_flight)+ self.log.append(f"started:{source_id}")+ self.started[source_id].set()+ try:+ result = await self.results[source_id]+ except asyncio.CancelledError:+ self.log.append(f"cancelled:{source_id}")+ raise+ except Exception:+ self.log.append(f"failed:{source_id}")+ raise+ finally:+ self.in_flight -= 1+ self.log.append(f"completed:{source_id}")+ return result++ def started_sources(self) -> list[str]:+ return [entry.removeprefix("started:") for entry in self.log if entry.startswith("started:")]++ def telemetry(self) -> AccountPropertySegmentTelemetryInput:+ telemetry = self.inputs[emit_warehouse_account_property_segment_telemetry_activity.__name__]+ assert isinstance(telemetry, AccountPropertySegmentTelemetryInput)+ return telemetry+++async def _wait(event: asyncio.Event) -> None:+ # Guards against a hang if the coordinator never starts the source. Correct runs never reach it.+ await asyncio.wait_for(event.wait(), timeout=5)+++async def test_fanout_runs_sources_in_parallel_up_to_the_cap() -> None:+ source_ids = ["source-1", "source-2", "source-3", "source-4", "source-5"]+ fake = _FakeFanoutActivities(source_ids)++ with (+ patch.object(workflow, "execute_activity", new=fake.execute_activity),+ patch.object(workflow, "time", side_effect=[100.0, 145.0]),+ ):+ run = asyncio.create_task(SyncWarehouseAccountPropertiesFanoutWorkflow().run(_segment_input()))+ await _wait(fake.started["source-3"])+ assert fake.started_sources() == ["source-1", "source-2", "source-3"]++ fake.results["source-2"].set_result(dict(_FANOUT_COUNTS))+ await _wait(fake.started["source-4"])+ assert fake.in_flight == 3+ assert fake.started_sources() == ["source-1", "source-2", "source-3", "source-4"]++ for source_id in ("source-1", "source-3", "source-4", "source-5"):+ if not fake.results[source_id].done():+ fake.results[source_id].set_result(dict(_FANOUT_COUNTS))+ await asyncio.wait_for(run, timeout=5)++ assert fake.max_in_flight == 3+ complete = complete_warehouse_account_property_segment_activity.__name__+ assert fake.log.count(complete) == 1+ assert all(fake.log.index(f"completed:{source_id}") < fake.log.index(complete) for source_id in source_ids)+ telemetry = fake.telemetry()+ assert (telemetry.path, telemetry.status, telemetry.duration_seconds, telemetry.source_count) == (+ "fanout",+ "completed",+ 45.0,+ 5,+ )+ assert telemetry.counts == {+ "rows_read": 50,+ "changed": 20,+ "matched": 15,+ "written": 10,+ "source_errors": 0,+ "source_count": 5,+ }+++async def test_fanout_waits_for_running_siblings_before_failing_the_segment() -> None:+ fake = _FakeFanoutActivities(["source-a", "source-b", "source-c"])+ fake.results["source-a"].set_result(dict(_FANOUT_COUNTS))+ fake.results["source-b"].set_exception(RuntimeError("source failed"))++ with (+ patch.object(workflow, "execute_activity", new=fake.execute_activity),+ patch.object(workflow, "time", return_value=100.0),+ ):+ run = asyncio.create_task(SyncWarehouseAccountPropertiesFanoutWorkflow().run(_segment_input()))+ await _wait(fake.started["source-c"])+ fake.results["source-c"].set_result(dict(_FANOUT_COUNTS))+ with pytest.raises(ApplicationError):+ await asyncio.wait_for(run, timeout=5)++ finalize = finalize_warehouse_account_property_runs_activity.__name__+ assert "cancelled:source-c" not in fake.log+ assert fake.log.index("completed:source-c") < fake.log.index(finalize)+ assert complete_warehouse_account_property_segment_activity.__name__ not in fake.log+ finalization = fake.inputs[finalize]+ assert isinstance(finalization, FinalizeAccountPropertySyncRunsInput)+ assert (finalization.status, finalization.segment) == ("failed", "tracked")+ telemetry = fake.telemetry()+ assert (telemetry.status, telemetry.source_count) == ("failed", 3)+ assert telemetry.counts == {+ "rows_read": 20,+ "changed": 8,+ "matched": 6,+ "written": 4,+ "source_errors": 1,+ "source_count": 2,+ }+++async def test_fanout_finishes_a_segment_without_sources() -> None:+ fake = _FakeFanoutActivities([])++ with (+ patch.object(workflow, "execute_activity", new=fake.execute_activity),+ patch.object(workflow, "time", return_value=100.0),+ ):+ await SyncWarehouseAccountPropertiesFanoutWorkflow().run(_segment_input())++ assert fake.log == [+ complete_warehouse_account_property_segment_activity.__name__,+ emit_warehouse_account_property_segment_telemetry_activity.__name__,+ ]+ assert (fake.telemetry().status, fake.telemetry().source_count) == ("completed", 0)+++async def test_fanout_cancellation_keeps_runs_open_and_reports_cancelled() -> None:+ fake = _FakeFanoutActivities(["source-a"])++ with (+ patch.object(workflow, "execute_activity", new=fake.execute_activity),+ patch.object(workflow, "time", return_value=100.0),+ ):+ run = asyncio.create_task(SyncWarehouseAccountPropertiesFanoutWorkflow().run(_segment_input()))+ await _wait(fake.started["source-a"])+ run.cancel()+ with pytest.raises(asyncio.CancelledError):+ await run++ assert finalize_warehouse_account_property_runs_activity.__name__ not in fake.log+ assert complete_warehouse_account_property_segment_activity.__name__ not in fake.log+ assert fake.telemetry().status == "cancelled"+++async def test_fanout_workflow_retries_each_source_independently_and_replays() -> None:+ attempts: list[tuple[str, int]] = []+ completed: list[str] = []+ telemetry: list[AccountPropertySegmentTelemetryInput] = []++ @activity.defn(name="list-warehouse-account-property-sources")+ async def list_sources(_input: AccountPropertySyncInput) -> list[str]:+ return ["source-a", "source-b"]++ @activity.defn(name="sync-warehouse-account-property-source")+ async def sync_source(input: AccountPropertySourceSyncInput) -> dict[str, int]:+ attempt = activity.info().attempt+ attempts.append((input.source_id, attempt))+ if input.source_id == "source-a" and attempt == 1:+ raise RuntimeError("warehouse temporarily unavailable")+ return dict(_FANOUT_COUNTS)++ @activity.defn(name="complete-warehouse-account-property-segment")+ async def complete_segment(input: AccountPropertySyncInput) -> None:+ completed.append(input.segment)++ @activity.defn(name="finalize-warehouse-account-property-runs")+ async def finalize_runs(_input: FinalizeAccountPropertySyncRunsInput) -> None:+ raise AssertionError("A successful segment must not finalize runs as failed")++ @activity.defn(name="emit-warehouse-account-property-segment-telemetry")+ async def emit_telemetry(input: AccountPropertySegmentTelemetryInput) -> None:+ telemetry.append(input)++ fakes: list[Callable[..., Any]] = [list_sources, sync_source, complete_segment, finalize_runs, emit_telemetry]+ registered = {getattr(fn, "__temporal_activity_definition").name for fn in ACCOUNT_PROPERTY_SYNC_ACTIVITIES}+ assert {getattr(fn, "__temporal_activity_definition").name for fn in fakes} <= registered++ task_queue = str(uuid.uuid4())+ async with await WorkflowEnvironment.start_time_skipping() as environment:+ async with Worker(+ environment.client,+ task_queue=task_queue,+ workflows=ACCOUNT_PROPERTY_SYNC_WORKFLOWS,+ activities=fakes,+ workflow_runner=UnsandboxedWorkflowRunner(),+ ):+ handle = await environment.client.start_workflow(+ SyncWarehouseAccountPropertiesFanoutWorkflow.run,+ _segment_input(),+ id=str(uuid.uuid4()),+ task_queue=task_queue,+ execution_timeout=timedelta(minutes=10),+ )+ await handle.result()+ history = await handle.fetch_history()++ await Replayer(+ workflows=[SyncWarehouseAccountPropertiesFanoutWorkflow], workflow_runner=UnsandboxedWorkflowRunner()+ ).replay_workflow(history)+ assert sorted(attempts) == [("source-a", 1), ("source-a", 2), ("source-b", 1)]+ assert completed == ["tracked"]+ assert [(event.status, event.source_count) for event in telemetry] == [("completed", 2)]++@workflow.defn(name=ACCOUNT_PROPERTY_SYNC_WORKFLOW_NAME)class _PreTelemetrySyncWorkflow:@workflow.run@@ -417,26 +659,60 @@ async def test_legacy_workflow_replays_histories_from_before_and_after_the_telem}+_ATTEMPT_OUTCOMES = [+ ("completed", dict(_SOURCE_COUNTS), "completed", None),+ ("transient_failure", RuntimeError("warehouse unavailable"), "failed", "RuntimeError"),+ (+ "invalid_values",+ AccountPropertySourceValueError("bad value", counts=_FAILED_COUNTS),+ "failed",+ "AccountPropertySourceValueError",+ ),+ ("cancelled", asyncio.CancelledError(), "cancelled", None),+]++@pytest.mark.parametrize(- "runtime_result,expected_status,expected_error_type",+ "path,activity_fn,runtime_target,activity_input,expected_source_id",[- (dict(_SOURCE_COUNTS), "completed", None),- (RuntimeError("warehouse unavailable"), "failed", "RuntimeError"),(- AccountPropertySourceValueError("bad value", counts=_FAILED_COUNTS),- "failed",- "AccountPropertySourceValueError",+ "legacy",+ sync_warehouse_account_properties_activity,+ "run_account_property_segment_sync",+ _segment_input(),+ None,+ ),+ (+ "fanout",+ sync_warehouse_account_property_source_activity,+ "run_account_property_source_sync",+ AccountPropertySourceSyncInput(+ team_id=7,+ saved_query_id="019f0000-0000-7000-8000-000000000001",+ job_id="job-1",+ segment="tracked",+ source_id="019f0000-0000-7000-8000-0000000000aa",+ ),+ "019f0000-0000-7000-8000-0000000000aa",),- (asyncio.CancelledError(), "cancelled", None),],)[email protected]("_outcome,runtime_result,expected_status,expected_error_type", _ATTEMPT_OUTCOMES)async def test_sync_activity_reports_attempt_lifecycle(- runtime_result: object, expected_status: str, expected_error_type: str | None+ path: str,+ activity_fn: Callable[[Any], Any],+ runtime_target: str,+ activity_input: object,+ expected_source_id: str | None,+ _outcome: str,+ runtime_result: object,+ expected_status: str,+ expected_error_type: str | None,) -> None:runtime = AsyncMock(side_effect=[runtime_result])with (- patch(f"{_MODULE}.run_account_property_segment_sync", new=runtime),+ patch(f"{_MODULE}.{runtime_target}", new=runtime),patch(f"{_MODULE}.capture_account_property_sync_event") as capture_event,patch(f"{_MODULE}.capture_exception"),patch(f"{_MODULE}.saved_query_binding"),@@ -449,18 +725,18 @@ async def test_sync_activity_reports_attempt_lifecycle(activity_info.return_value.workflow_run_id = "segment-run"if isinstance(runtime_result, BaseException):with pytest.raises(BaseException) as raised:- await sync_warehouse_account_properties_activity(_segment_input())+ await activity_fn(activity_input)if isinstance(runtime_result, AccountPropertySourceValueError):assert isinstance(raised.value, ApplicationError) and raised.value.non_retryableassert raised.value.type == "AccountPropertySourceValueError"assert raised.value.details == (_FAILED_COUNTS,)else:- assert await sync_warehouse_account_properties_activity(_segment_input()) == runtime_result+ assert await activity_fn(activity_input) == runtime_resultevents = [call.kwargs for call in capture_event.call_args_list]assert [event["status"] for event in events] == ["started", expected_status]assert {(event["path"], event["scope"], event["source_id"], event["attempt"]) for event in events} == {- ("legacy", "attempt", None, 2)+ (path, "attempt", expected_source_id, 2)}assert events[1]["error_type"] == expected_error_typeassert events[1]["duration_seconds"] is not None