Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(customer-analytics): add flagged account property fanout (#119580)

Arthur Moreira de Deus56405f4bfe7a
diff --git a/products/customer_analytics/backend/facade/temporal_contracts.py b/products/customer_analytics/backend/facade/temporal_contracts.py
index dc36a9f2afcc..6ef31ca6e402 100644
--- a/products/customer_analytics/backend/facade/temporal_contracts.py
+++ b/products/customer_analytics/backend/facade/temporal_contracts.py
@@ -27,6 +27,8 @@ class FinalizeAccountPropertySyncRunsInput:
status: str
phase: str
error: str | None = None
+ # None closes open runs in both segments. Histories recorded before this field existed decode to None.
+ segment: str | None = None
@frozen
@@ -37,6 +39,15 @@ class AccountPropertySyncInput:
segment: str
+@frozen
+class AccountPropertySourceSyncInput:
+ team_id: int
+ saved_query_id: str
+ job_id: str
+ segment: str
+ source_id: str
+
+
@frozen
class AccountPropertySegmentTelemetryInput:
team_id: int
diff --git a/products/customer_analytics/backend/logic/account_property_fanout_flags.py b/products/customer_analytics/backend/logic/account_property_fanout_flags.py
new file mode 100644
index 000000000000..83919554a87f
--- /dev/null
+++ b/products/customer_analytics/backend/logic/account_property_fanout_flags.py
@@ -0,0 +1,30 @@
+import posthoganalytics
+
+from posthog.exceptions_capture import capture_exception
+from posthog.models import Team
+
+# Selects the per-source fanout workflow for new account-property segment syncs. The flag is
+# created default-off, so an absent flag keeps every team on the legacy workflow.
+ACCOUNT_PROPERTY_SYNC_FANOUT_FLAG = "warehouse-account-property-sync-fanout"
+
+
+def account_property_sync_fanout_enabled(team_id: int) -> bool:
+ """Whether new segment syncs for the team use the fanout workflow, targeted by the team's organization.
+
+ Fails closed: an unknown team, a missing flag, or any lookup or flag-service error selects the legacy workflow.
+ """
+ try:
+ organization_id = str(Team.objects.only("organization_id").get(id=team_id).organization_id)
+ enabled = posthoganalytics.feature_enabled(
+ ACCOUNT_PROPERTY_SYNC_FANOUT_FLAG,
+ organization_id,
+ groups={"organization": organization_id},
+ only_evaluate_locally=False,
+ send_feature_flag_events=False,
+ )
+ except Team.DoesNotExist:
+ return False
+ except Exception as error:
+ capture_exception(error)
+ return False
+ return enabled is True
diff --git a/products/customer_analytics/backend/logic/account_property_fanout_flags_test.py b/products/customer_analytics/backend/logic/account_property_fanout_flags_test.py
new file mode 100644
index 000000000000..976fd57e30ef
--- /dev/null
+++ b/products/customer_analytics/backend/logic/account_property_fanout_flags_test.py
@@ -0,0 +1,48 @@
+from posthog.test.base import BaseTest
+from unittest.mock import patch
+
+from parameterized import parameterized
+
+from products.customer_analytics.backend.logic.account_property_fanout_flags import (
+ ACCOUNT_PROPERTY_SYNC_FANOUT_FLAG,
+ account_property_sync_fanout_enabled,
+)
+
+_FEATURE_ENABLED = (
+ "products.customer_analytics.backend.logic.account_property_fanout_flags.posthoganalytics.feature_enabled"
+)
+
+
+class AccountPropertySyncFanoutFlagTest(BaseTest):
+ @parameterized.expand(
+ [
+ ("enabled", True, True),
+ ("disabled", False, False),
+ ("missing_flag", None, False),
+ ("flag_service_error", RuntimeError("flag service unavailable"), False),
+ ]
+ )
+ def test_targets_the_team_organization_and_fails_closed(
+ self, _name: str, flag_result: object, expected: bool
+ ) -> None:
+ organization_id = str(self.team.organization_id)
+
+ with (
+ patch(_FEATURE_ENABLED, side_effect=[flag_result]) as feature_enabled,
+ patch("products.customer_analytics.backend.logic.account_property_fanout_flags.capture_exception"),
+ ):
+ assert account_property_sync_fanout_enabled(self.team.id) is expected
+
+ feature_enabled.assert_called_once_with(
+ ACCOUNT_PROPERTY_SYNC_FANOUT_FLAG,
+ organization_id,
+ groups={"organization": organization_id},
+ only_evaluate_locally=False,
+ send_feature_flag_events=False,
+ )
+
+ def test_unknown_team_selects_legacy_without_calling_the_flag_service(self) -> None:
+ with patch(_FEATURE_ENABLED) as feature_enabled:
+ assert account_property_sync_fanout_enabled(self.team.id + 10_000) is False
+
+ feature_enabled.assert_not_called()
diff --git a/products/customer_analytics/backend/logic/account_property_sync.py b/products/customer_analytics/backend/logic/account_property_sync.py
index 3deff65b2a97..6f455d2cb4cc 100644
--- a/products/customer_analytics/backend/logic/account_property_sync.py
+++ b/products/customer_analytics/backend/logic/account_property_sync.py
@@ -7,7 +7,7 @@ from dataclasses import field
from datetime import date, datetime, time
from decimal import Decimal
from enum import StrEnum
-from typing import Any, Protocol
+from typing import TYPE_CHECKING, Any, Protocol
from uuid import UUID
from django.db.models import QuerySet
@@ -38,7 +38,7 @@ from products.customer_analytics.backend.models.custom_property_sync_run import
SyncSegment as AccountPropertySyncSegment,
SyncStatus,
)
-from products.data_warehouse.backend.facade.api import aget_s3_client
+from products.data_warehouse.backend.facade.api import aget_s3_client, get_s3_client
from products.warehouse_sources.backend.facade.hooks import WarehouseBinding
from products.warehouse_sources.backend.facade.temporal import (
account_property_completion_prefix,
@@ -46,6 +46,9 @@ from products.warehouse_sources.backend.facade.temporal import (
account_property_snapshot_prefix,
)
+if TYPE_CHECKING:
+ from s3fs import S3FileSystem
+
logger = structlog.get_logger(__name__)
_ACCOUNT_LOOKUP_CHUNK_SIZE = 1_000
@@ -178,8 +181,14 @@ def _parquet_batches(data: bytes) -> Iterator[pa.RecordBatch]:
return pq.ParquetFile(io.BytesIO(data)).iter_batches(batch_size=_PARQUET_BATCH_SIZE)
+def _projected_parquet_batches(s3_client: "S3FileSystem", path: str, columns: list[str]) -> Iterator[pa.RecordBatch]:
+ # Read-ahead can fetch unselected column chunks, which defeats per-source projection.
+ with s3_client.open(path, "rb", cache_type="none") as file:
+ yield from pq.ParquetFile(file, pre_buffer=False).iter_batches(batch_size=_PARQUET_BATCH_SIZE, columns=columns)
+
+
async def _iter_parquet_row_batches(
- team_id: int, binding: WarehouseBinding, job_id: str
+ team_id: int, binding: WarehouseBinding, job_id: str, *, columns: list[str] | None = None
) -> AsyncIterator[list[dict[str, Any]]]:
prefix = account_property_job_staged_prefix(team_id, binding, job_id)
async with aget_s3_client() as s3_client:
@@ -190,8 +199,11 @@ async def _iter_parquet_row_batches(
entries = listing.values() if isinstance(listing, dict) else listing
file_paths = sorted(entry["Key"] for entry in entries if entry.get("type") != "directory")
for file_path in file_paths:
- data = await s3_client._cat_file(_s3_uri(file_path))
- batches = await asyncio.to_thread(_parquet_batches, data)
+ if columns is None:
+ data = await s3_client._cat_file(_s3_uri(file_path))
+ batches = await asyncio.to_thread(_parquet_batches, data)
+ else:
+ batches = _projected_parquet_batches(get_s3_client(), _s3_uri(file_path), columns)
while (batch := await asyncio.to_thread(next, batches, None)) is not None:
yield await asyncio.to_thread(batch.to_pylist)
@@ -476,9 +488,10 @@ async def _apply_staged_rows(
states: list[SourceSyncState],
counts: dict[str, int],
log: FilteringBoundLogger,
+ columns: list[str] | None = None,
) -> None:
batch_index = 0
- batches = aiter(_iter_parquet_row_batches(team_id, binding, job_id))
+ batches = aiter(_iter_parquet_row_batches(team_id, binding, job_id, columns=columns))
while True:
phase_started_at = asyncio.get_running_loop().time()
try:
@@ -728,7 +741,16 @@ async def run_account_property_source_sync(
states = [state]
await _load_prior_hashes(team_id=team_id, binding=binding, segment=segment, states=states, log=log)
await _apply_staged_rows(
- team_id=team_id, binding=binding, job_id=job_id, segment=segment, states=states, counts=counts, log=log
+ team_id=team_id,
+ binding=binding,
+ job_id=job_id,
+ segment=segment,
+ states=states,
+ counts=counts,
+ log=log,
+ columns=list(
+ dict.fromkeys(column for column in (source.key_column, source.source_column) if column is not None)
+ ),
)
await _persist_states(
team_id=team_id, binding=binding, job_id=job_id, segment=segment, states=states, counts=counts, log=log
diff --git a/products/customer_analytics/backend/logic/account_property_sync_test.py b/products/customer_analytics/backend/logic/account_property_sync_test.py
index fd6d9563ca28..0571845626dd 100644
--- a/products/customer_analytics/backend/logic/account_property_sync_test.py
+++ b/products/customer_analytics/backend/logic/account_property_sync_test.py
@@ -1,3 +1,5 @@
+import io
+import hashlib
from collections.abc import AsyncIterator
from contextlib import AbstractContextManager, ExitStack
from datetime import UTC, datetime
@@ -163,6 +165,7 @@ class AccountPropertySegmentTest(TeamScopedTestMixin, BaseTest):
def _fanout_boundaries(self, fake: "_FakeS3") -> ExitStack:
stack = ExitStack()
stack.enter_context(_fake_s3_patch(fake))
+ stack.enter_context(patch(f"{_MODULE}.get_s3_client", return_value=fake))
stack.enter_context(patch(f"{_MODULE}._matching_account_ids", side_effect=_match_known_accounts))
stack.enter_context(patch(f"{_MODULE}._apply_source_values", side_effect=_apply_all_values))
return stack
@@ -423,7 +426,7 @@ class AccountPropertySegmentTest(TeamScopedTestMixin, BaseTest):
workflow_run_id="00000000-0000-4000-8000-000000000001",
)
- async def batches(*args):
+ async def batches(*args: object, **_kwargs: object) -> AsyncIterator[list[dict[str, Any]]]:
yield [{"external_id": "acme", "plan": "enterprise"}]
with (
@@ -467,7 +470,7 @@ class AccountPropertySegmentTest(TeamScopedTestMixin, BaseTest):
client._ls = AsyncMock(return_value=[{"Key": "prefix/deleted.parquet", "type": "file"}])
client._cat_file = AsyncMock(side_effect=FileNotFoundError())
- async def no_batches(*args: object) -> AsyncIterator[list[dict[str, Any]]]:
+ async def no_batches(*args: object, **_kwargs: object) -> AsyncIterator[list[dict[str, Any]]]:
for _ in range(0):
yield []
@@ -565,7 +568,7 @@ class AccountPropertySegmentTest(TeamScopedTestMixin, BaseTest):
source.is_enabled = False
source.save(update_fields=["is_enabled"])
- async def no_batches(*args):
+ async def no_batches(*args: object, **_kwargs: object) -> AsyncIterator[list[dict[str, Any]]]:
for _ in range(0):
yield []
@@ -635,7 +638,7 @@ async def test_each_staged_batch_is_shared_across_sources(source_columns: tuple[
sources.append(source)
batch_calls = 0
- async def batches(*args):
+ async def batches(*args: object, **_kwargs: object) -> AsyncIterator[list[dict[str, Any]]]:
nonlocal batch_calls
batch_calls += 1
yield [{"organization_id": "org-1", "value_0": 1, "value_1": 2}]
@@ -672,7 +675,7 @@ async def test_sync_records_each_phase_duration(invalid_source_values: bool) ->
source.source_column = "value"
phase_duration_recorder = MagicMock()
- async def batches(*args):
+ async def batches(*args: object, **_kwargs: object) -> AsyncIterator[list[dict[str, Any]]]:
yield [{"organization_id": "org-1", "value": 1}, {"organization_id": "org-2", "value": "invalid"}]
with (
@@ -725,27 +728,49 @@ async def test_sync_records_each_phase_duration(invalid_source_values: bool) ->
]
+class _ReadTrackingFile(io.BytesIO):
+ def __init__(self, data: bytes) -> None:
+ super().__init__(data)
+ self.bytes_read = 0
+
+ def read(self, size: int | None = -1, /) -> bytes:
+ data = super().read(size)
+ self.bytes_read += len(data)
+ return data
+
+
@pytest.mark.asyncio
-async def test_staged_parquet_is_decoded_in_bounded_batches() -> None:
[email protected]("columns", [None, ["organization_id", "plan"], ["organization_id", "missing"]])
+async def test_staged_parquet_is_decoded_in_bounded_batches(columns: list[str] | None) -> None:
binding = saved_query_binding("019f0000-0000-7000-8000-000000000001")
- table = pa.table({"organization_id": [f"org-{index}" for index in range(50_001)]})
+ table = pa.table(
+ {
+ "organization_id": [f"org-{index}" for index in range(50_001)],
+ "plan": ["gold"] * 50_001,
+ "unused": [hashlib.sha256(str(index).encode()).digest() for index in range(50_001)],
+ }
+ )
buffer = pa.BufferOutputStream()
pq.write_table(table, buffer)
+ data = buffer.getvalue().to_pybytes()
+ projected_file = _ReadTrackingFile(data)
client = MagicMock()
client._ls = AsyncMock(return_value=[{"Key": "prefix/chunk.parquet", "type": "file"}])
- client._cat_file = AsyncMock(return_value=buffer.getvalue().to_pybytes())
+ client._cat_file = AsyncMock(return_value=data)
+ client.open.return_value = projected_file
- with patch(f"{_MODULE}.aget_s3_client", return_value=_S3ClientContext(client)):
- batch_sizes = [
- len(rows)
- async for rows in _iter_parquet_row_batches(
- 7,
- binding,
- "job-1",
- )
- ]
+ with (
+ patch(f"{_MODULE}.aget_s3_client", return_value=_S3ClientContext(client)),
+ patch(f"{_MODULE}.get_s3_client", return_value=client),
+ ):
+ batches = [rows async for rows in _iter_parquet_row_batches(7, binding, "job-1", columns=columns)]
- assert batch_sizes == [50_000, 1]
+ assert [len(rows) for rows in batches] == [50_000, 1]
+ expected_columns = set(table.column_names) if columns is None else set(columns) & set(table.column_names)
+ assert [set(rows[0]) for rows in batches] == [expected_columns, expected_columns]
+ if columns is not None:
+ client._cat_file.assert_not_awaited()
+ assert 0 < projected_file.bytes_read < len(data)
@pytest.mark.asyncio
@@ -799,6 +824,9 @@ class _FakeS3:
raise FileNotFoundError(path)
return self.store[key]
+ def open(self, path: str, mode: str, *, cache_type: str) -> io.BytesIO:
+ return io.BytesIO(self.store[aps._s3_key(path)])
+
async def _pipe_file(self, path: str, data: bytes) -> None:
self._clock += 1
key = aps._s3_key(path)
diff --git a/products/customer_analytics/backend/temporal/account_property_sync.py b/products/customer_analytics/backend/temporal/account_property_sync.py
index 9e3b666d3eb0..e7e696ddc1cc 100644
--- a/products/customer_analytics/backend/temporal/account_property_sync.py
+++ b/products/customer_analytics/backend/temporal/account_property_sync.py
@@ -16,6 +16,7 @@ from posthog.temporal.common.heartbeat import LivenessHeartbeater as Heartbeater
from products.customer_analytics.backend.facade.temporal_contracts import (
AccountPropertySegmentTelemetryInput,
+ AccountPropertySourceSyncInput,
AccountPropertySyncInput,
DispatchAccountPropertySyncInput,
FinalizeAccountPropertySyncRunsInput,
@@ -24,25 +25,34 @@ from products.customer_analytics.backend.facade.temporal_contracts import (
with workflow.unsafe.imports_passed_through():
from datetime import timedelta
+ from uuid import UUID
from django.conf import settings
import structlog
from prometheus_client import Counter, Histogram
- from posthog.sync import database_sync_to_async
+ from posthog.sync import database_sync_to_async, database_sync_to_async_pool
from posthog.temporal.common.client import async_connect
+ from products.customer_analytics.backend.logic.account_property_fanout_flags import (
+ account_property_sync_fanout_enabled,
+ )
from products.customer_analytics.backend.logic.account_property_runs import (
AccountPropertySyncRunContext,
+ AccountPropertySyncSegmentIncompleteError,
finalize_account_property_sync_runs,
start_account_property_sync_runs,
update_account_property_sync_runs_phase,
)
from products.customer_analytics.backend.logic.account_property_sync import (
+ AccountPropertySourceRejectedError,
AccountPropertySourceValueError,
AccountPropertySyncSegment,
+ complete_account_property_sync_segment,
+ get_account_property_sync_source_ids,
run_account_property_segment_sync,
+ run_account_property_source_sync,
)
from products.customer_analytics.backend.logic.account_property_sync_monitoring import (
capture_account_property_sync_event,
@@ -55,9 +65,13 @@ logger = structlog.get_logger(__name__)
ACCOUNT_PROPERTY_STAGING_WORKFLOW_NAME = "stage-warehouse-account-properties"
ACCOUNT_PROPERTY_SYNC_WORKFLOW_NAME = "sync-warehouse-account-properties"
+ACCOUNT_PROPERTY_SYNC_FANOUT_WORKFLOW_NAME = "sync-warehouse-account-properties-fanout"
ACCOUNT_PROPERTY_RUN_HISTORY_PATCH = "account-property-run-history-2026-08"
ACCOUNT_PROPERTY_SEGMENT_TELEMETRY_PATCH = "account-property-segment-telemetry-2026-10"
ACCOUNT_PROPERTY_ACTIVITY_MAX_ATTEMPTS = 5
+ACCOUNT_PROPERTY_SOURCE_CONCURRENCY = 3
+# Source IDs are UUID strings, so this keeps the discovery result far below the 256 KB field rule.
+ACCOUNT_PROPERTY_FANOUT_MAX_SOURCES = 2_000
_STAGING_FAILED_ERROR = (
"Couldn't prepare warehouse rows. Run the source view again. If it keeps failing, contact support."
@@ -65,6 +79,8 @@ _STAGING_FAILED_ERROR = (
_DISPATCH_FAILED_ERROR = (
"Couldn't start account updates. Run the source view again. If it keeps failing, contact support."
)
+_SYNC_FAILED_ERROR = "Couldn't update accounts. Run the source view again. If it keeps failing, contact support."
+_SOURCE_SYNC_FAILED_ERROR_TYPE = "AccountPropertySourceSyncFailed"
ACCOUNT_PROPERTY_SYNC_TOTAL = Counter(
"warehouse_account_property_sync_total",
@@ -83,6 +99,11 @@ _SyncPath = Literal["legacy", "fanout"]
_TerminalStatus = Literal["completed", "failed", "cancelled"]
+def _segment_workflow_id(job_id: str, segment: str) -> str:
+ # Both workflow types share this ID, so a retried dispatch can't start a second path for a segment.
+ return f"sync-warehouse-account-properties-{job_id}-{segment}"
+
+
def _failure_type(error: BaseException) -> str:
cause = error.cause if isinstance(error, ActivityError) and error.cause is not None else error
if isinstance(cause, ApplicationError) and cause.type:
@@ -209,6 +230,7 @@ async def finalize_warehouse_account_property_runs_activity(input: FinalizeAccou
status=SyncStatus(input.status),
phase=SyncPhase(input.phase),
error=input.error,
+ segment=None if input.segment is None else AccountPropertySyncSegment(input.segment),
)
@@ -266,19 +288,23 @@ async def dispatch_warehouse_account_property_sync_activity(input: DispatchAccou
workflow_run_id=activity_info.workflow_run_id,
attempt=activity_info.attempt,
)
+ # Evaluated once per dispatch so both segments of this attempt start the same workflow type. A flag
+ # change only affects segments that are not started yet, because both types share one workflow ID.
+ use_fanout = await database_sync_to_async_pool(account_property_sync_fanout_enabled)(input.team_id)
+ workflow_name = ACCOUNT_PROPERTY_SYNC_FANOUT_WORKFLOW_NAME if use_fanout else ACCOUNT_PROPERTY_SYNC_WORKFLOW_NAME
+ path: _SyncPath = "fanout" if use_fanout else "legacy"
client = await async_connect()
for segment in ("tracked", "ignored"):
- workflow_id = f"sync-warehouse-account-properties-{input.job_id}-{segment}"
try:
await client.start_workflow(
- ACCOUNT_PROPERTY_SYNC_WORKFLOW_NAME,
+ workflow_name,
AccountPropertySyncInput(
team_id=input.team_id,
saved_query_id=input.saved_query_id,
job_id=input.job_id,
segment=segment,
),
- id=workflow_id,
+ id=_segment_workflow_id(input.job_id, segment),
id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY,
task_queue=settings.DATA_WAREHOUSE_METADATA_TASK_QUEUE,
execution_timeout=timedelta(hours=24),
@@ -296,10 +322,17 @@ async def dispatch_warehouse_account_property_sync_activity(input: DispatchAccou
saved_query_id=input.saved_query_id,
job_id=input.job_id,
segment=segment,
- path="legacy",
+ path=path,
scope="segment",
status="started",
- workflow_id=workflow_id,
+ workflow_id=_segment_workflow_id(input.job_id, segment),
+ )
+ logger.info(
+ "Account-property segment sync started",
+ team_id=input.team_id,
+ job_id=input.job_id,
+ segment=segment,
+ path=path,
)
@@ -364,6 +397,103 @@ async def sync_warehouse_account_properties_activity(input: AccountPropertySyncI
return result
[email protected](name="list-warehouse-account-property-sources")
+async def list_warehouse_account_property_sources_activity(input: AccountPropertySyncInput) -> list[str]:
+ source_ids = await database_sync_to_async(get_account_property_sync_source_ids, thread_sensitive=False)(
+ input.team_id, saved_query_binding(input.saved_query_id)
+ )
+ if len(source_ids) > ACCOUNT_PROPERTY_FANOUT_MAX_SOURCES:
+ raise ApplicationError(
+ f"View has {len(source_ids)} account-property sources, above the fanout limit of "
+ f"{ACCOUNT_PROPERTY_FANOUT_MAX_SOURCES}",
+ type="AccountPropertyFanoutSourceLimitExceeded",
+ non_retryable=True,
+ )
+ return source_ids
+
+
[email protected](name="sync-warehouse-account-property-source")
+async def sync_warehouse_account_property_source_activity(input: AccountPropertySourceSyncInput) -> dict[str, int]:
+ segment = AccountPropertySyncSegment(input.segment)
+ activity_info = activity.info()
+ log = logger.bind(
+ team_id=input.team_id,
+ saved_query_id=input.saved_query_id,
+ job_id=input.job_id,
+ segment=segment.value,
+ source_id=input.source_id,
+ )
+ telemetry = _AttemptTelemetry.start(
+ team_id=input.team_id,
+ saved_query_id=input.saved_query_id,
+ job_id=input.job_id,
+ segment=segment.value,
+ path="fanout",
+ source_id=input.source_id,
+ )
+ try:
+ source_uuid = UUID(input.source_id)
+ except ValueError as error:
+ telemetry.finish("failed", error=error)
+ raise ApplicationError("Account-property source ID is not a UUID", non_retryable=True) from error
+ try:
+ await database_sync_to_async(update_account_property_sync_runs_phase)(
+ AccountPropertySyncRunContext(
+ team_id=input.team_id,
+ saved_query_id=input.saved_query_id,
+ job_id=input.job_id,
+ ),
+ phase=SyncPhase.SYNCING,
+ workflow_id=activity_info.workflow_id,
+ workflow_run_id=activity_info.workflow_run_id,
+ attempt=activity_info.attempt,
+ segment=segment,
+ source_id=source_uuid,
+ )
+ async with Heartbeater():
+ result = await run_account_property_source_sync(
+ team_id=input.team_id,
+ binding=saved_query_binding(input.saved_query_id),
+ job_id=input.job_id,
+ segment=segment,
+ source_id=input.source_id,
+ final_attempt=activity_info.attempt >= ACCOUNT_PROPERTY_ACTIVITY_MAX_ATTEMPTS,
+ )
+ except (AccountPropertySourceValueError, AccountPropertySourceRejectedError) as error:
+ # A retry reads the same staged rows and source config, so it can't fix these errors.
+ failure_counts = error.counts if isinstance(error, AccountPropertySourceValueError) else None
+ telemetry.finish("failed", counts=failure_counts, error=error)
+ log.warning("Account-property source sync rejected", error_type=type(error).__name__)
+ details = () if failure_counts is None else (failure_counts,)
+ raise ApplicationError(str(error), *details, type=type(error).__name__, non_retryable=True) from error
+ except asyncio.CancelledError:
+ telemetry.finish("cancelled")
+ raise
+ except Exception as error:
+ telemetry.finish("failed", error=error)
+ log.exception("Account-property source sync failed")
+ capture_exception(error)
+ raise
+
+ telemetry.finish("completed", counts=result)
+ log.info("Account-property source sync completed", **result)
+ return result
+
+
[email protected](name="complete-warehouse-account-property-segment")
+async def complete_warehouse_account_property_segment_activity(input: AccountPropertySyncInput) -> None:
+ try:
+ async with Heartbeater():
+ await complete_account_property_sync_segment(
+ team_id=input.team_id,
+ binding=saved_query_binding(input.saved_query_id),
+ job_id=input.job_id,
+ segment=AccountPropertySyncSegment(input.segment),
+ )
+ except AccountPropertySyncSegmentIncompleteError as error:
+ raise ApplicationError(str(error), type=type(error).__name__, non_retryable=True) from error
+
+
@activity.defn(name="emit-warehouse-account-property-segment-telemetry")
async def emit_warehouse_account_property_segment_telemetry_activity(
input: AccountPropertySegmentTelemetryInput,
@@ -561,9 +691,135 @@ class SyncWarehouseAccountPropertiesWorkflow(PostHogWorkflow):
)
+@frozen
+class _SettledSources:
+ counts: dict[str, int]
+ failed: int
+
+
+async def _sync_sources(input: AccountPropertySyncInput, source_ids: list[str]) -> _SettledSources:
+ semaphore = asyncio.Semaphore(ACCOUNT_PROPERTY_SOURCE_CONCURRENCY)
+
+ async def sync_source(source_id: str) -> dict[str, int]:
+ async with semaphore:
+ return await workflow.execute_activity(
+ sync_warehouse_account_property_source_activity,
+ AccountPropertySourceSyncInput(
+ team_id=input.team_id,
+ saved_query_id=input.saved_query_id,
+ job_id=input.job_id,
+ segment=input.segment,
+ source_id=source_id,
+ ),
+ activity_id=f"sync-account-property-source-{source_id}",
+ start_to_close_timeout=timedelta(hours=6),
+ heartbeat_timeout=timedelta(minutes=5),
+ retry_policy=RetryPolicy(
+ maximum_attempts=ACCOUNT_PROPERTY_ACTIVITY_MAX_ATTEMPTS,
+ initial_interval=timedelta(seconds=30),
+ ),
+ )
+
+ # return_exceptions keeps one failed source from ending the gather while its siblings still run.
+ # The coordinator must not finalize the segment before every source activity settles.
+ results = await asyncio.gather(*(sync_source(source_id) for source_id in source_ids), return_exceptions=True)
+ counts: dict[str, int] = {}
+ failed = 0
+ for result in results:
+ if isinstance(result, BaseException):
+ failed += 1
+ continue
+ for key, value in result.items():
+ counts[key] = counts.get(key, 0) + value
+ counts["source_errors"] = failed
+ return _SettledSources(counts=counts, failed=failed)
+
+
[email protected](name=ACCOUNT_PROPERTY_SYNC_FANOUT_WORKFLOW_NAME)
+class SyncWarehouseAccountPropertiesFanoutWorkflow(PostHogWorkflow):
+ @staticmethod
+ def parse_inputs(inputs: list[str]) -> AccountPropertySyncInput:
+ return AccountPropertySyncInput(**json.loads(inputs[0]))
+
+ @workflow.run
+ async def run(self, input: AccountPropertySyncInput) -> None:
+ started_at = workflow.time()
+ source_count: int | None = None
+ counts: dict[str, int] | None = None
+ try:
+ listed_source_ids = await workflow.execute_activity(
+ list_warehouse_account_property_sources_activity,
+ input,
+ start_to_close_timeout=timedelta(minutes=5),
+ retry_policy=RetryPolicy(maximum_attempts=ACCOUNT_PROPERTY_ACTIVITY_MAX_ATTEMPTS),
+ )
+ source_ids = list(dict.fromkeys(listed_source_ids))
+ source_count = len(source_ids)
+ settled = await _sync_sources(input, source_ids)
+ counts = settled.counts
+ if settled.failed:
+ raise ApplicationError(
+ f"{settled.failed} of {source_count} account-property sources failed",
+ type=_SOURCE_SYNC_FAILED_ERROR_TYPE,
+ non_retryable=True,
+ )
+ await workflow.execute_activity(
+ complete_warehouse_account_property_segment_activity,
+ input,
+ start_to_close_timeout=timedelta(hours=1),
+ heartbeat_timeout=timedelta(minutes=5),
+ retry_policy=RetryPolicy(
+ maximum_attempts=ACCOUNT_PROPERTY_ACTIVITY_MAX_ATTEMPTS,
+ initial_interval=timedelta(seconds=30),
+ ),
+ )
+ except (asyncio.CancelledError, Exception) as error:
+ # Workflow eviction raises a BaseException that is not caught here, so cleanup runs only for
+ # a real failure or cancellation.
+ cancelled = is_cancelled_exception(error)
+ try:
+ if not cancelled:
+ # Closes only the runs still open in this segment. Completed sibling runs keep their
+ # result, and the staged rows stay in place for the next run of the source view.
+ await workflow.execute_activity(
+ finalize_warehouse_account_property_runs_activity,
+ FinalizeAccountPropertySyncRunsInput(
+ team_id=input.team_id,
+ saved_query_id=input.saved_query_id,
+ job_id=input.job_id,
+ status=SyncStatus.FAILED.value,
+ phase=SyncPhase.SYNCING.value,
+ error=_SYNC_FAILED_ERROR,
+ segment=input.segment,
+ ),
+ start_to_close_timeout=timedelta(minutes=5),
+ retry_policy=RetryPolicy(maximum_attempts=ACCOUNT_PROPERTY_ACTIVITY_MAX_ATTEMPTS),
+ )
+ finally:
+ await _emit_segment_telemetry(
+ input,
+ path="fanout",
+ status="cancelled" if cancelled else "failed",
+ started_at=started_at,
+ source_count=source_count,
+ counts=counts,
+ error=error,
+ )
+ raise
+ await _emit_segment_telemetry(
+ input,
+ path="fanout",
+ status="completed",
+ started_at=started_at,
+ source_count=source_count,
+ counts=counts,
+ )
+
+
ACCOUNT_PROPERTY_SYNC_WORKFLOWS = [
StageWarehouseAccountPropertiesWorkflow,
SyncWarehouseAccountPropertiesWorkflow,
+ SyncWarehouseAccountPropertiesFanoutWorkflow,
]
ACCOUNT_PROPERTY_SYNC_ACTIVITIES = [
start_warehouse_account_property_runs_activity,
@@ -571,5 +827,8 @@ ACCOUNT_PROPERTY_SYNC_ACTIVITIES = [
stage_warehouse_account_property_files_activity,
dispatch_warehouse_account_property_sync_activity,
sync_warehouse_account_properties_activity,
+ list_warehouse_account_property_sources_activity,
+ sync_warehouse_account_property_source_activity,
+ complete_warehouse_account_property_segment_activity,
emit_warehouse_account_property_segment_telemetry_activity,
]
diff --git a/products/customer_analytics/backend/temporal/account_property_sync_test.py b/products/customer_analytics/backend/temporal/account_property_sync_test.py
index 295b0447a268..5852a03c2667 100644
--- a/products/customer_analytics/backend/temporal/account_property_sync_test.py
+++ b/products/customer_analytics/backend/temporal/account_property_sync_test.py
@@ -1,8 +1,9 @@
import uuid
import asyncio
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Callable
from contextlib import asynccontextmanager
from datetime import timedelta
+from typing import Any
import pytest
from unittest.mock import AsyncMock, MagicMock, patch
@@ -22,22 +23,29 @@ from temporalio.worker import Replayer, UnsandboxedWorkflowRunner, Worker
from products.customer_analytics.backend.facade.temporal_contracts import (
AccountPropertySegmentTelemetryInput,
+ AccountPropertySourceSyncInput,
DispatchAccountPropertySyncInput,
FinalizeAccountPropertySyncRunsInput,
StageAccountPropertySyncInput,
)
from products.customer_analytics.backend.logic.account_property_sync import AccountPropertySourceValueError
from products.customer_analytics.backend.temporal.account_property_sync import (
+ ACCOUNT_PROPERTY_SYNC_ACTIVITIES,
ACCOUNT_PROPERTY_SYNC_WORKFLOW_NAME,
+ ACCOUNT_PROPERTY_SYNC_WORKFLOWS,
AccountPropertySyncInput,
StageWarehouseAccountPropertiesWorkflow,
+ SyncWarehouseAccountPropertiesFanoutWorkflow,
SyncWarehouseAccountPropertiesWorkflow,
+ complete_warehouse_account_property_segment_activity,
dispatch_warehouse_account_property_sync_activity,
emit_warehouse_account_property_segment_telemetry_activity,
finalize_warehouse_account_property_runs_activity,
+ list_warehouse_account_property_sources_activity,
stage_warehouse_account_property_files_activity,
start_warehouse_account_property_runs_activity,
sync_warehouse_account_properties_activity,
+ sync_warehouse_account_property_source_activity,
)
pytestmark = pytest.mark.asyncio
@@ -45,6 +53,7 @@ pytestmark = pytest.mark.asyncio
_MODULE = "products.customer_analytics.backend.temporal.account_property_sync"
_SOURCE_COUNTS = {"rows_read": 10, "changed": 4, "matched": 3, "written": 2, "source_errors": 0, "source_count": 2}
_FAILED_COUNTS = {**_SOURCE_COUNTS, "written": 1, "source_errors": 1}
+_FANOUT_COUNTS = {**_SOURCE_COUNTS, "source_count": 1}
def _create_activity_error(cause: Exception) -> ActivityError:
@@ -221,7 +230,16 @@ async def test_staging_and_dispatch_recover_from_transient_activity_failures() -
assert attempts == [("stage", 1), ("stage", 2), ("dispatch", 1), ("dispatch", 2)]
-async def test_dispatch_starts_missing_segment_when_its_sibling_already_runs() -> None:
+ "fanout_enabled,workflow_name",
+ [
+ (False, "sync-warehouse-account-properties"),
+ (True, "sync-warehouse-account-properties-fanout"),
+ ],
+)
+async def test_dispatch_selects_one_workflow_type_and_keeps_stable_segment_ids(
+ fanout_enabled: bool, workflow_name: str
+) -> None:
client = AsyncMock()
client.start_workflow.side_effect = [WorkflowAlreadyStartedError("tracked", "workflow"), None]
input = DispatchAccountPropertySyncInput(
@@ -231,25 +249,23 @@ async def test_dispatch_starts_missing_segment_when_its_sibling_already_runs() -
)
with (
- patch("products.customer_analytics.backend.temporal.account_property_sync.async_connect", return_value=client),
- patch("products.customer_analytics.backend.temporal.account_property_sync.activity.info") as activity_info,
- patch(
- "products.customer_analytics.backend.temporal.account_property_sync.update_account_property_sync_runs_phase"
- ),
+ patch(f"{_MODULE}.async_connect", return_value=client),
+ patch(f"{_MODULE}.account_property_sync_fanout_enabled", return_value=fanout_enabled) as flag,
patch(
"products.customer_analytics.backend.logic.account_property_sync_monitoring.ph_background_capture"
) as capture_factory,
+ patch(f"{_MODULE}.activity.info") as activity_info,
+ patch(f"{_MODULE}.update_account_property_sync_runs_phase"),
):
activity_info.return_value.attempt = 1
activity_info.return_value.workflow_id = "stage-workflow-job-1"
await dispatch_warehouse_account_property_sync_activity(input)
+ flag.assert_called_once_with(7)
calls = client.start_workflow.await_args_list
- assert [call.args[0] for call in calls] == [
- ACCOUNT_PROPERTY_SYNC_WORKFLOW_NAME,
- ACCOUNT_PROPERTY_SYNC_WORKFLOW_NAME,
- ]
+ assert [call.args[0] for call in calls] == [workflow_name, workflow_name]
assert [call.args[1].segment for call in calls] == ["tracked", "ignored"]
+ # Both workflow types share these IDs, so a retry after a flag change finds the running segment.
assert [call.kwargs["id"] for call in calls] == [
"sync-warehouse-account-properties-job-1-tracked",
"sync-warehouse-account-properties-job-1-ignored",
@@ -260,7 +276,7 @@ async def test_dispatch_starts_missing_segment_when_its_sibling_already_runs() -
assert {key: properties[key] for key in ("scope", "status", "path", "segment", "workflow_id")} == {
"scope": "segment",
"status": "started",
- "path": "legacy",
+ "path": "fanout" if fanout_enabled else "legacy",
"segment": "ignored",
"workflow_id": "sync-warehouse-account-properties-job-1-ignored",
}
@@ -340,6 +356,232 @@ async def test_legacy_workflow_preserves_sync_outcome_when_telemetry_times_out(s
await SyncWarehouseAccountPropertiesWorkflow().run(_segment_input())
+class _FakeFanoutActivities:
+ def __init__(self, source_ids: list[str]) -> None:
+ loop = asyncio.get_running_loop()
+ self.source_ids = source_ids
+ self.log: list[str] = []
+ self.inputs: dict[str, object] = {}
+ self.in_flight = 0
+ self.max_in_flight = 0
+ self.started = {source_id: asyncio.Event() for source_id in source_ids}
+ self.results: dict[str, asyncio.Future[dict[str, int]]] = {
+ source_id: loop.create_future() for source_id in source_ids
+ }
+
+ async def execute_activity(self, activity_fn: Callable[..., Any], arg: object, **_kwargs: object) -> object:
+ if activity_fn is list_warehouse_account_property_sources_activity:
+ return list(self.source_ids)
+ if activity_fn is sync_warehouse_account_property_source_activity:
+ assert isinstance(arg, AccountPropertySourceSyncInput)
+ return await self._sync_source(arg.source_id)
+ self.log.append(activity_fn.__name__)
+ self.inputs[activity_fn.__name__] = arg
+ return None
+
+ async def _sync_source(self, source_id: str) -> dict[str, int]:
+ self.in_flight += 1
+ self.max_in_flight = max(self.max_in_flight, self.in_flight)
+ self.log.append(f"started:{source_id}")
+ self.started[source_id].set()
+ try:
+ result = await self.results[source_id]
+ except asyncio.CancelledError:
+ self.log.append(f"cancelled:{source_id}")
+ raise
+ except Exception:
+ self.log.append(f"failed:{source_id}")
+ raise
+ finally:
+ self.in_flight -= 1
+ self.log.append(f"completed:{source_id}")
+ return result
+
+ def started_sources(self) -> list[str]:
+ return [entry.removeprefix("started:") for entry in self.log if entry.startswith("started:")]
+
+ def telemetry(self) -> AccountPropertySegmentTelemetryInput:
+ telemetry = self.inputs[emit_warehouse_account_property_segment_telemetry_activity.__name__]
+ assert isinstance(telemetry, AccountPropertySegmentTelemetryInput)
+ return telemetry
+
+
+async def _wait(event: asyncio.Event) -> None:
+ # Guards against a hang if the coordinator never starts the source. Correct runs never reach it.
+ await asyncio.wait_for(event.wait(), timeout=5)
+
+
+async def test_fanout_runs_sources_in_parallel_up_to_the_cap() -> None:
+ source_ids = ["source-1", "source-2", "source-3", "source-4", "source-5"]
+ fake = _FakeFanoutActivities(source_ids)
+
+ with (
+ patch.object(workflow, "execute_activity", new=fake.execute_activity),
+ patch.object(workflow, "time", side_effect=[100.0, 145.0]),
+ ):
+ run = asyncio.create_task(SyncWarehouseAccountPropertiesFanoutWorkflow().run(_segment_input()))
+ await _wait(fake.started["source-3"])
+ assert fake.started_sources() == ["source-1", "source-2", "source-3"]
+
+ fake.results["source-2"].set_result(dict(_FANOUT_COUNTS))
+ await _wait(fake.started["source-4"])
+ assert fake.in_flight == 3
+ assert fake.started_sources() == ["source-1", "source-2", "source-3", "source-4"]
+
+ for source_id in ("source-1", "source-3", "source-4", "source-5"):
+ if not fake.results[source_id].done():
+ fake.results[source_id].set_result(dict(_FANOUT_COUNTS))
+ await asyncio.wait_for(run, timeout=5)
+
+ assert fake.max_in_flight == 3
+ complete = complete_warehouse_account_property_segment_activity.__name__
+ assert fake.log.count(complete) == 1
+ assert all(fake.log.index(f"completed:{source_id}") < fake.log.index(complete) for source_id in source_ids)
+ telemetry = fake.telemetry()
+ assert (telemetry.path, telemetry.status, telemetry.duration_seconds, telemetry.source_count) == (
+ "fanout",
+ "completed",
+ 45.0,
+ 5,
+ )
+ assert telemetry.counts == {
+ "rows_read": 50,
+ "changed": 20,
+ "matched": 15,
+ "written": 10,
+ "source_errors": 0,
+ "source_count": 5,
+ }
+
+
+async def test_fanout_waits_for_running_siblings_before_failing_the_segment() -> None:
+ fake = _FakeFanoutActivities(["source-a", "source-b", "source-c"])
+ fake.results["source-a"].set_result(dict(_FANOUT_COUNTS))
+ fake.results["source-b"].set_exception(RuntimeError("source failed"))
+
+ with (
+ patch.object(workflow, "execute_activity", new=fake.execute_activity),
+ patch.object(workflow, "time", return_value=100.0),
+ ):
+ run = asyncio.create_task(SyncWarehouseAccountPropertiesFanoutWorkflow().run(_segment_input()))
+ await _wait(fake.started["source-c"])
+ fake.results["source-c"].set_result(dict(_FANOUT_COUNTS))
+ with pytest.raises(ApplicationError):
+ await asyncio.wait_for(run, timeout=5)
+
+ finalize = finalize_warehouse_account_property_runs_activity.__name__
+ assert "cancelled:source-c" not in fake.log
+ assert fake.log.index("completed:source-c") < fake.log.index(finalize)
+ assert complete_warehouse_account_property_segment_activity.__name__ not in fake.log
+ finalization = fake.inputs[finalize]
+ assert isinstance(finalization, FinalizeAccountPropertySyncRunsInput)
+ assert (finalization.status, finalization.segment) == ("failed", "tracked")
+ telemetry = fake.telemetry()
+ assert (telemetry.status, telemetry.source_count) == ("failed", 3)
+ assert telemetry.counts == {
+ "rows_read": 20,
+ "changed": 8,
+ "matched": 6,
+ "written": 4,
+ "source_errors": 1,
+ "source_count": 2,
+ }
+
+
+async def test_fanout_finishes_a_segment_without_sources() -> None:
+ fake = _FakeFanoutActivities([])
+
+ with (
+ patch.object(workflow, "execute_activity", new=fake.execute_activity),
+ patch.object(workflow, "time", return_value=100.0),
+ ):
+ await SyncWarehouseAccountPropertiesFanoutWorkflow().run(_segment_input())
+
+ assert fake.log == [
+ complete_warehouse_account_property_segment_activity.__name__,
+ emit_warehouse_account_property_segment_telemetry_activity.__name__,
+ ]
+ assert (fake.telemetry().status, fake.telemetry().source_count) == ("completed", 0)
+
+
+async def test_fanout_cancellation_keeps_runs_open_and_reports_cancelled() -> None:
+ fake = _FakeFanoutActivities(["source-a"])
+
+ with (
+ patch.object(workflow, "execute_activity", new=fake.execute_activity),
+ patch.object(workflow, "time", return_value=100.0),
+ ):
+ run = asyncio.create_task(SyncWarehouseAccountPropertiesFanoutWorkflow().run(_segment_input()))
+ await _wait(fake.started["source-a"])
+ run.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await run
+
+ assert finalize_warehouse_account_property_runs_activity.__name__ not in fake.log
+ assert complete_warehouse_account_property_segment_activity.__name__ not in fake.log
+ assert fake.telemetry().status == "cancelled"
+
+
+async def test_fanout_workflow_retries_each_source_independently_and_replays() -> None:
+ attempts: list[tuple[str, int]] = []
+ completed: list[str] = []
+ telemetry: list[AccountPropertySegmentTelemetryInput] = []
+
+ @activity.defn(name="list-warehouse-account-property-sources")
+ async def list_sources(_input: AccountPropertySyncInput) -> list[str]:
+ return ["source-a", "source-b"]
+
+ @activity.defn(name="sync-warehouse-account-property-source")
+ async def sync_source(input: AccountPropertySourceSyncInput) -> dict[str, int]:
+ attempt = activity.info().attempt
+ attempts.append((input.source_id, attempt))
+ if input.source_id == "source-a" and attempt == 1:
+ raise RuntimeError("warehouse temporarily unavailable")
+ return dict(_FANOUT_COUNTS)
+
+ @activity.defn(name="complete-warehouse-account-property-segment")
+ async def complete_segment(input: AccountPropertySyncInput) -> None:
+ completed.append(input.segment)
+
+ @activity.defn(name="finalize-warehouse-account-property-runs")
+ async def finalize_runs(_input: FinalizeAccountPropertySyncRunsInput) -> None:
+ raise AssertionError("A successful segment must not finalize runs as failed")
+
+ @activity.defn(name="emit-warehouse-account-property-segment-telemetry")
+ async def emit_telemetry(input: AccountPropertySegmentTelemetryInput) -> None:
+ telemetry.append(input)
+
+ fakes: list[Callable[..., Any]] = [list_sources, sync_source, complete_segment, finalize_runs, emit_telemetry]
+ registered = {getattr(fn, "__temporal_activity_definition").name for fn in ACCOUNT_PROPERTY_SYNC_ACTIVITIES}
+ assert {getattr(fn, "__temporal_activity_definition").name for fn in fakes} <= registered
+
+ task_queue = str(uuid.uuid4())
+ async with await WorkflowEnvironment.start_time_skipping() as environment:
+ async with Worker(
+ environment.client,
+ task_queue=task_queue,
+ workflows=ACCOUNT_PROPERTY_SYNC_WORKFLOWS,
+ activities=fakes,
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ):
+ handle = await environment.client.start_workflow(
+ SyncWarehouseAccountPropertiesFanoutWorkflow.run,
+ _segment_input(),
+ id=str(uuid.uuid4()),
+ task_queue=task_queue,
+ execution_timeout=timedelta(minutes=10),
+ )
+ await handle.result()
+ history = await handle.fetch_history()
+
+ await Replayer(
+ workflows=[SyncWarehouseAccountPropertiesFanoutWorkflow], workflow_runner=UnsandboxedWorkflowRunner()
+ ).replay_workflow(history)
+ assert sorted(attempts) == [("source-a", 1), ("source-a", 2), ("source-b", 1)]
+ assert completed == ["tracked"]
+ assert [(event.status, event.source_count) for event in telemetry] == [("completed", 2)]
+
+
@workflow.defn(name=ACCOUNT_PROPERTY_SYNC_WORKFLOW_NAME)
class _PreTelemetrySyncWorkflow:
@workflow.run
@@ -417,26 +659,60 @@ async def test_legacy_workflow_replays_histories_from_before_and_after_the_telem
}
+_ATTEMPT_OUTCOMES = [
+ ("completed", dict(_SOURCE_COUNTS), "completed", None),
+ ("transient_failure", RuntimeError("warehouse unavailable"), "failed", "RuntimeError"),
+ (
+ "invalid_values",
+ AccountPropertySourceValueError("bad value", counts=_FAILED_COUNTS),
+ "failed",
+ "AccountPropertySourceValueError",
+ ),
+ ("cancelled", asyncio.CancelledError(), "cancelled", None),
+]
+
+
@pytest.mark.parametrize(
- "runtime_result,expected_status,expected_error_type",
+ "path,activity_fn,runtime_target,activity_input,expected_source_id",
[
- (dict(_SOURCE_COUNTS), "completed", None),
- (RuntimeError("warehouse unavailable"), "failed", "RuntimeError"),
(
- AccountPropertySourceValueError("bad value", counts=_FAILED_COUNTS),
- "failed",
- "AccountPropertySourceValueError",
+ "legacy",
+ sync_warehouse_account_properties_activity,
+ "run_account_property_segment_sync",
+ _segment_input(),
+ None,
+ ),
+ (
+ "fanout",
+ sync_warehouse_account_property_source_activity,
+ "run_account_property_source_sync",
+ AccountPropertySourceSyncInput(
+ team_id=7,
+ saved_query_id="019f0000-0000-7000-8000-000000000001",
+ job_id="job-1",
+ segment="tracked",
+ source_id="019f0000-0000-7000-8000-0000000000aa",
+ ),
+ "019f0000-0000-7000-8000-0000000000aa",
),
- (asyncio.CancelledError(), "cancelled", None),
],
)
[email protected]("_outcome,runtime_result,expected_status,expected_error_type", _ATTEMPT_OUTCOMES)
async def test_sync_activity_reports_attempt_lifecycle(
- runtime_result: object, expected_status: str, expected_error_type: str | None
+ path: str,
+ activity_fn: Callable[[Any], Any],
+ runtime_target: str,
+ activity_input: object,
+ expected_source_id: str | None,
+ _outcome: str,
+ runtime_result: object,
+ expected_status: str,
+ expected_error_type: str | None,
) -> None:
runtime = AsyncMock(side_effect=[runtime_result])
with (
- patch(f"{_MODULE}.run_account_property_segment_sync", new=runtime),
+ patch(f"{_MODULE}.{runtime_target}", new=runtime),
patch(f"{_MODULE}.capture_account_property_sync_event") as capture_event,
patch(f"{_MODULE}.capture_exception"),
patch(f"{_MODULE}.saved_query_binding"),
@@ -449,18 +725,18 @@ async def test_sync_activity_reports_attempt_lifecycle(
activity_info.return_value.workflow_run_id = "segment-run"
if isinstance(runtime_result, BaseException):
with pytest.raises(BaseException) as raised:
- await sync_warehouse_account_properties_activity(_segment_input())
+ await activity_fn(activity_input)
if isinstance(runtime_result, AccountPropertySourceValueError):
assert isinstance(raised.value, ApplicationError) and raised.value.non_retryable
assert raised.value.type == "AccountPropertySourceValueError"
assert raised.value.details == (_FAILED_COUNTS,)
else:
- assert await sync_warehouse_account_properties_activity(_segment_input()) == runtime_result
+ assert await activity_fn(activity_input) == runtime_result
events = [call.kwargs for call in capture_event.call_args_list]
assert [event["status"] for event in events] == ["started", expected_status]
assert {(event["path"], event["scope"], event["source_id"], event["attempt"]) for event in events} == {
- ("legacy", "attempt", None, 2)
+ (path, "attempt", expected_source_id, 2)
}
assert events[1]["error_type"] == expected_error_type
assert events[1]["duration_seconds"] is not None