:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(customer-analytics): add source-scoped account property sync (#119579)
Arthur Moreira de Deus827667b92aa9
diff --git a/products/customer_analytics/backend/logic/account_property_runs.py b/products/customer_analytics/backend/logic/account_property_runs.pyindex 32324ca18a3f..61f182530d2b 100644--- a/products/customer_analytics/backend/logic/account_property_runs.py+++ b/products/customer_analytics/backend/logic/account_property_runs.py@@ -40,6 +40,10 @@ class AccountPropertySyncRunOutcome:_ACCOUNT_SEGMENTS = {segment.value for segment in SyncSegment}+class AccountPropertySyncSegmentIncompleteError(Exception):+ pass++def start_account_property_sync_runs(context: AccountPropertySyncRunContext,*,@@ -127,6 +131,7 @@ def update_account_property_sync_runs_phase(workflow_run_id: str | None,attempt: int,segment: SyncSegment | None = None,+ source_id: UUID | None = None,) -> None:runs = CustomPropertySyncRun.objects.for_team(context.team_id).filter(job_id=context.job_id,@@ -135,6 +140,8 @@ def update_account_property_sync_runs_phase()if segment is not None:runs = runs.filter(segment=segment.value)+ if source_id is not None:+ runs = runs.filter(source_id=source_id)if phase == SyncPhase.DISPATCHING:runs = runs.filter(phase__in=[SyncPhase.STAGING.value, SyncPhase.DISPATCHING.value])runs.update(@@ -199,6 +206,7 @@ def finalize_account_property_sync_runs(phase: SyncPhase,error: str | None = None,segment: SyncSegment | None = None,+ source_id: UUID | None = None,) -> None:with transaction.atomic():runs = (@@ -212,6 +220,8 @@ def finalize_account_property_sync_runs()if segment is not None:runs = runs.filter(segment=segment.value)+ if source_id is not None:+ runs = runs.filter(source_id=source_id)transitioned_source_ids = set(runs.values_list("source_id", flat=True))runs.update(status=status.value,@@ -222,6 +232,44 @@ def finalize_account_property_sync_runs(_update_source_status_for_terminal_runs(context.team_id, context.job_id, transitioned_source_ids)+def complete_account_property_sync_segment_runs(context: AccountPropertySyncRunContext, segment: SyncSegment) -> None:+ with transaction.atomic():+ runs = list(+ CustomPropertySyncRun.objects.for_team(context.team_id)+ .select_for_update()+ .filter(+ job_id=context.job_id,+ saved_query_id=context.saved_query_id,+ segment=segment.value,+ )+ )+ eligible_source_ids = set(+ CustomPropertySource.objects.for_team(context.team_id)+ .filter(+ saved_query_id=context.saved_query_id,+ is_enabled=True,+ definition__target_type=TargetType.ACCOUNT.value,+ source_column__isnull=False,+ )+ .values_list("id", flat=True)+ )+ # Staged rows must remain available until every eligible source completes.+ if eligible_source_ids - {run.source_id for run in runs} or any(+ run.status == SyncStatus.FAILED.value+ or (run.source_id in eligible_source_ids and run.status != SyncStatus.COMPLETED.value)+ for run in runs+ ):+ raise AccountPropertySyncSegmentIncompleteError(+ f"Account-property segment {segment.value} has unfinished or failed source runs"+ )+ finalize_account_property_sync_runs(+ context,+ status=SyncStatus.COMPLETED,+ phase=SyncPhase.COMPLETED,+ segment=segment,+ )++def finish_account_property_sync_runs(context: AccountPropertySyncRunContext,segment: SyncSegment,diff --git a/products/customer_analytics/backend/logic/account_property_runs_test.py b/products/customer_analytics/backend/logic/account_property_runs_test.pyindex 72649a18a237..53c6fcb19057 100644--- a/products/customer_analytics/backend/logic/account_property_runs_test.py+++ b/products/customer_analytics/backend/logic/account_property_runs_test.py@@ -1,4 +1,6 @@+from collections.abc import Callablefrom datetime import timedelta+from uuid import UUIDfrom posthog.test.base import BaseTestfrom unittest.mock import patch@@ -6,6 +8,8 @@ from unittest.mock import patchfrom django.apps import appsfrom django.utils import timezone+from parameterized import parameterized+from products.customer_analytics.backend.facade import apifrom products.customer_analytics.backend.logic.account_property_runs import (AccountPropertySyncRunContext,@@ -163,6 +167,63 @@ class TestAccountPropertyRuns(TeamScopedTestMixin, BaseTest):"00000000-0000-4000-8000-000000000001",)+ @parameterized.expand(+ [+ (+ "phase",+ lambda context, source_id: update_account_property_sync_runs_phase(+ context,+ phase=SyncPhase.SYNCING,+ workflow_id="sync-workflow-job-1-tracked",+ workflow_run_id="00000000-0000-4000-8000-000000000002",+ attempt=2,+ segment=SyncSegment.TRACKED,+ source_id=source_id,+ ),+ ("running", "syncing"),+ ),+ (+ "finalize",+ lambda context, source_id: finalize_account_property_sync_runs(+ context,+ status=SyncStatus.FAILED,+ phase=SyncPhase.SYNCING,+ error="Couldn't update accounts.",+ segment=SyncSegment.TRACKED,+ source_id=source_id,+ ),+ ("failed", "syncing"),+ ),+ ]+ )+ def test_source_scoped_updates_leave_sibling_sources_untouched(+ self,+ _case: str,+ update: Callable[[AccountPropertySyncRunContext, UUID], None],+ expected: tuple[str, str],+ ) -> None:+ sibling = CustomPropertySource.objects.create(+ team=self.team,+ definition=create_custom_property_definition(team_id=self.team.id, name="Tier"),+ saved_query=self.saved_query,+ key_column="external_id",+ source_column="tier",+ )+ self._start_runs()++ update(self.context, self.source.id)++ states = {+ (run.source_id, run.segment): (run.status, run.phase)+ for run in CustomPropertySyncRun.objects.for_team(self.team.id).filter(job_id="job-1")+ }+ assert states == {+ (self.source.id, "tracked"): expected,+ (self.source.id, "ignored"): ("running", "staging"),+ (sibling.id, "tracked"): ("running", "staging"),+ (sibling.id, "ignored"): ("running", "staging"),+ }+def test_account_runs_remain_active_during_their_workflow_timeout(self) -> None:self._start_runs()CustomPropertySyncRun.objects.for_team(self.team.id).filter(source=self.source).update(diff --git a/products/customer_analytics/backend/logic/account_property_sync.py b/products/customer_analytics/backend/logic/account_property_sync.pyindex 2008ab9c4ef5..3deff65b2a97 100644--- a/products/customer_analytics/backend/logic/account_property_sync.py+++ b/products/customer_analytics/backend/logic/account_property_sync.py@@ -10,6 +10,8 @@ from enum import StrEnumfrom typing import Any, Protocolfrom uuid import UUID+from django.db.models import QuerySet+import pyarrow as paimport structlogimport pyarrow.parquet as pq@@ -21,6 +23,7 @@ from posthog.sync import database_sync_to_asyncfrom products.customer_analytics.backend.logic.account_property_runs import (AccountPropertySyncRunContext,AccountPropertySyncRunOutcome,+ complete_account_property_sync_segment_runs,finalize_account_property_sync_runs,finish_account_property_sync_runs,)@@ -48,6 +51,7 @@ logger = structlog.get_logger(__name__)_ACCOUNT_LOOKUP_CHUNK_SIZE = 1_000_PARQUET_BATCH_SIZE = 50_000_SEGMENTS_REQUIRED_FOR_CLEANUP = frozenset({"tracked", "ignored"})+_COUNT_KEYS = ("rows_read", "changed", "matched", "written", "source_errors", "source_count")_RUN_FAILED_ERROR = "Couldn't update accounts. Run the source view again. If it keeps failing, contact support."_INVALID_VALUE_ERROR = ("A warehouse value doesn't match this property's type. Update the source data, then run the view again."@@ -69,6 +73,14 @@ class AccountPropertySourceValueError(Exception):self.counts = counts.copy()+class AccountPropertySourceRejectedError(Exception):+ pass+++def _empty_counts() -> dict[str, int]:+ return dict.fromkeys(_COUNT_KEYS, 0)++@frozenclass AppliedSourceValues:written: int@@ -323,19 +335,47 @@ def _apply_source_values(return AppliedSourceValues(written=written, hashes=applied_hashes, failed=source_failed)+def _enabled_sources_queryset(team_id: int, binding: WarehouseBinding) -> QuerySet[CustomPropertySource]:+ return CustomPropertySource.objects.for_team(team_id).filter(+ saved_query_id=binding.id,+ is_enabled=True,+ definition__target_type=TargetType.ACCOUNT.value,+ source_column__isnull=False,+ )++def _enabled_sources(team_id: int, binding: WarehouseBinding) -> list[CustomPropertySource]:- return list(- CustomPropertySource.objects.for_team(team_id)- .select_related("definition")- .filter(- saved_query_id=binding.id,- is_enabled=True,- definition__target_type=TargetType.ACCOUNT.value,- source_column__isnull=False,- )+ return list(_enabled_sources_queryset(team_id, binding).select_related("definition"))+++def get_account_property_sync_source_ids(team_id: int, binding: WarehouseBinding) -> list[str]:+ source_ids = _enabled_sources_queryset(team_id, binding).order_by("id").values_list("id", flat=True)+ return [str(source_id) for source_id in source_ids]+++def _is_source_applicable(source: CustomPropertySource) -> bool:+ return (+ source.is_enabled+ and source.definition.target_type == TargetType.ACCOUNT.value+ and source.source_column is not None)+def _parse_source_id(source_id: str) -> UUID:+ try:+ return UUID(source_id)+ except ValueError as error:+ raise AccountPropertySourceRejectedError("Account-property source ID is not a UUID") from error+++def _resolve_source(team_id: int, binding: WarehouseBinding, source_id: UUID) -> CustomPropertySource | None:+ sources = CustomPropertySource.objects.for_team(team_id).select_related("definition")+ source = sources.filter(id=source_id, saved_query_id=binding.id).first()+ if source is None and sources.filter(id=source_id).exists():+ raise AccountPropertySourceRejectedError("Account-property source belongs to a different view")+ return source++async def _segment_already_completed(team_id: int, binding: WarehouseBinding, job_id: str, segment: AccountPropertySyncSegment) -> bool:@@ -348,6 +388,41 @@ async def _segment_already_completed(return True+def _source_completion_uri(+ team_id: int, binding: WarehouseBinding, job_id: str, segment: AccountPropertySyncSegment, source_id: UUID+) -> str:+ prefix = account_property_completion_prefix(team_id, binding, job_id)+ # Source markers stay in a subdirectory, so the cleanup barrier never reads them as segment markers.+ return f"s3://{prefix}/sources/{segment.value}/{source_id}.done"+++async def _read_source_completion(+ team_id: int, binding: WarehouseBinding, job_id: str, segment: AccountPropertySyncSegment, source_id: UUID+) -> dict[str, int] | None:+ async with aget_s3_client() as s3_client:+ try:+ data = await s3_client._cat_file(_source_completion_uri(team_id, binding, job_id, segment, source_id))+ except FileNotFoundError:+ return None+ stored = json.loads(data)+ return {key: int(stored.get(key, 0)) for key in _COUNT_KEYS}+++async def _write_source_completion(+ team_id: int,+ binding: WarehouseBinding,+ job_id: str,+ segment: AccountPropertySyncSegment,+ source_id: UUID,+ counts: dict[str, int],+) -> None:+ async with aget_s3_client() as s3_client:+ await s3_client._pipe_file(+ _source_completion_uri(team_id, binding, job_id, segment, source_id),+ json.dumps(counts, sort_keys=True).encode("utf-8"),+ )++async def _mark_completed_and_maybe_cleanup(team_id: int, binding: WarehouseBinding, job_id: str, segment: AccountPropertySyncSegment) -> None:@@ -372,6 +447,167 @@ async def _mark_completed_and_maybe_cleanup(pass+async def _load_prior_hashes(+ *,+ team_id: int,+ binding: WarehouseBinding,+ segment: AccountPropertySyncSegment,+ states: list[SourceSyncState],+ log: FilteringBoundLogger,+) -> None:+ phase_started_at = asyncio.get_running_loop().time()+ for state in states:+ state.prior_hashes.update(await _read_snapshot_hashes(team_id, binding, str(state.source.id), segment))+ _record_phase_duration(+ log,+ segment,+ AccountPropertySyncPhase.LOAD_STATE,+ phase_started_at,+ {"source_count": len(states)},+ )+++async def _apply_staged_rows(+ *,+ team_id: int,+ binding: WarehouseBinding,+ job_id: str,+ segment: AccountPropertySyncSegment,+ states: list[SourceSyncState],+ counts: dict[str, int],+ log: FilteringBoundLogger,+) -> None:+ batch_index = 0+ batches = aiter(_iter_parquet_row_batches(team_id, binding, job_id))+ while True:+ phase_started_at = asyncio.get_running_loop().time()+ try:+ rows = await anext(batches)+ except StopAsyncIteration:+ break+ _record_phase_duration(+ log,+ segment,+ AccountPropertySyncPhase.READ_STAGED_ROWS,+ phase_started_at,+ {"batch_index": batch_index, "batch_rows": len(rows)},+ )+ counts["rows_read"] += len(rows)+ for state in states:+ state.rows_read += len(rows)+ source = state.source+ source_column = source.source_column+ if source_column is None:+ continue++ phase_started_at = asyncio.get_running_loop().time()+ values_by_external_id = _source_values(rows, source.key_column, source_column)+ changed = {+ external_id: value+ for external_id, value in values_by_external_id.items()+ if state.prior_hashes.get(external_id) != _value_hash(value)+ }+ state.changed += len(changed)+ counts["changed"] += len(changed)+ phase_details: dict[str, bool | float | int | str] = {+ "batch_index": batch_index,+ "source_id": str(source.id),+ "candidate_values": len(values_by_external_id),+ "changed_values": len(changed),+ }+ _record_phase_duration(+ log,+ segment,+ AccountPropertySyncPhase.DIFF_VALUES,+ phase_started_at,+ phase_details,+ )+ if not changed:+ continue++ phase_started_at = asyncio.get_running_loop().time()+ account_ids = await database_sync_to_async(_matching_account_ids, thread_sensitive=False)(+ team_id, segment, list(changed)+ )+ state.matched += len(account_ids)+ counts["matched"] += len(account_ids)+ _record_phase_duration(+ log,+ segment,+ AccountPropertySyncPhase.MATCH_ACCOUNTS,+ phase_started_at,+ {**phase_details, "matched_accounts": len(account_ids)},+ )++ phase_started_at = asyncio.get_running_loop().time()+ applied = await database_sync_to_async(_apply_source_values, thread_sensitive=False)(+ team_id, source, account_ids, changed, segment+ )+ state.written += applied.written+ counts["written"] += applied.written+ if applied.failed:+ state.error = _INVALID_VALUE_ERROR+ state.applied_hashes.update(applied.hashes)+ state.prior_hashes.update(applied.hashes)+ _record_phase_duration(+ log,+ segment,+ AccountPropertySyncPhase.APPLY_VALUES,+ phase_started_at,+ {+ **phase_details,+ "matched_accounts": len(account_ids),+ "written_values": applied.written,+ "source_failed": applied.failed,+ },+ )+ batch_index += 1+++async def _persist_states(+ *,+ team_id: int,+ binding: WarehouseBinding,+ job_id: str,+ segment: AccountPropertySyncSegment,+ states: list[SourceSyncState],+ counts: dict[str, int],+ log: FilteringBoundLogger,+) -> None:+ phase_started_at = asyncio.get_running_loop().time()+ persisted_hashes = 0+ for state in states:+ if state.error is not None:+ counts["source_errors"] += 1+ await _write_snapshot_hashes(+ team_id,+ binding,+ str(state.source.id),+ segment,+ job_id,+ state.applied_hashes,+ )+ persisted_hashes += len(state.applied_hashes)+ _record_phase_duration(+ log,+ segment,+ AccountPropertySyncPhase.PERSIST_STATE,+ phase_started_at,+ {"source_count": len(states), "persisted_hashes": persisted_hashes},+ )+++def _run_outcome(state: SourceSyncState, error: str | None) -> AccountPropertySyncRunOutcome:+ return AccountPropertySyncRunOutcome(+ source_id=state.source.id,+ rows_read=state.rows_read,+ changed=state.changed,+ matched=state.matched,+ written=state.written,+ error=error,+ )++async def run_account_property_segment_sync(*,team_id: int,@@ -386,7 +622,7 @@ async def run_account_property_segment_sync(job_id=job_id,segment=segment.value,)- counts = {"rows_read": 0, "changed": 0, "matched": 0, "written": 0, "source_errors": 0, "source_count": 0}+ counts = _empty_counts()states: list[SourceSyncState] = []run_context = AccountPropertySyncRunContext(@@ -404,140 +640,19 @@ async def run_account_property_segment_sync(SourceSyncState(source=source, prior_hashes={}) for source in sources if source.source_column is not None]counts["source_count"] = len(states)- phase_started_at = asyncio.get_running_loop().time()- for state in states:- state.prior_hashes.update(await _read_snapshot_hashes(team_id, binding, str(state.source.id), segment))- _record_phase_duration(- log,- segment,- AccountPropertySyncPhase.LOAD_STATE,- phase_started_at,- {"source_count": len(states)},+ await _load_prior_hashes(team_id=team_id, binding=binding, segment=segment, states=states, log=log)+ await _apply_staged_rows(+ team_id=team_id, binding=binding, job_id=job_id, segment=segment, states=states, counts=counts, log=log)-- batch_index = 0- batches = aiter(_iter_parquet_row_batches(team_id, binding, job_id))- while True:- phase_started_at = asyncio.get_running_loop().time()- try:- rows = await anext(batches)- except StopAsyncIteration:- break- _record_phase_duration(- log,- segment,- AccountPropertySyncPhase.READ_STAGED_ROWS,- phase_started_at,- {"batch_index": batch_index, "batch_rows": len(rows)},- )- counts["rows_read"] += len(rows)- for state in states:- state.rows_read += len(rows)- source = state.source- source_column = source.source_column- if source_column is None:- continue-- phase_started_at = asyncio.get_running_loop().time()- values_by_external_id = _source_values(rows, source.key_column, source_column)- changed = {- external_id: value- for external_id, value in values_by_external_id.items()- if state.prior_hashes.get(external_id) != _value_hash(value)- }- state.changed += len(changed)- counts["changed"] += len(changed)- phase_details: dict[str, bool | float | int | str] = {- "batch_index": batch_index,- "source_id": str(source.id),- "candidate_values": len(values_by_external_id),- "changed_values": len(changed),- }- _record_phase_duration(- log,- segment,- AccountPropertySyncPhase.DIFF_VALUES,- phase_started_at,- phase_details,- )- if not changed:- continue-- phase_started_at = asyncio.get_running_loop().time()- account_ids = await database_sync_to_async(_matching_account_ids, thread_sensitive=False)(- team_id, segment, list(changed)- )- state.matched += len(account_ids)- counts["matched"] += len(account_ids)- _record_phase_duration(- log,- segment,- AccountPropertySyncPhase.MATCH_ACCOUNTS,- phase_started_at,- {**phase_details, "matched_accounts": len(account_ids)},- )-- phase_started_at = asyncio.get_running_loop().time()- applied = await database_sync_to_async(_apply_source_values, thread_sensitive=False)(- team_id, source, account_ids, changed, segment- )- state.written += applied.written- counts["written"] += applied.written- if applied.failed:- state.error = _INVALID_VALUE_ERROR- state.applied_hashes.update(applied.hashes)- state.prior_hashes.update(applied.hashes)- _record_phase_duration(- log,- segment,- AccountPropertySyncPhase.APPLY_VALUES,- phase_started_at,- {- **phase_details,- "matched_accounts": len(account_ids),- "written_values": applied.written,- "source_failed": applied.failed,- },- )- batch_index += 1-- phase_started_at = asyncio.get_running_loop().time()- persisted_hashes = 0- for state in states:- if state.error is not None:- counts["source_errors"] += 1- await _write_snapshot_hashes(- team_id,- binding,- str(state.source.id),- segment,- job_id,- state.applied_hashes,- )- persisted_hashes += len(state.applied_hashes)- _record_phase_duration(- log,- segment,- AccountPropertySyncPhase.PERSIST_STATE,- phase_started_at,- {"source_count": len(states), "persisted_hashes": persisted_hashes},+ await _persist_states(+ team_id=team_id, binding=binding, job_id=job_id, segment=segment, states=states, counts=counts, log=log)except Exception:if final_attempt:await database_sync_to_async(finish_account_property_sync_runs)(run_context,segment,- [- AccountPropertySyncRunOutcome(- source_id=state.source.id,- rows_read=state.rows_read,- changed=state.changed,- matched=state.matched,- written=state.written,- error=_RUN_FAILED_ERROR,- )- for state in states- ],+ [_run_outcome(state, _RUN_FAILED_ERROR) for state in states],)await database_sync_to_async(finalize_account_property_sync_runs)(run_context,@@ -551,17 +666,7 @@ async def run_account_property_segment_sync(await database_sync_to_async(finish_account_property_sync_runs)(run_context,segment,- [- AccountPropertySyncRunOutcome(- source_id=state.source.id,- rows_read=state.rows_read,- changed=state.changed,- matched=state.matched,- written=state.written,- error=state.error,- )- for state in states- ],+ [_run_outcome(state, state.error) for state in states],)await database_sync_to_async(finalize_account_property_sync_runs)(run_context,@@ -578,3 +683,91 @@ async def run_account_property_segment_sync(await _mark_completed_and_maybe_cleanup(team_id, binding, job_id, segment)return counts+++async def run_account_property_source_sync(+ *,+ team_id: int,+ binding: WarehouseBinding,+ job_id: str,+ segment: AccountPropertySyncSegment,+ source_id: str,+ final_attempt: bool = False,+) -> dict[str, int]:+ source_uuid = _parse_source_id(source_id)+ log = logger.bind(+ team_id=team_id,+ saved_query_id=binding.id,+ job_id=job_id,+ segment=segment.value,+ source_id=str(source_uuid),+ )+ counts = _empty_counts()+ state: SourceSyncState | None = None+ run_context = AccountPropertySyncRunContext(+ team_id=team_id,+ saved_query_id=binding.id,+ job_id=job_id,+ )++ try:+ if await _segment_already_completed(team_id, binding, job_id, segment):+ return counts+ if await _read_source_completion(team_id, binding, job_id, segment, source_uuid) is not None:+ return counts++ source = await database_sync_to_async(_resolve_source)(team_id, binding, source_uuid)+ if source is None:+ # Deletion cascades to the source's runs, so nothing is left to finish.+ log.warning("Account-property source not found for this team, skipping it")+ return counts++ state = SourceSyncState(source=source, prior_hashes={})+ if _is_source_applicable(source):+ counts["source_count"] = 1+ states = [state]+ await _load_prior_hashes(team_id=team_id, binding=binding, segment=segment, states=states, log=log)+ await _apply_staged_rows(+ team_id=team_id, binding=binding, job_id=job_id, segment=segment, states=states, counts=counts, log=log+ )+ await _persist_states(+ team_id=team_id, binding=binding, job_id=job_id, segment=segment, states=states, counts=counts, log=log+ )+ else:+ log.info("Account-property source is no longer eligible, finishing it without changes")+ except AccountPropertySourceRejectedError:+ raise+ except Exception:+ if final_attempt:+ failed_outcome = (+ _run_outcome(state, _RUN_FAILED_ERROR)+ if state is not None+ else AccountPropertySyncRunOutcome(+ source_id=source_uuid, rows_read=0, changed=0, matched=0, written=0, error=_RUN_FAILED_ERROR+ )+ )+ await database_sync_to_async(finish_account_property_sync_runs)(run_context, segment, [failed_outcome])+ raise++ await database_sync_to_async(finish_account_property_sync_runs)(+ run_context, segment, [_run_outcome(state, state.error)]+ )+ if state.error is not None:+ raise AccountPropertySourceValueError("An account-property source contained invalid values", counts=counts)++ await _write_source_completion(team_id, binding, job_id, segment, source_uuid, counts)+ return counts+++async def complete_account_property_sync_segment(+ *,+ team_id: int,+ binding: WarehouseBinding,+ job_id: str,+ segment: AccountPropertySyncSegment,+) -> None:+ await database_sync_to_async(complete_account_property_sync_segment_runs)(+ AccountPropertySyncRunContext(team_id=team_id, saved_query_id=binding.id, job_id=job_id),+ segment,+ )+ await _mark_completed_and_maybe_cleanup(team_id, binding, job_id, segment)diff --git a/products/customer_analytics/backend/logic/account_property_sync_test.py b/products/customer_analytics/backend/logic/account_property_sync_test.pyindex 1fe4071452e6..fd6d9563ca28 100644--- a/products/customer_analytics/backend/logic/account_property_sync_test.py+++ b/products/customer_analytics/backend/logic/account_property_sync_test.py@@ -1,8 +1,8 @@from collections.abc import AsyncIterator-from contextlib import AbstractContextManager+from contextlib import AbstractContextManager, ExitStackfrom datetime import UTC, datetime-from typing import Any, Generic, TypeVar-from uuid import uuid4+from typing import TYPE_CHECKING, Any, Generic, TypeVar+from uuid import UUID, uuid4import pytestfrom posthog.test.base import BaseTest@@ -13,13 +13,19 @@ from django.apps import appsimport pyarrow as paimport pyarrow.parquet as pqfrom asgiref.sync import async_to_sync+from parameterized import parameterized++from posthog.dataclasses import frozen+from posthog.models import Teamfrom products.customer_analytics.backend.logic import account_property_sync as apsfrom products.customer_analytics.backend.logic.account_property_runs import (AccountPropertySyncRunContext,+ AccountPropertySyncSegmentIncompleteError,start_account_property_sync_runs,)from products.customer_analytics.backend.logic.account_property_sync import (+ AccountPropertySourceRejectedError,AccountPropertySourceValueError,AccountPropertySyncPhase,AccountPropertySyncSegment,@@ -33,37 +39,356 @@ from products.customer_analytics.backend.logic.account_property_sync import (_source_values,_value_hash,_write_snapshot_hashes,+ complete_account_property_sync_segment,+ get_account_property_sync_source_ids,run_account_property_segment_sync,+ run_account_property_source_sync,)from products.customer_analytics.backend.models import CustomPropertySource, CustomPropertySyncRunfrom products.customer_analytics.backend.models.team_scoped_test_base import TeamScopedTestMixinfrom products.customer_analytics.backend.test.factories import create_account, create_custom_property_definition-from products.warehouse_sources.backend.facade.hooks import saved_query_binding+from products.warehouse_sources.backend.facade.hooks import WarehouseBinding, saved_query_bindingfrom products.warehouse_sources.backend.facade.temporal import (+ account_property_completion_prefix,account_property_job_staged_prefix,account_property_snapshot_prefix,)+if TYPE_CHECKING:+ from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery as SavedQueryModel+_MODULE = "products.customer_analytics.backend.logic.account_property_sync"DataWarehouseSavedQuery = apps.get_model("data_modeling", "DataWarehouseSavedQuery")+_ACCOUNT_IDS = {"acct-1": UUID("00000000-0000-4000-8000-0000000000a1")}+_EMPTY_COUNTS = {"rows_read": 0, "changed": 0, "matched": 0, "written": 0, "source_errors": 0, "source_count": 0}+++@frozen+class _RunState:+ status: str+ phase: str+ rows_read: int+ produced: int+++_STAGING = _RunState(status="running", phase="staging", rows_read=0, produced=0)+_COMPLETED_ONE_ROW = _RunState(status="completed", phase="completed", rows_read=1, produced=1)+_COMPLETED_NO_ROWS = _RunState(status="completed", phase="completed", rows_read=0, produced=0)+++@frozen+class _ViewWithTwoSources:+ binding: WarehouseBinding+ plan_source: CustomPropertySource+ tier_source: CustomPropertySource+++def _match_known_accounts(+ team_id: int, segment: AccountPropertySyncSegment, external_ids: list[str]+) -> dict[str, UUID]:+ return {external_id: _ACCOUNT_IDS[external_id] for external_id in external_ids if external_id in _ACCOUNT_IDS}+++def _apply_all_values(+ team_id: int,+ source: CustomPropertySource,+ account_ids: dict[str, UUID],+ changed: dict[str, Any],+ segment: AccountPropertySyncSegment,+) -> AppliedSourceValues:+ hashes = {external_id: _value_hash(changed[external_id]) for external_id in account_ids}+ return AppliedSourceValues(written=len(hashes), hashes=hashes, failed=False)class AccountPropertySegmentTest(TeamScopedTestMixin, BaseTest):- def _create_source(self) -> CustomPropertySource:- saved_query = DataWarehouseSavedQuery.objects.create(- team=self.team,- name="accounts",- columns={"external_id": {}, "plan": {}},- )- definition = create_custom_property_definition(team_id=self.team.id, name="Plan")- return CustomPropertySource.objects.create(- team=self.team,+ def _create_source(+ self,+ *,+ saved_query: "SavedQueryModel | None" = None,+ team: Team | None = None,+ name: str = "Plan",+ column: str = "plan",+ ) -> CustomPropertySource:+ team = team or self.team+ if saved_query is None:+ saved_query = DataWarehouseSavedQuery.objects.create(+ team=team,+ name="accounts",+ columns={"external_id": {}, "plan": {}, "tier": {}},+ )+ definition = create_custom_property_definition(team_id=team.id, name=name)+ return CustomPropertySource.objects.unscoped().create(+ team=team,definition=definition,saved_query=saved_query,key_column="external_id",- source_column="plan",+ source_column=column,+ )++ def _start_runs(self, source: CustomPropertySource) -> None:+ start_account_property_sync_runs(+ AccountPropertySyncRunContext(+ team_id=source.team_id,+ saved_query_id=str(source.saved_query_id),+ job_id="job-1",+ ),+ workflow_id="stage-workflow-job-1",+ workflow_run_id="00000000-0000-4000-8000-000000000001",)+ def _run_states(self, source: CustomPropertySource) -> dict[str, _RunState]:+ states: dict[str, _RunState] = {}+ for run in CustomPropertySyncRun.objects.for_team(source.team_id).filter(source=source, job_id="job-1"):+ assert run.segment is not None+ assert run.phase is not None+ states[run.segment] = _RunState(+ status=run.status, phase=run.phase, rows_read=run.rows_read, produced=run.produced+ )+ return states++ def _create_view_with_two_sources(self, fake: "_FakeS3", plan: str = "enterprise") -> _ViewWithTwoSources:+ plan_source = self._create_source()+ tier_source = self._create_source(saved_query=plan_source.saved_query, name="Tier", column="tier")+ self._start_runs(plan_source)+ binding = saved_query_binding(str(plan_source.saved_query_id))+ self._stage_rows(fake, binding, [{"external_id": "acct-1", "plan": plan, "tier": "gold"}])+ return _ViewWithTwoSources(binding=binding, plan_source=plan_source, tier_source=tier_source)++ def _stage_rows(self, fake: "_FakeS3", binding: WarehouseBinding, rows: list[dict[str, Any]]) -> None:+ buffer = pa.BufferOutputStream()+ pq.write_table(pa.Table.from_pylist(rows), buffer)+ path = f"{account_property_job_staged_prefix(self.team.id, binding, 'job-1')}/chunk-0.parquet"+ async_to_sync(fake._pipe_file)(path, buffer.getvalue().to_pybytes())++ def _fanout_boundaries(self, fake: "_FakeS3") -> ExitStack:+ stack = ExitStack()+ stack.enter_context(_fake_s3_patch(fake))+ stack.enter_context(patch(f"{_MODULE}._matching_account_ids", side_effect=_match_known_accounts))+ stack.enter_context(patch(f"{_MODULE}._apply_source_values", side_effect=_apply_all_values))+ return stack++ def _run_source(+ self,+ fake: "_FakeS3",+ binding: WarehouseBinding,+ source_id: str,+ *,+ segment: AccountPropertySyncSegment = AccountPropertySyncSegment.TRACKED,+ final_attempt: bool = False,+ ) -> dict[str, int]:+ with self._fanout_boundaries(fake):+ return async_to_sync(run_account_property_source_sync)(+ team_id=self.team.id,+ binding=binding,+ job_id="job-1",+ segment=segment,+ source_id=source_id,+ final_attempt=final_attempt,+ )++ def _complete_segment(+ self, fake: "_FakeS3", binding: WarehouseBinding, segment: AccountPropertySyncSegment+ ) -> None:+ with self._fanout_boundaries(fake):+ async_to_sync(complete_account_property_sync_segment)(+ team_id=self.team.id, binding=binding, job_id="job-1", segment=segment+ )++ def _snapshot(self, fake: "_FakeS3", binding: WarehouseBinding, source: CustomPropertySource) -> dict[str, str]:+ with _fake_s3_patch(fake):+ return async_to_sync(_read_snapshot_hashes)(+ self.team.id, binding, str(source.id), AccountPropertySyncSegment.TRACKED+ )++ def _completion_markers(self, fake: "_FakeS3", binding: WarehouseBinding) -> set[str]:+ prefix = f"{account_property_completion_prefix(self.team.id, binding, 'job-1')}/"+ return {key.removeprefix(prefix) for key in fake.store if key.startswith(prefix)}++ def _staged_files(self, fake: "_FakeS3", binding: WarehouseBinding) -> list[str]:+ prefix = account_property_job_staged_prefix(self.team.id, binding, "job-1")+ return [key for key in fake.store if key.startswith(prefix)]++ def test_source_ids_are_this_views_eligible_sources_in_a_stable_order(self) -> None:+ sources = [self._create_source()]+ view = sources[0].saved_query+ assert view is not None+ sources.append(self._create_source(saved_query=view, name="Tier", column="tier"))+ disabled = self._create_source(saved_query=view, name="Region", column="region")+ disabled.is_enabled = False+ disabled.save(update_fields=["is_enabled"])+ other_view = DataWarehouseSavedQuery.objects.create(team=self.team, name="other_accounts", columns={})+ self._create_source(saved_query=other_view, name="Seats", column="seats")+ other_team = Team.objects.create(organization=self.organization)+ self._create_source(saved_query=view, team=other_team, name="Plan", column="plan")++ source_ids = get_account_property_sync_source_ids(self.team.id, saved_query_binding(str(view.id)))++ assert source_ids == sorted(str(source.id) for source in sources)++ def test_source_sync_updates_only_its_own_source_and_segment(self) -> None:+ fake = _FakeS3()+ view = self._create_view_with_two_sources(fake)++ counts = self._run_source(fake, view.binding, str(view.plan_source.id))++ assert counts == {**_EMPTY_COUNTS, "rows_read": 1, "changed": 1, "matched": 1, "written": 1, "source_count": 1}+ assert self._run_states(view.plan_source) == {+ "tracked": _COMPLETED_ONE_ROW,+ "ignored": _STAGING,+ }+ assert self._run_states(view.tier_source) == {+ "tracked": _STAGING,+ "ignored": _STAGING,+ }+ assert self._snapshot(fake, view.binding, view.plan_source) == {"acct-1": _value_hash("enterprise")}+ assert self._snapshot(fake, view.binding, view.tier_source) == {}+ assert self._completion_markers(fake, view.binding) == {f"sources/tracked/{view.plan_source.id}.done"}+ assert len(self._staged_files(fake, view.binding)) == 1++ def test_completed_source_is_not_repeated_on_retry(self) -> None:+ fake = _FakeS3()+ view = self._create_view_with_two_sources(fake)+ first_counts = self._run_source(fake, view.binding, str(view.plan_source.id))+ self._stage_rows(fake, view.binding, [{"external_id": "acct-1", "plan": "starter", "tier": "gold"}])++ retry_counts = self._run_source(fake, view.binding, str(view.plan_source.id))++ assert first_counts["written"] == 1+ assert retry_counts == _EMPTY_COUNTS+ assert self._run_states(view.plan_source)["tracked"] == _COMPLETED_ONE_ROW+ assert self._snapshot(fake, view.binding, view.plan_source) == {"acct-1": _value_hash("enterprise")}++ @parameterized.expand(+ [+ ("other_view", AccountPropertySourceRejectedError),+ ("malformed_id", AccountPropertySourceRejectedError),+ ("other_team", None),+ ("deleted", None),+ ]+ )+ def test_source_ids_outside_this_view_are_not_processed(+ self, case: str, expected_error: type[Exception] | None+ ) -> None:+ fake = _FakeS3()+ view = self._create_view_with_two_sources(fake)+ other_view = DataWarehouseSavedQuery.objects.create(team=self.team, name="other_accounts", columns={})+ other_view_source = self._create_source(saved_query=other_view, name="Seats", column="seats")+ other_team_source = self._create_source(team=Team.objects.create(organization=self.organization))+ for source in (other_view_source, other_team_source):+ self._start_runs(source)+ source_ids = {+ "other_view": str(other_view_source.id),+ "malformed_id": "not-a-source-id",+ "other_team": str(other_team_source.id),+ "deleted": str(uuid4()),+ }++ if expected_error is None:+ assert self._run_source(fake, view.binding, source_ids[case], final_attempt=True) == _EMPTY_COUNTS+ else:+ with pytest.raises(expected_error):+ self._run_source(fake, view.binding, source_ids[case], final_attempt=True)++ running = {"tracked": _STAGING, "ignored": _STAGING}+ for source in (view.plan_source, view.tier_source, other_view_source, other_team_source):+ assert self._run_states(source) == running+ assert self._completion_markers(fake, view.binding) == set()+ assert [key for key in fake.store if "account_property_snapshot" in key] == []++ @parameterized.expand([("disabled", "is_enabled", False), ("column_removed", "source_column", None)])+ def test_ineligible_source_finishes_only_its_own_run_without_changes(+ self, _case: str, field: str, value: object+ ) -> None:+ fake = _FakeS3()+ view = self._create_view_with_two_sources(fake)+ setattr(view.plan_source, field, value)+ view.plan_source.save(update_fields=[field])++ counts = self._run_source(fake, view.binding, str(view.plan_source.id))++ assert counts == _EMPTY_COUNTS+ assert self._run_states(view.plan_source) == {+ "tracked": _COMPLETED_NO_ROWS,+ "ignored": _STAGING,+ }+ assert self._run_states(view.tier_source)["tracked"] == _STAGING+ assert self._snapshot(fake, view.binding, view.plan_source) == {}++ @parameterized.expand([("retryable", False, "running"), ("final_attempt", True, "failed")])+ def test_source_failure_keeps_completed_siblings_and_staged_rows(+ self, _case: str, final_attempt: bool, expected_status: str+ ) -> None:+ fake = _FakeS3()+ view = self._create_view_with_two_sources(fake)+ pending_source = self._create_source(saved_query=view.plan_source.saved_query, name="Seats", column="seats")+ self._start_runs(pending_source)+ self._run_source(fake, view.binding, str(view.tier_source.id))++ with (+ patch(f"{_MODULE}._iter_parquet_row_batches", side_effect=OSError("S3 unavailable")),+ pytest.raises(OSError, match="S3 unavailable"),+ ):+ self._run_source(fake, view.binding, str(view.plan_source.id), final_attempt=final_attempt)++ assert self._run_states(view.plan_source)["tracked"].status == expected_status+ assert self._run_states(view.plan_source)["ignored"].status == "running"+ assert self._run_states(view.tier_source)["tracked"] == _COMPLETED_ONE_ROW+ assert self._run_states(pending_source)["tracked"] == _STAGING+ assert self._completion_markers(fake, view.binding) == {f"sources/tracked/{view.tier_source.id}.done"}+ assert len(self._staged_files(fake, view.binding)) == 1++ @parameterized.expand([("disabled", "is_enabled", False), ("column_removed", "source_column", None)])+ def test_segment_completion_closes_leftover_runs_and_cleans_up_after_both_segments(+ self, _case: str, field: str, value: object+ ) -> None:+ fake = _FakeS3()+ view = self._create_view_with_two_sources(fake)+ setattr(view.tier_source, field, value)+ view.tier_source.save(update_fields=[field])+ for segment in AccountPropertySyncSegment:+ self._run_source(fake, view.binding, str(view.plan_source.id), segment=segment)++ self._complete_segment(fake, view.binding, AccountPropertySyncSegment.TRACKED)++ assert "tracked.done" in self._completion_markers(fake, view.binding)+ assert len(self._staged_files(fake, view.binding)) == 1++ self._complete_segment(fake, view.binding, AccountPropertySyncSegment.IGNORED)++ assert self._run_states(view.plan_source) == {"tracked": _COMPLETED_ONE_ROW, "ignored": _COMPLETED_ONE_ROW}+ assert self._run_states(view.tier_source) == {"tracked": _COMPLETED_NO_ROWS, "ignored": _COMPLETED_NO_ROWS}+ assert self._staged_files(fake, view.binding) == []++ @parameterized.expand([("not_dispatched",), ("retryable_failure",), ("final_attempt",), ("missing_run",)])+ def test_segment_completion_refuses_a_segment_with_an_incomplete_source(self, case: str) -> None:+ fake = _FakeS3()+ view = self._create_view_with_two_sources(fake)+ for source in (view.plan_source, view.tier_source):+ self._run_source(fake, view.binding, str(source.id), segment=AccountPropertySyncSegment.IGNORED)+ self._complete_segment(fake, view.binding, AccountPropertySyncSegment.IGNORED)+ self._run_source(fake, view.binding, str(view.tier_source.id))+ if case in {"retryable_failure", "final_attempt"}:+ with (+ patch(f"{_MODULE}._iter_parquet_row_batches", side_effect=OSError("S3 unavailable")),+ pytest.raises(OSError),+ ):+ self._run_source(fake, view.binding, str(view.plan_source.id), final_attempt=case == "final_attempt")+ elif case == "missing_run":+ CustomPropertySyncRun.objects.for_team(self.team.id).filter(+ source=view.plan_source, job_id="job-1", segment="tracked"+ ).delete()+ source_runs_before = self._run_states(view.plan_source)++ with pytest.raises(AccountPropertySyncSegmentIncompleteError):+ self._complete_segment(fake, view.binding, AccountPropertySyncSegment.TRACKED)++ markers = self._completion_markers(fake, view.binding)+ assert "ignored.done" in markers+ assert "tracked.done" not in markers+ assert self._run_states(view.plan_source) == source_runs_before+ assert self._run_states(view.tier_source)["tracked"] == _COMPLETED_ONE_ROW+ assert len(self._staged_files(fake, view.binding)) == 1+def test_segments_tracked_and_ignored_accounts_and_excludes_churned_accounts(self) -> None:tracked = create_account(team_id=self.team.id, external_id="tracked")ignored = create_account(team_id=self.team.id, external_id="ignored", ignored_at=datetime.now(UTC))@@ -481,11 +806,11 @@ class _FakeS3:self.times[key] = self._clockasync def _rm(self, paths: str | list[str], *, recursive: bool = False) -> None:- del recursivefor path in [paths] if isinstance(paths, str) else paths:key = aps._s3_key(path)- self.store.pop(key, None)- self.times.pop(key, None)+ for stored_key in [stored for stored in self.store if stored.startswith(key)] if recursive else [key]:+ self.store.pop(stored_key, None)+ self.times.pop(stored_key, None)def _fake_s3_patch(fake: _FakeS3) -> AbstractContextManager[object]: