Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(customer-analytics): add source-scoped account property sync (#119579)

Arthur Moreira de Deus827667b92aa9
diff --git a/products/customer_analytics/backend/logic/account_property_runs.py b/products/customer_analytics/backend/logic/account_property_runs.py
index 32324ca18a3f..61f182530d2b 100644
--- a/products/customer_analytics/backend/logic/account_property_runs.py
+++ b/products/customer_analytics/backend/logic/account_property_runs.py
@@ -40,6 +40,10 @@ class AccountPropertySyncRunOutcome:
_ACCOUNT_SEGMENTS = {segment.value for segment in SyncSegment}
+class AccountPropertySyncSegmentIncompleteError(Exception):
+ pass
+
+
def start_account_property_sync_runs(
context: AccountPropertySyncRunContext,
*,
@@ -127,6 +131,7 @@ def update_account_property_sync_runs_phase(
workflow_run_id: str | None,
attempt: int,
segment: SyncSegment | None = None,
+ source_id: UUID | None = None,
) -> None:
runs = CustomPropertySyncRun.objects.for_team(context.team_id).filter(
job_id=context.job_id,
@@ -135,6 +140,8 @@ def update_account_property_sync_runs_phase(
)
if segment is not None:
runs = runs.filter(segment=segment.value)
+ if source_id is not None:
+ runs = runs.filter(source_id=source_id)
if phase == SyncPhase.DISPATCHING:
runs = runs.filter(phase__in=[SyncPhase.STAGING.value, SyncPhase.DISPATCHING.value])
runs.update(
@@ -199,6 +206,7 @@ def finalize_account_property_sync_runs(
phase: SyncPhase,
error: str | None = None,
segment: SyncSegment | None = None,
+ source_id: UUID | None = None,
) -> None:
with transaction.atomic():
runs = (
@@ -212,6 +220,8 @@ def finalize_account_property_sync_runs(
)
if segment is not None:
runs = runs.filter(segment=segment.value)
+ if source_id is not None:
+ runs = runs.filter(source_id=source_id)
transitioned_source_ids = set(runs.values_list("source_id", flat=True))
runs.update(
status=status.value,
@@ -222,6 +232,44 @@ def finalize_account_property_sync_runs(
_update_source_status_for_terminal_runs(context.team_id, context.job_id, transitioned_source_ids)
+def complete_account_property_sync_segment_runs(context: AccountPropertySyncRunContext, segment: SyncSegment) -> None:
+ with transaction.atomic():
+ runs = list(
+ CustomPropertySyncRun.objects.for_team(context.team_id)
+ .select_for_update()
+ .filter(
+ job_id=context.job_id,
+ saved_query_id=context.saved_query_id,
+ segment=segment.value,
+ )
+ )
+ eligible_source_ids = set(
+ CustomPropertySource.objects.for_team(context.team_id)
+ .filter(
+ saved_query_id=context.saved_query_id,
+ is_enabled=True,
+ definition__target_type=TargetType.ACCOUNT.value,
+ source_column__isnull=False,
+ )
+ .values_list("id", flat=True)
+ )
+ # Staged rows must remain available until every eligible source completes.
+ if eligible_source_ids - {run.source_id for run in runs} or any(
+ run.status == SyncStatus.FAILED.value
+ or (run.source_id in eligible_source_ids and run.status != SyncStatus.COMPLETED.value)
+ for run in runs
+ ):
+ raise AccountPropertySyncSegmentIncompleteError(
+ f"Account-property segment {segment.value} has unfinished or failed source runs"
+ )
+ finalize_account_property_sync_runs(
+ context,
+ status=SyncStatus.COMPLETED,
+ phase=SyncPhase.COMPLETED,
+ segment=segment,
+ )
+
+
def finish_account_property_sync_runs(
context: AccountPropertySyncRunContext,
segment: SyncSegment,
diff --git a/products/customer_analytics/backend/logic/account_property_runs_test.py b/products/customer_analytics/backend/logic/account_property_runs_test.py
index 72649a18a237..53c6fcb19057 100644
--- a/products/customer_analytics/backend/logic/account_property_runs_test.py
+++ b/products/customer_analytics/backend/logic/account_property_runs_test.py
@@ -1,4 +1,6 @@
+from collections.abc import Callable
from datetime import timedelta
+from uuid import UUID
from posthog.test.base import BaseTest
from unittest.mock import patch
@@ -6,6 +8,8 @@ from unittest.mock import patch
from django.apps import apps
from django.utils import timezone
+from parameterized import parameterized
+
from products.customer_analytics.backend.facade import api
from products.customer_analytics.backend.logic.account_property_runs import (
AccountPropertySyncRunContext,
@@ -163,6 +167,63 @@ class TestAccountPropertyRuns(TeamScopedTestMixin, BaseTest):
"00000000-0000-4000-8000-000000000001",
)
+ @parameterized.expand(
+ [
+ (
+ "phase",
+ lambda context, source_id: update_account_property_sync_runs_phase(
+ context,
+ phase=SyncPhase.SYNCING,
+ workflow_id="sync-workflow-job-1-tracked",
+ workflow_run_id="00000000-0000-4000-8000-000000000002",
+ attempt=2,
+ segment=SyncSegment.TRACKED,
+ source_id=source_id,
+ ),
+ ("running", "syncing"),
+ ),
+ (
+ "finalize",
+ lambda context, source_id: finalize_account_property_sync_runs(
+ context,
+ status=SyncStatus.FAILED,
+ phase=SyncPhase.SYNCING,
+ error="Couldn't update accounts.",
+ segment=SyncSegment.TRACKED,
+ source_id=source_id,
+ ),
+ ("failed", "syncing"),
+ ),
+ ]
+ )
+ def test_source_scoped_updates_leave_sibling_sources_untouched(
+ self,
+ _case: str,
+ update: Callable[[AccountPropertySyncRunContext, UUID], None],
+ expected: tuple[str, str],
+ ) -> None:
+ sibling = CustomPropertySource.objects.create(
+ team=self.team,
+ definition=create_custom_property_definition(team_id=self.team.id, name="Tier"),
+ saved_query=self.saved_query,
+ key_column="external_id",
+ source_column="tier",
+ )
+ self._start_runs()
+
+ update(self.context, self.source.id)
+
+ states = {
+ (run.source_id, run.segment): (run.status, run.phase)
+ for run in CustomPropertySyncRun.objects.for_team(self.team.id).filter(job_id="job-1")
+ }
+ assert states == {
+ (self.source.id, "tracked"): expected,
+ (self.source.id, "ignored"): ("running", "staging"),
+ (sibling.id, "tracked"): ("running", "staging"),
+ (sibling.id, "ignored"): ("running", "staging"),
+ }
+
def test_account_runs_remain_active_during_their_workflow_timeout(self) -> None:
self._start_runs()
CustomPropertySyncRun.objects.for_team(self.team.id).filter(source=self.source).update(
diff --git a/products/customer_analytics/backend/logic/account_property_sync.py b/products/customer_analytics/backend/logic/account_property_sync.py
index 2008ab9c4ef5..3deff65b2a97 100644
--- a/products/customer_analytics/backend/logic/account_property_sync.py
+++ b/products/customer_analytics/backend/logic/account_property_sync.py
@@ -10,6 +10,8 @@ from enum import StrEnum
from typing import Any, Protocol
from uuid import UUID
+from django.db.models import QuerySet
+
import pyarrow as pa
import structlog
import pyarrow.parquet as pq
@@ -21,6 +23,7 @@ from posthog.sync import database_sync_to_async
from products.customer_analytics.backend.logic.account_property_runs import (
AccountPropertySyncRunContext,
AccountPropertySyncRunOutcome,
+ complete_account_property_sync_segment_runs,
finalize_account_property_sync_runs,
finish_account_property_sync_runs,
)
@@ -48,6 +51,7 @@ logger = structlog.get_logger(__name__)
_ACCOUNT_LOOKUP_CHUNK_SIZE = 1_000
_PARQUET_BATCH_SIZE = 50_000
_SEGMENTS_REQUIRED_FOR_CLEANUP = frozenset({"tracked", "ignored"})
+_COUNT_KEYS = ("rows_read", "changed", "matched", "written", "source_errors", "source_count")
_RUN_FAILED_ERROR = "Couldn't update accounts. Run the source view again. If it keeps failing, contact support."
_INVALID_VALUE_ERROR = (
"A warehouse value doesn't match this property's type. Update the source data, then run the view again."
@@ -69,6 +73,14 @@ class AccountPropertySourceValueError(Exception):
self.counts = counts.copy()
+class AccountPropertySourceRejectedError(Exception):
+ pass
+
+
+def _empty_counts() -> dict[str, int]:
+ return dict.fromkeys(_COUNT_KEYS, 0)
+
+
@frozen
class AppliedSourceValues:
written: int
@@ -323,19 +335,47 @@ def _apply_source_values(
return AppliedSourceValues(written=written, hashes=applied_hashes, failed=source_failed)
+def _enabled_sources_queryset(team_id: int, binding: WarehouseBinding) -> QuerySet[CustomPropertySource]:
+ return CustomPropertySource.objects.for_team(team_id).filter(
+ saved_query_id=binding.id,
+ is_enabled=True,
+ definition__target_type=TargetType.ACCOUNT.value,
+ source_column__isnull=False,
+ )
+
+
def _enabled_sources(team_id: int, binding: WarehouseBinding) -> list[CustomPropertySource]:
- return list(
- CustomPropertySource.objects.for_team(team_id)
- .select_related("definition")
- .filter(
- saved_query_id=binding.id,
- is_enabled=True,
- definition__target_type=TargetType.ACCOUNT.value,
- source_column__isnull=False,
- )
+ return list(_enabled_sources_queryset(team_id, binding).select_related("definition"))
+
+
+def get_account_property_sync_source_ids(team_id: int, binding: WarehouseBinding) -> list[str]:
+ source_ids = _enabled_sources_queryset(team_id, binding).order_by("id").values_list("id", flat=True)
+ return [str(source_id) for source_id in source_ids]
+
+
+def _is_source_applicable(source: CustomPropertySource) -> bool:
+ return (
+ source.is_enabled
+ and source.definition.target_type == TargetType.ACCOUNT.value
+ and source.source_column is not None
)
+def _parse_source_id(source_id: str) -> UUID:
+ try:
+ return UUID(source_id)
+ except ValueError as error:
+ raise AccountPropertySourceRejectedError("Account-property source ID is not a UUID") from error
+
+
+def _resolve_source(team_id: int, binding: WarehouseBinding, source_id: UUID) -> CustomPropertySource | None:
+ sources = CustomPropertySource.objects.for_team(team_id).select_related("definition")
+ source = sources.filter(id=source_id, saved_query_id=binding.id).first()
+ if source is None and sources.filter(id=source_id).exists():
+ raise AccountPropertySourceRejectedError("Account-property source belongs to a different view")
+ return source
+
+
async def _segment_already_completed(
team_id: int, binding: WarehouseBinding, job_id: str, segment: AccountPropertySyncSegment
) -> bool:
@@ -348,6 +388,41 @@ async def _segment_already_completed(
return True
+def _source_completion_uri(
+ team_id: int, binding: WarehouseBinding, job_id: str, segment: AccountPropertySyncSegment, source_id: UUID
+) -> str:
+ prefix = account_property_completion_prefix(team_id, binding, job_id)
+ # Source markers stay in a subdirectory, so the cleanup barrier never reads them as segment markers.
+ return f"s3://{prefix}/sources/{segment.value}/{source_id}.done"
+
+
+async def _read_source_completion(
+ team_id: int, binding: WarehouseBinding, job_id: str, segment: AccountPropertySyncSegment, source_id: UUID
+) -> dict[str, int] | None:
+ async with aget_s3_client() as s3_client:
+ try:
+ data = await s3_client._cat_file(_source_completion_uri(team_id, binding, job_id, segment, source_id))
+ except FileNotFoundError:
+ return None
+ stored = json.loads(data)
+ return {key: int(stored.get(key, 0)) for key in _COUNT_KEYS}
+
+
+async def _write_source_completion(
+ team_id: int,
+ binding: WarehouseBinding,
+ job_id: str,
+ segment: AccountPropertySyncSegment,
+ source_id: UUID,
+ counts: dict[str, int],
+) -> None:
+ async with aget_s3_client() as s3_client:
+ await s3_client._pipe_file(
+ _source_completion_uri(team_id, binding, job_id, segment, source_id),
+ json.dumps(counts, sort_keys=True).encode("utf-8"),
+ )
+
+
async def _mark_completed_and_maybe_cleanup(
team_id: int, binding: WarehouseBinding, job_id: str, segment: AccountPropertySyncSegment
) -> None:
@@ -372,6 +447,167 @@ async def _mark_completed_and_maybe_cleanup(
pass
+async def _load_prior_hashes(
+ *,
+ team_id: int,
+ binding: WarehouseBinding,
+ segment: AccountPropertySyncSegment,
+ states: list[SourceSyncState],
+ log: FilteringBoundLogger,
+) -> None:
+ phase_started_at = asyncio.get_running_loop().time()
+ for state in states:
+ state.prior_hashes.update(await _read_snapshot_hashes(team_id, binding, str(state.source.id), segment))
+ _record_phase_duration(
+ log,
+ segment,
+ AccountPropertySyncPhase.LOAD_STATE,
+ phase_started_at,
+ {"source_count": len(states)},
+ )
+
+
+async def _apply_staged_rows(
+ *,
+ team_id: int,
+ binding: WarehouseBinding,
+ job_id: str,
+ segment: AccountPropertySyncSegment,
+ states: list[SourceSyncState],
+ counts: dict[str, int],
+ log: FilteringBoundLogger,
+) -> None:
+ batch_index = 0
+ batches = aiter(_iter_parquet_row_batches(team_id, binding, job_id))
+ while True:
+ phase_started_at = asyncio.get_running_loop().time()
+ try:
+ rows = await anext(batches)
+ except StopAsyncIteration:
+ break
+ _record_phase_duration(
+ log,
+ segment,
+ AccountPropertySyncPhase.READ_STAGED_ROWS,
+ phase_started_at,
+ {"batch_index": batch_index, "batch_rows": len(rows)},
+ )
+ counts["rows_read"] += len(rows)
+ for state in states:
+ state.rows_read += len(rows)
+ source = state.source
+ source_column = source.source_column
+ if source_column is None:
+ continue
+
+ phase_started_at = asyncio.get_running_loop().time()
+ values_by_external_id = _source_values(rows, source.key_column, source_column)
+ changed = {
+ external_id: value
+ for external_id, value in values_by_external_id.items()
+ if state.prior_hashes.get(external_id) != _value_hash(value)
+ }
+ state.changed += len(changed)
+ counts["changed"] += len(changed)
+ phase_details: dict[str, bool | float | int | str] = {
+ "batch_index": batch_index,
+ "source_id": str(source.id),
+ "candidate_values": len(values_by_external_id),
+ "changed_values": len(changed),
+ }
+ _record_phase_duration(
+ log,
+ segment,
+ AccountPropertySyncPhase.DIFF_VALUES,
+ phase_started_at,
+ phase_details,
+ )
+ if not changed:
+ continue
+
+ phase_started_at = asyncio.get_running_loop().time()
+ account_ids = await database_sync_to_async(_matching_account_ids, thread_sensitive=False)(
+ team_id, segment, list(changed)
+ )
+ state.matched += len(account_ids)
+ counts["matched"] += len(account_ids)
+ _record_phase_duration(
+ log,
+ segment,
+ AccountPropertySyncPhase.MATCH_ACCOUNTS,
+ phase_started_at,
+ {**phase_details, "matched_accounts": len(account_ids)},
+ )
+
+ phase_started_at = asyncio.get_running_loop().time()
+ applied = await database_sync_to_async(_apply_source_values, thread_sensitive=False)(
+ team_id, source, account_ids, changed, segment
+ )
+ state.written += applied.written
+ counts["written"] += applied.written
+ if applied.failed:
+ state.error = _INVALID_VALUE_ERROR
+ state.applied_hashes.update(applied.hashes)
+ state.prior_hashes.update(applied.hashes)
+ _record_phase_duration(
+ log,
+ segment,
+ AccountPropertySyncPhase.APPLY_VALUES,
+ phase_started_at,
+ {
+ **phase_details,
+ "matched_accounts": len(account_ids),
+ "written_values": applied.written,
+ "source_failed": applied.failed,
+ },
+ )
+ batch_index += 1
+
+
+async def _persist_states(
+ *,
+ team_id: int,
+ binding: WarehouseBinding,
+ job_id: str,
+ segment: AccountPropertySyncSegment,
+ states: list[SourceSyncState],
+ counts: dict[str, int],
+ log: FilteringBoundLogger,
+) -> None:
+ phase_started_at = asyncio.get_running_loop().time()
+ persisted_hashes = 0
+ for state in states:
+ if state.error is not None:
+ counts["source_errors"] += 1
+ await _write_snapshot_hashes(
+ team_id,
+ binding,
+ str(state.source.id),
+ segment,
+ job_id,
+ state.applied_hashes,
+ )
+ persisted_hashes += len(state.applied_hashes)
+ _record_phase_duration(
+ log,
+ segment,
+ AccountPropertySyncPhase.PERSIST_STATE,
+ phase_started_at,
+ {"source_count": len(states), "persisted_hashes": persisted_hashes},
+ )
+
+
+def _run_outcome(state: SourceSyncState, error: str | None) -> AccountPropertySyncRunOutcome:
+ return AccountPropertySyncRunOutcome(
+ source_id=state.source.id,
+ rows_read=state.rows_read,
+ changed=state.changed,
+ matched=state.matched,
+ written=state.written,
+ error=error,
+ )
+
+
async def run_account_property_segment_sync(
*,
team_id: int,
@@ -386,7 +622,7 @@ async def run_account_property_segment_sync(
job_id=job_id,
segment=segment.value,
)
- counts = {"rows_read": 0, "changed": 0, "matched": 0, "written": 0, "source_errors": 0, "source_count": 0}
+ counts = _empty_counts()
states: list[SourceSyncState] = []
run_context = AccountPropertySyncRunContext(
@@ -404,140 +640,19 @@ async def run_account_property_segment_sync(
SourceSyncState(source=source, prior_hashes={}) for source in sources if source.source_column is not None
]
counts["source_count"] = len(states)
- phase_started_at = asyncio.get_running_loop().time()
- for state in states:
- state.prior_hashes.update(await _read_snapshot_hashes(team_id, binding, str(state.source.id), segment))
- _record_phase_duration(
- log,
- segment,
- AccountPropertySyncPhase.LOAD_STATE,
- phase_started_at,
- {"source_count": len(states)},
+ await _load_prior_hashes(team_id=team_id, binding=binding, segment=segment, states=states, log=log)
+ await _apply_staged_rows(
+ team_id=team_id, binding=binding, job_id=job_id, segment=segment, states=states, counts=counts, log=log
)
-
- batch_index = 0
- batches = aiter(_iter_parquet_row_batches(team_id, binding, job_id))
- while True:
- phase_started_at = asyncio.get_running_loop().time()
- try:
- rows = await anext(batches)
- except StopAsyncIteration:
- break
- _record_phase_duration(
- log,
- segment,
- AccountPropertySyncPhase.READ_STAGED_ROWS,
- phase_started_at,
- {"batch_index": batch_index, "batch_rows": len(rows)},
- )
- counts["rows_read"] += len(rows)
- for state in states:
- state.rows_read += len(rows)
- source = state.source
- source_column = source.source_column
- if source_column is None:
- continue
-
- phase_started_at = asyncio.get_running_loop().time()
- values_by_external_id = _source_values(rows, source.key_column, source_column)
- changed = {
- external_id: value
- for external_id, value in values_by_external_id.items()
- if state.prior_hashes.get(external_id) != _value_hash(value)
- }
- state.changed += len(changed)
- counts["changed"] += len(changed)
- phase_details: dict[str, bool | float | int | str] = {
- "batch_index": batch_index,
- "source_id": str(source.id),
- "candidate_values": len(values_by_external_id),
- "changed_values": len(changed),
- }
- _record_phase_duration(
- log,
- segment,
- AccountPropertySyncPhase.DIFF_VALUES,
- phase_started_at,
- phase_details,
- )
- if not changed:
- continue
-
- phase_started_at = asyncio.get_running_loop().time()
- account_ids = await database_sync_to_async(_matching_account_ids, thread_sensitive=False)(
- team_id, segment, list(changed)
- )
- state.matched += len(account_ids)
- counts["matched"] += len(account_ids)
- _record_phase_duration(
- log,
- segment,
- AccountPropertySyncPhase.MATCH_ACCOUNTS,
- phase_started_at,
- {**phase_details, "matched_accounts": len(account_ids)},
- )
-
- phase_started_at = asyncio.get_running_loop().time()
- applied = await database_sync_to_async(_apply_source_values, thread_sensitive=False)(
- team_id, source, account_ids, changed, segment
- )
- state.written += applied.written
- counts["written"] += applied.written
- if applied.failed:
- state.error = _INVALID_VALUE_ERROR
- state.applied_hashes.update(applied.hashes)
- state.prior_hashes.update(applied.hashes)
- _record_phase_duration(
- log,
- segment,
- AccountPropertySyncPhase.APPLY_VALUES,
- phase_started_at,
- {
- **phase_details,
- "matched_accounts": len(account_ids),
- "written_values": applied.written,
- "source_failed": applied.failed,
- },
- )
- batch_index += 1
-
- phase_started_at = asyncio.get_running_loop().time()
- persisted_hashes = 0
- for state in states:
- if state.error is not None:
- counts["source_errors"] += 1
- await _write_snapshot_hashes(
- team_id,
- binding,
- str(state.source.id),
- segment,
- job_id,
- state.applied_hashes,
- )
- persisted_hashes += len(state.applied_hashes)
- _record_phase_duration(
- log,
- segment,
- AccountPropertySyncPhase.PERSIST_STATE,
- phase_started_at,
- {"source_count": len(states), "persisted_hashes": persisted_hashes},
+ await _persist_states(
+ team_id=team_id, binding=binding, job_id=job_id, segment=segment, states=states, counts=counts, log=log
)
except Exception:
if final_attempt:
await database_sync_to_async(finish_account_property_sync_runs)(
run_context,
segment,
- [
- AccountPropertySyncRunOutcome(
- source_id=state.source.id,
- rows_read=state.rows_read,
- changed=state.changed,
- matched=state.matched,
- written=state.written,
- error=_RUN_FAILED_ERROR,
- )
- for state in states
- ],
+ [_run_outcome(state, _RUN_FAILED_ERROR) for state in states],
)
await database_sync_to_async(finalize_account_property_sync_runs)(
run_context,
@@ -551,17 +666,7 @@ async def run_account_property_segment_sync(
await database_sync_to_async(finish_account_property_sync_runs)(
run_context,
segment,
- [
- AccountPropertySyncRunOutcome(
- source_id=state.source.id,
- rows_read=state.rows_read,
- changed=state.changed,
- matched=state.matched,
- written=state.written,
- error=state.error,
- )
- for state in states
- ],
+ [_run_outcome(state, state.error) for state in states],
)
await database_sync_to_async(finalize_account_property_sync_runs)(
run_context,
@@ -578,3 +683,91 @@ async def run_account_property_segment_sync(
await _mark_completed_and_maybe_cleanup(team_id, binding, job_id, segment)
return counts
+
+
+async def run_account_property_source_sync(
+ *,
+ team_id: int,
+ binding: WarehouseBinding,
+ job_id: str,
+ segment: AccountPropertySyncSegment,
+ source_id: str,
+ final_attempt: bool = False,
+) -> dict[str, int]:
+ source_uuid = _parse_source_id(source_id)
+ log = logger.bind(
+ team_id=team_id,
+ saved_query_id=binding.id,
+ job_id=job_id,
+ segment=segment.value,
+ source_id=str(source_uuid),
+ )
+ counts = _empty_counts()
+ state: SourceSyncState | None = None
+ run_context = AccountPropertySyncRunContext(
+ team_id=team_id,
+ saved_query_id=binding.id,
+ job_id=job_id,
+ )
+
+ try:
+ if await _segment_already_completed(team_id, binding, job_id, segment):
+ return counts
+ if await _read_source_completion(team_id, binding, job_id, segment, source_uuid) is not None:
+ return counts
+
+ source = await database_sync_to_async(_resolve_source)(team_id, binding, source_uuid)
+ if source is None:
+ # Deletion cascades to the source's runs, so nothing is left to finish.
+ log.warning("Account-property source not found for this team, skipping it")
+ return counts
+
+ state = SourceSyncState(source=source, prior_hashes={})
+ if _is_source_applicable(source):
+ counts["source_count"] = 1
+ states = [state]
+ await _load_prior_hashes(team_id=team_id, binding=binding, segment=segment, states=states, log=log)
+ await _apply_staged_rows(
+ team_id=team_id, binding=binding, job_id=job_id, segment=segment, states=states, counts=counts, log=log
+ )
+ await _persist_states(
+ team_id=team_id, binding=binding, job_id=job_id, segment=segment, states=states, counts=counts, log=log
+ )
+ else:
+ log.info("Account-property source is no longer eligible, finishing it without changes")
+ except AccountPropertySourceRejectedError:
+ raise
+ except Exception:
+ if final_attempt:
+ failed_outcome = (
+ _run_outcome(state, _RUN_FAILED_ERROR)
+ if state is not None
+ else AccountPropertySyncRunOutcome(
+ source_id=source_uuid, rows_read=0, changed=0, matched=0, written=0, error=_RUN_FAILED_ERROR
+ )
+ )
+ await database_sync_to_async(finish_account_property_sync_runs)(run_context, segment, [failed_outcome])
+ raise
+
+ await database_sync_to_async(finish_account_property_sync_runs)(
+ run_context, segment, [_run_outcome(state, state.error)]
+ )
+ if state.error is not None:
+ raise AccountPropertySourceValueError("An account-property source contained invalid values", counts=counts)
+
+ await _write_source_completion(team_id, binding, job_id, segment, source_uuid, counts)
+ return counts
+
+
+async def complete_account_property_sync_segment(
+ *,
+ team_id: int,
+ binding: WarehouseBinding,
+ job_id: str,
+ segment: AccountPropertySyncSegment,
+) -> None:
+ await database_sync_to_async(complete_account_property_sync_segment_runs)(
+ AccountPropertySyncRunContext(team_id=team_id, saved_query_id=binding.id, job_id=job_id),
+ segment,
+ )
+ await _mark_completed_and_maybe_cleanup(team_id, binding, job_id, segment)
diff --git a/products/customer_analytics/backend/logic/account_property_sync_test.py b/products/customer_analytics/backend/logic/account_property_sync_test.py
index 1fe4071452e6..fd6d9563ca28 100644
--- a/products/customer_analytics/backend/logic/account_property_sync_test.py
+++ b/products/customer_analytics/backend/logic/account_property_sync_test.py
@@ -1,8 +1,8 @@
from collections.abc import AsyncIterator
-from contextlib import AbstractContextManager
+from contextlib import AbstractContextManager, ExitStack
from datetime import UTC, datetime
-from typing import Any, Generic, TypeVar
-from uuid import uuid4
+from typing import TYPE_CHECKING, Any, Generic, TypeVar
+from uuid import UUID, uuid4
import pytest
from posthog.test.base import BaseTest
@@ -13,13 +13,19 @@ from django.apps import apps
import pyarrow as pa
import pyarrow.parquet as pq
from asgiref.sync import async_to_sync
+from parameterized import parameterized
+
+from posthog.dataclasses import frozen
+from posthog.models import Team
from products.customer_analytics.backend.logic import account_property_sync as aps
from products.customer_analytics.backend.logic.account_property_runs import (
AccountPropertySyncRunContext,
+ AccountPropertySyncSegmentIncompleteError,
start_account_property_sync_runs,
)
from products.customer_analytics.backend.logic.account_property_sync import (
+ AccountPropertySourceRejectedError,
AccountPropertySourceValueError,
AccountPropertySyncPhase,
AccountPropertySyncSegment,
@@ -33,37 +39,356 @@ from products.customer_analytics.backend.logic.account_property_sync import (
_source_values,
_value_hash,
_write_snapshot_hashes,
+ complete_account_property_sync_segment,
+ get_account_property_sync_source_ids,
run_account_property_segment_sync,
+ run_account_property_source_sync,
)
from products.customer_analytics.backend.models import CustomPropertySource, CustomPropertySyncRun
from products.customer_analytics.backend.models.team_scoped_test_base import TeamScopedTestMixin
from products.customer_analytics.backend.test.factories import create_account, create_custom_property_definition
-from products.warehouse_sources.backend.facade.hooks import saved_query_binding
+from products.warehouse_sources.backend.facade.hooks import WarehouseBinding, saved_query_binding
from products.warehouse_sources.backend.facade.temporal import (
+ account_property_completion_prefix,
account_property_job_staged_prefix,
account_property_snapshot_prefix,
)
+if TYPE_CHECKING:
+ from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery as SavedQueryModel
+
_MODULE = "products.customer_analytics.backend.logic.account_property_sync"
DataWarehouseSavedQuery = apps.get_model("data_modeling", "DataWarehouseSavedQuery")
+_ACCOUNT_IDS = {"acct-1": UUID("00000000-0000-4000-8000-0000000000a1")}
+_EMPTY_COUNTS = {"rows_read": 0, "changed": 0, "matched": 0, "written": 0, "source_errors": 0, "source_count": 0}
+
+
+@frozen
+class _RunState:
+ status: str
+ phase: str
+ rows_read: int
+ produced: int
+
+
+_STAGING = _RunState(status="running", phase="staging", rows_read=0, produced=0)
+_COMPLETED_ONE_ROW = _RunState(status="completed", phase="completed", rows_read=1, produced=1)
+_COMPLETED_NO_ROWS = _RunState(status="completed", phase="completed", rows_read=0, produced=0)
+
+
+@frozen
+class _ViewWithTwoSources:
+ binding: WarehouseBinding
+ plan_source: CustomPropertySource
+ tier_source: CustomPropertySource
+
+
+def _match_known_accounts(
+ team_id: int, segment: AccountPropertySyncSegment, external_ids: list[str]
+) -> dict[str, UUID]:
+ return {external_id: _ACCOUNT_IDS[external_id] for external_id in external_ids if external_id in _ACCOUNT_IDS}
+
+
+def _apply_all_values(
+ team_id: int,
+ source: CustomPropertySource,
+ account_ids: dict[str, UUID],
+ changed: dict[str, Any],
+ segment: AccountPropertySyncSegment,
+) -> AppliedSourceValues:
+ hashes = {external_id: _value_hash(changed[external_id]) for external_id in account_ids}
+ return AppliedSourceValues(written=len(hashes), hashes=hashes, failed=False)
class AccountPropertySegmentTest(TeamScopedTestMixin, BaseTest):
- def _create_source(self) -> CustomPropertySource:
- saved_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="accounts",
- columns={"external_id": {}, "plan": {}},
- )
- definition = create_custom_property_definition(team_id=self.team.id, name="Plan")
- return CustomPropertySource.objects.create(
- team=self.team,
+ def _create_source(
+ self,
+ *,
+ saved_query: "SavedQueryModel | None" = None,
+ team: Team | None = None,
+ name: str = "Plan",
+ column: str = "plan",
+ ) -> CustomPropertySource:
+ team = team or self.team
+ if saved_query is None:
+ saved_query = DataWarehouseSavedQuery.objects.create(
+ team=team,
+ name="accounts",
+ columns={"external_id": {}, "plan": {}, "tier": {}},
+ )
+ definition = create_custom_property_definition(team_id=team.id, name=name)
+ return CustomPropertySource.objects.unscoped().create(
+ team=team,
definition=definition,
saved_query=saved_query,
key_column="external_id",
- source_column="plan",
+ source_column=column,
+ )
+
+ def _start_runs(self, source: CustomPropertySource) -> None:
+ start_account_property_sync_runs(
+ AccountPropertySyncRunContext(
+ team_id=source.team_id,
+ saved_query_id=str(source.saved_query_id),
+ job_id="job-1",
+ ),
+ workflow_id="stage-workflow-job-1",
+ workflow_run_id="00000000-0000-4000-8000-000000000001",
)
+ def _run_states(self, source: CustomPropertySource) -> dict[str, _RunState]:
+ states: dict[str, _RunState] = {}
+ for run in CustomPropertySyncRun.objects.for_team(source.team_id).filter(source=source, job_id="job-1"):
+ assert run.segment is not None
+ assert run.phase is not None
+ states[run.segment] = _RunState(
+ status=run.status, phase=run.phase, rows_read=run.rows_read, produced=run.produced
+ )
+ return states
+
+ def _create_view_with_two_sources(self, fake: "_FakeS3", plan: str = "enterprise") -> _ViewWithTwoSources:
+ plan_source = self._create_source()
+ tier_source = self._create_source(saved_query=plan_source.saved_query, name="Tier", column="tier")
+ self._start_runs(plan_source)
+ binding = saved_query_binding(str(plan_source.saved_query_id))
+ self._stage_rows(fake, binding, [{"external_id": "acct-1", "plan": plan, "tier": "gold"}])
+ return _ViewWithTwoSources(binding=binding, plan_source=plan_source, tier_source=tier_source)
+
+ def _stage_rows(self, fake: "_FakeS3", binding: WarehouseBinding, rows: list[dict[str, Any]]) -> None:
+ buffer = pa.BufferOutputStream()
+ pq.write_table(pa.Table.from_pylist(rows), buffer)
+ path = f"{account_property_job_staged_prefix(self.team.id, binding, 'job-1')}/chunk-0.parquet"
+ async_to_sync(fake._pipe_file)(path, buffer.getvalue().to_pybytes())
+
+ def _fanout_boundaries(self, fake: "_FakeS3") -> ExitStack:
+ stack = ExitStack()
+ stack.enter_context(_fake_s3_patch(fake))
+ stack.enter_context(patch(f"{_MODULE}._matching_account_ids", side_effect=_match_known_accounts))
+ stack.enter_context(patch(f"{_MODULE}._apply_source_values", side_effect=_apply_all_values))
+ return stack
+
+ def _run_source(
+ self,
+ fake: "_FakeS3",
+ binding: WarehouseBinding,
+ source_id: str,
+ *,
+ segment: AccountPropertySyncSegment = AccountPropertySyncSegment.TRACKED,
+ final_attempt: bool = False,
+ ) -> dict[str, int]:
+ with self._fanout_boundaries(fake):
+ return async_to_sync(run_account_property_source_sync)(
+ team_id=self.team.id,
+ binding=binding,
+ job_id="job-1",
+ segment=segment,
+ source_id=source_id,
+ final_attempt=final_attempt,
+ )
+
+ def _complete_segment(
+ self, fake: "_FakeS3", binding: WarehouseBinding, segment: AccountPropertySyncSegment
+ ) -> None:
+ with self._fanout_boundaries(fake):
+ async_to_sync(complete_account_property_sync_segment)(
+ team_id=self.team.id, binding=binding, job_id="job-1", segment=segment
+ )
+
+ def _snapshot(self, fake: "_FakeS3", binding: WarehouseBinding, source: CustomPropertySource) -> dict[str, str]:
+ with _fake_s3_patch(fake):
+ return async_to_sync(_read_snapshot_hashes)(
+ self.team.id, binding, str(source.id), AccountPropertySyncSegment.TRACKED
+ )
+
+ def _completion_markers(self, fake: "_FakeS3", binding: WarehouseBinding) -> set[str]:
+ prefix = f"{account_property_completion_prefix(self.team.id, binding, 'job-1')}/"
+ return {key.removeprefix(prefix) for key in fake.store if key.startswith(prefix)}
+
+ def _staged_files(self, fake: "_FakeS3", binding: WarehouseBinding) -> list[str]:
+ prefix = account_property_job_staged_prefix(self.team.id, binding, "job-1")
+ return [key for key in fake.store if key.startswith(prefix)]
+
+ def test_source_ids_are_this_views_eligible_sources_in_a_stable_order(self) -> None:
+ sources = [self._create_source()]
+ view = sources[0].saved_query
+ assert view is not None
+ sources.append(self._create_source(saved_query=view, name="Tier", column="tier"))
+ disabled = self._create_source(saved_query=view, name="Region", column="region")
+ disabled.is_enabled = False
+ disabled.save(update_fields=["is_enabled"])
+ other_view = DataWarehouseSavedQuery.objects.create(team=self.team, name="other_accounts", columns={})
+ self._create_source(saved_query=other_view, name="Seats", column="seats")
+ other_team = Team.objects.create(organization=self.organization)
+ self._create_source(saved_query=view, team=other_team, name="Plan", column="plan")
+
+ source_ids = get_account_property_sync_source_ids(self.team.id, saved_query_binding(str(view.id)))
+
+ assert source_ids == sorted(str(source.id) for source in sources)
+
+ def test_source_sync_updates_only_its_own_source_and_segment(self) -> None:
+ fake = _FakeS3()
+ view = self._create_view_with_two_sources(fake)
+
+ counts = self._run_source(fake, view.binding, str(view.plan_source.id))
+
+ assert counts == {**_EMPTY_COUNTS, "rows_read": 1, "changed": 1, "matched": 1, "written": 1, "source_count": 1}
+ assert self._run_states(view.plan_source) == {
+ "tracked": _COMPLETED_ONE_ROW,
+ "ignored": _STAGING,
+ }
+ assert self._run_states(view.tier_source) == {
+ "tracked": _STAGING,
+ "ignored": _STAGING,
+ }
+ assert self._snapshot(fake, view.binding, view.plan_source) == {"acct-1": _value_hash("enterprise")}
+ assert self._snapshot(fake, view.binding, view.tier_source) == {}
+ assert self._completion_markers(fake, view.binding) == {f"sources/tracked/{view.plan_source.id}.done"}
+ assert len(self._staged_files(fake, view.binding)) == 1
+
+ def test_completed_source_is_not_repeated_on_retry(self) -> None:
+ fake = _FakeS3()
+ view = self._create_view_with_two_sources(fake)
+ first_counts = self._run_source(fake, view.binding, str(view.plan_source.id))
+ self._stage_rows(fake, view.binding, [{"external_id": "acct-1", "plan": "starter", "tier": "gold"}])
+
+ retry_counts = self._run_source(fake, view.binding, str(view.plan_source.id))
+
+ assert first_counts["written"] == 1
+ assert retry_counts == _EMPTY_COUNTS
+ assert self._run_states(view.plan_source)["tracked"] == _COMPLETED_ONE_ROW
+ assert self._snapshot(fake, view.binding, view.plan_source) == {"acct-1": _value_hash("enterprise")}
+
+ @parameterized.expand(
+ [
+ ("other_view", AccountPropertySourceRejectedError),
+ ("malformed_id", AccountPropertySourceRejectedError),
+ ("other_team", None),
+ ("deleted", None),
+ ]
+ )
+ def test_source_ids_outside_this_view_are_not_processed(
+ self, case: str, expected_error: type[Exception] | None
+ ) -> None:
+ fake = _FakeS3()
+ view = self._create_view_with_two_sources(fake)
+ other_view = DataWarehouseSavedQuery.objects.create(team=self.team, name="other_accounts", columns={})
+ other_view_source = self._create_source(saved_query=other_view, name="Seats", column="seats")
+ other_team_source = self._create_source(team=Team.objects.create(organization=self.organization))
+ for source in (other_view_source, other_team_source):
+ self._start_runs(source)
+ source_ids = {
+ "other_view": str(other_view_source.id),
+ "malformed_id": "not-a-source-id",
+ "other_team": str(other_team_source.id),
+ "deleted": str(uuid4()),
+ }
+
+ if expected_error is None:
+ assert self._run_source(fake, view.binding, source_ids[case], final_attempt=True) == _EMPTY_COUNTS
+ else:
+ with pytest.raises(expected_error):
+ self._run_source(fake, view.binding, source_ids[case], final_attempt=True)
+
+ running = {"tracked": _STAGING, "ignored": _STAGING}
+ for source in (view.plan_source, view.tier_source, other_view_source, other_team_source):
+ assert self._run_states(source) == running
+ assert self._completion_markers(fake, view.binding) == set()
+ assert [key for key in fake.store if "account_property_snapshot" in key] == []
+
+ @parameterized.expand([("disabled", "is_enabled", False), ("column_removed", "source_column", None)])
+ def test_ineligible_source_finishes_only_its_own_run_without_changes(
+ self, _case: str, field: str, value: object
+ ) -> None:
+ fake = _FakeS3()
+ view = self._create_view_with_two_sources(fake)
+ setattr(view.plan_source, field, value)
+ view.plan_source.save(update_fields=[field])
+
+ counts = self._run_source(fake, view.binding, str(view.plan_source.id))
+
+ assert counts == _EMPTY_COUNTS
+ assert self._run_states(view.plan_source) == {
+ "tracked": _COMPLETED_NO_ROWS,
+ "ignored": _STAGING,
+ }
+ assert self._run_states(view.tier_source)["tracked"] == _STAGING
+ assert self._snapshot(fake, view.binding, view.plan_source) == {}
+
+ @parameterized.expand([("retryable", False, "running"), ("final_attempt", True, "failed")])
+ def test_source_failure_keeps_completed_siblings_and_staged_rows(
+ self, _case: str, final_attempt: bool, expected_status: str
+ ) -> None:
+ fake = _FakeS3()
+ view = self._create_view_with_two_sources(fake)
+ pending_source = self._create_source(saved_query=view.plan_source.saved_query, name="Seats", column="seats")
+ self._start_runs(pending_source)
+ self._run_source(fake, view.binding, str(view.tier_source.id))
+
+ with (
+ patch(f"{_MODULE}._iter_parquet_row_batches", side_effect=OSError("S3 unavailable")),
+ pytest.raises(OSError, match="S3 unavailable"),
+ ):
+ self._run_source(fake, view.binding, str(view.plan_source.id), final_attempt=final_attempt)
+
+ assert self._run_states(view.plan_source)["tracked"].status == expected_status
+ assert self._run_states(view.plan_source)["ignored"].status == "running"
+ assert self._run_states(view.tier_source)["tracked"] == _COMPLETED_ONE_ROW
+ assert self._run_states(pending_source)["tracked"] == _STAGING
+ assert self._completion_markers(fake, view.binding) == {f"sources/tracked/{view.tier_source.id}.done"}
+ assert len(self._staged_files(fake, view.binding)) == 1
+
+ @parameterized.expand([("disabled", "is_enabled", False), ("column_removed", "source_column", None)])
+ def test_segment_completion_closes_leftover_runs_and_cleans_up_after_both_segments(
+ self, _case: str, field: str, value: object
+ ) -> None:
+ fake = _FakeS3()
+ view = self._create_view_with_two_sources(fake)
+ setattr(view.tier_source, field, value)
+ view.tier_source.save(update_fields=[field])
+ for segment in AccountPropertySyncSegment:
+ self._run_source(fake, view.binding, str(view.plan_source.id), segment=segment)
+
+ self._complete_segment(fake, view.binding, AccountPropertySyncSegment.TRACKED)
+
+ assert "tracked.done" in self._completion_markers(fake, view.binding)
+ assert len(self._staged_files(fake, view.binding)) == 1
+
+ self._complete_segment(fake, view.binding, AccountPropertySyncSegment.IGNORED)
+
+ assert self._run_states(view.plan_source) == {"tracked": _COMPLETED_ONE_ROW, "ignored": _COMPLETED_ONE_ROW}
+ assert self._run_states(view.tier_source) == {"tracked": _COMPLETED_NO_ROWS, "ignored": _COMPLETED_NO_ROWS}
+ assert self._staged_files(fake, view.binding) == []
+
+ @parameterized.expand([("not_dispatched",), ("retryable_failure",), ("final_attempt",), ("missing_run",)])
+ def test_segment_completion_refuses_a_segment_with_an_incomplete_source(self, case: str) -> None:
+ fake = _FakeS3()
+ view = self._create_view_with_two_sources(fake)
+ for source in (view.plan_source, view.tier_source):
+ self._run_source(fake, view.binding, str(source.id), segment=AccountPropertySyncSegment.IGNORED)
+ self._complete_segment(fake, view.binding, AccountPropertySyncSegment.IGNORED)
+ self._run_source(fake, view.binding, str(view.tier_source.id))
+ if case in {"retryable_failure", "final_attempt"}:
+ with (
+ patch(f"{_MODULE}._iter_parquet_row_batches", side_effect=OSError("S3 unavailable")),
+ pytest.raises(OSError),
+ ):
+ self._run_source(fake, view.binding, str(view.plan_source.id), final_attempt=case == "final_attempt")
+ elif case == "missing_run":
+ CustomPropertySyncRun.objects.for_team(self.team.id).filter(
+ source=view.plan_source, job_id="job-1", segment="tracked"
+ ).delete()
+ source_runs_before = self._run_states(view.plan_source)
+
+ with pytest.raises(AccountPropertySyncSegmentIncompleteError):
+ self._complete_segment(fake, view.binding, AccountPropertySyncSegment.TRACKED)
+
+ markers = self._completion_markers(fake, view.binding)
+ assert "ignored.done" in markers
+ assert "tracked.done" not in markers
+ assert self._run_states(view.plan_source) == source_runs_before
+ assert self._run_states(view.tier_source)["tracked"] == _COMPLETED_ONE_ROW
+ assert len(self._staged_files(fake, view.binding)) == 1
+
def test_segments_tracked_and_ignored_accounts_and_excludes_churned_accounts(self) -> None:
tracked = create_account(team_id=self.team.id, external_id="tracked")
ignored = create_account(team_id=self.team.id, external_id="ignored", ignored_at=datetime.now(UTC))
@@ -481,11 +806,11 @@ class _FakeS3:
self.times[key] = self._clock
async def _rm(self, paths: str | list[str], *, recursive: bool = False) -> None:
- del recursive
for path in [paths] if isinstance(paths, str) else paths:
key = aps._s3_key(path)
- self.store.pop(key, None)
- self.times.pop(key, None)
+ for stored_key in [stored for stored in self.store if stored.startswith(key)] if recursive else [key]:
+ self.store.pop(stored_key, None)
+ self.times.pop(stored_key, None)
def _fake_s3_patch(fake: _FakeS3) -> AbstractContextManager[object]: